diff --git a/infra/build-server/README.md b/infra/build-server/README.md new file mode 100644 index 000000000..49f7ba106 --- /dev/null +++ b/infra/build-server/README.md @@ -0,0 +1,37 @@ +# The build boxes (infra/build-server) + +Three Hetzner dedicated servers in Falkenstein run everything the Mac must not: builds, test suites, benchmarks, CPU proving, the +devnet hands and the observer. The Mac keeps macOS binaries, the DMG and Metal tests (CLAUDE.md, "Running agents on this Mac"). + +## No mining on any Hetzner box, ever + +the project lead's rule through main, 7 October 2026: Hetzner's policies forbid crypto mining. The boxes run nodes, builds, tests, benchmarks and +CPU proving only. The pool's fast-time network runs its miners on rented GPU pods (tools/fleet), never on a box; a box may run the +network's nodes. The capacity layer refuses a job that would start `igneum-miner mine` or a GPU worker (capacity/run.sh), and no +hands unit carries a miner. A node started with `--enable-unsynced-mining` is a node flag, not a miner; nothing feeds it blocks here. + +## The boxes and the kind map + +| Box | Host file on the Mac | Takes | Never | +|---|---|---|---| +| igneum-build-1 (188.40.146.49, AX162-1-LTD) | `~/.config/igneum/build-server` | release gates (`--priority gate`), builds and cross-builds, checks, the GPU workers' host side, the devnet hands (node 1, the observer node, the observer), the Devnet 2 seed, the CI runner, the dashboard feed | suites and benches once box 2 exists | +| igneum-build-2 (AX162-1, on order) | `~/.config/igneum/build-server-2` | suites (`cargo test`), benches (`cargo bench`), the attack rows (`--box 2`) | gates, hands | +| igneum-build-3 (AX102-1, on order) | `~/.config/igneum/build-server-3` | proving and aggregation CPU work (proving/igneum-prove builds and suites), the second prover's shadow runner, the pool's fast-time NETWORK (nodes only, `--box 3`) | miners of any kind | + +`tools/build-remote.sh` routes by class (lib.sh `bs_route`): suite and bench to box 2, the proving crate to box 3, everything else +to box 1; `--box N` overrides; a class whose box has no host file yet falls back to box 1 and says so. `--priority gate` always runs +on box 1. Each box has its own mirrors, slots, locks and JSONL log under /srv; `run-from-mac.sh --box N ` provisions a box and +writes its host file; the dashboard collector reads every box it is told about. + +## Files + +| File | What | +|---|---| +| `provision.sh` | the box itself: install mode (rescue system, Ubuntu 24.04, RAID 1, no swap) and provision mode (user build, toolchains, the pin, sccache, zig, CUDA headers, docker, Caddy, mirrors, slots, sshd, ufw) | +| `run-from-mac.sh` | ships provision.sh, writes the host file, wires the `build` remotes and pushes every branch | +| `lib.sh`, `remote-run.sh` | the Mac and box halves of a remote run: sync, checkout, slots, scheduling classes, the JSONL line | +| `hands/` | the devnet hands' units, the mover and the restart read-backs | +| `capacity/` | the capacity layer (the box-work lane's): background jobs under the build slots, never a miner | +| `repro/`, `night/`, `prover/`, `runner/`, `workers/` | other lanes' pieces that live on the boxes | + +Plan, numbers and the gotchas: docs/plans/build-server.md; the hands: docs/plans/hands-on-build-1.md. diff --git a/infra/build-server/capacity/run.sh b/infra/build-server/capacity/run.sh index d522c59b6..95eb92f61 100755 --- a/infra/build-server/capacity/run.sh +++ b/infra/build-server/capacity/run.sh @@ -58,6 +58,11 @@ run_slice() { once() { for job in $SEQUENCE; do + # No mining on any Hetzner box, ever (the project lead through main, 7 October 2026; Hetzner's policies forbid it): a job that would start a + # miner or a GPU worker is refused here, whatever SEQUENCE says. Nodes, builds, tests, benchmarks and CPU proving only. + if grep -qE 'igneum-miner[[:space:]]+mine|igneum-worker-(cuda|opencl)|igneum-app.*--mine|cargo run.*-p[[:space:]]+igneum-miner' "$JOBS_DIR/$job.sh" 2>/dev/null; then + echo "capacity: REFUSED job $job: it would start a miner or a GPU worker; no mining on a Hetzner box (infra/build-server/README.md)" >&2; continue + fi [ -f "$JOBS_DIR/$job.sh" ] || { cap_say "no job $job"; continue; } # wait out a running build before starting a slice (do not even launch during a build) while cap_build_active; do diff --git a/infra/build-server/lib.sh b/infra/build-server/lib.sh index 04bf429c4..bb754b796 100755 --- a/infra/build-server/lib.sh +++ b/infra/build-server/lib.sh @@ -16,7 +16,19 @@ # shellcheck disable=SC2034 # shared with the scripts that source lib.sh BS_KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}" -BS_HOST_FILE="${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" # one line: build@ +BS_HOST_FILE="${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" # one line: build@ (box 1; box N is build-server-N) +# Several boxes (main, 7 October 2026: igneum-build-2 and igneum-build-3 on order). One host file per box: ~/.config/igneum/build-server +# is box 1, build-server-2 is box 2, build-server-3 is box 3 (run-from-mac.sh --box N writes it). The route by class, unless the +# caller passes --box: gates, builds, checks, cross-builds, the workers, the hands and the observer stay on box 1; suites, benches and +# the attack rows go to box 2; proving and aggregation CPU work, the second prover's shadow runner and the pool's fast-time NETWORK go +# to box 3. A class whose box has no host file yet falls back to box 1, and the log line says so. No box mines, ever (README.md). +bs_box_file() { case "${1:-1}" in 1) echo "$BS_HOST_FILE" ;; *) echo "${BS_HOST_FILE}-$1" ;; esac; } +bs_route() { # -> the box number, falling back to 1 + local want=1 + case "$1" in suite|bench|attack) want=2 ;; prove|shadow|fasttime) want=3 ;; esac + if [ "$want" != 1 ] && [ ! -s "$(bs_box_file "$want")" ]; then bs_log "class $1 routes to box $want, which has no host file yet ($(bs_box_file "$want")): box 1"; want=1; fi + echo "$want" +} BS_ROOT_REMOTE=/srv/builds BS_MIRROR_REPO=/srv/igneum.git BS_MIRROR_NODE=/srv/igneum-node.git @@ -24,11 +36,13 @@ BS_MIRROR_NODE=/srv/igneum-node.git bs_log() { printf '%s %s: %s\n' "$(date -u +%H:%M:%S)" "${BS_TOOL:-build-server}" "$*" >&2; } bs_die() { bs_log "ERROR: $*"; exit 1; } -bs_host() { +bs_host() { # [box number, default BS_BOX or 1] + BS_BOX="${1:-${BS_BOX:-1}}" BS_HOST="${BUILD_HOST:-}" if [ -z "$BS_HOST" ]; then - [ -s "$BS_HOST_FILE" ] || bs_die "no build server: write build@ to $BS_HOST_FILE (infra/build-server/run-from-mac.sh does) or set BUILD_HOST" - BS_HOST="$(head -1 "$BS_HOST_FILE" | tr -d '[:space:]')" + local f; f=$(bs_box_file "$BS_BOX") + [ -s "$f" ] || bs_die "no build server for box $BS_BOX: write build@ to $f (infra/build-server/run-from-mac.sh --box $BS_BOX does) or set BUILD_HOST" + BS_HOST="$(head -1 "$f" | tr -d '[:space:]')" fi case "$BS_HOST" in *@*) ;; *) bs_die "BUILD_HOST must be user@host, got '$BS_HOST'" ;; esac [ -r "$BS_KEY" ] || bs_die "no ssh key at $BS_KEY" diff --git a/infra/build-server/run-from-mac.sh b/infra/build-server/run-from-mac.sh index eeb94ea0e..3b783b1c2 100755 --- a/infra/build-server/run-from-mac.sh +++ b/infra/build-server/run-from-mac.sh @@ -20,9 +20,12 @@ BS_TOOL=run-from-mac # shellcheck source=lib.sh . "$HERE/lib.sh" +BOX=1; while [ "${1:-}" = --box ]; do BOX="$2"; shift 2; done # --box N: igneum-build-N, host file build-server-N (7 Oct 2026) IP="${1:-}"; shift || true -[ -n "$IP" ] || bs_die "usage: run-from-mac.sh [--wire-only]" +[ -n "$IP" ] || bs_die "usage: run-from-mac.sh [--box N] [--wire-only]" WIRE_ONLY=0; [ "${1:-}" = --wire-only ] && WIRE_ONLY=1 +[ "$BOX" = 1 ] || { BOX_HOSTNAME="${BOX_HOSTNAME:-igneum-build-$BOX}"; export BOX_HOSTNAME; } +BS_HOST_FILE=$(bs_box_file "$BOX"); export BS_BOX="$BOX" # lib.sh's bs_host reads BS_BOX # the toolchain pin travels from rust-toolchain.toml unless RUST_TOOLCHAIN is set by hand (one file pins every side, 7 Oct 2026) [ -n "${RUST_TOOLCHAIN:-}" ] || RUST_TOOLCHAIN=$(sed -n 's/^channel *= *"\([^"]*\)".*/\1/p' "$REPO/rust-toolchain.toml" 2>/dev/null | head -1); export RUST_TOOLCHAIN PASS="" # a string, not an array: bash 3.2 (the Mac) treats an empty array as unbound under set -u diff --git a/tools/build-remote.sh b/tools/build-remote.sh index 75d6b0046..05e40320f 100755 --- a/tools/build-remote.sh +++ b/tools/build-remote.sh @@ -75,7 +75,7 @@ BS_TOOL=build-remote # flight cannot reach the running copy (7 Oct 2026: build-remote.sh was edited mid-run and died on shifted bytes after a 4-min build) # JOBS empty = the box decides: 90 alone, 45 beside another slot holder (remote-run.sh, main's ruling 6 Oct 2026) -JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0; SHIP=0; SHIP_CLASS="${SHIP_CLASS:-seed}"; PRIORITY="${PRIORITY:-normal}"; PLAN=0; GLIBC="${GLIBC:-}" +JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0; SHIP=0; SHIP_CLASS="${SHIP_CLASS:-seed}"; PRIORITY="${PRIORITY:-normal}"; PLAN=0; BOX="${BOX:-}"; GLIBC="${GLIBC:-}" while [ $# -gt 0 ]; do case "$1" in --jobs) JOBS="$2"; shift 2 ;; @@ -86,6 +86,7 @@ while [ $# -gt 0 ]; do --self-test-repro) SELFTEST=1; shift ;; --ship) SHIP=1; case "${2:-}" in hive|rig|seed|linux|native) SHIP_CLASS="$2"; shift 2 ;; *) shift ;; esac ;; --priority) PRIORITY="$2"; shift 2 ;; + --box) BOX="$2"; BOX_GIVEN=1; shift 2 ;; --gate) PRIORITY=gate; shift ;; --plan) PLAN=1; shift ;; --glibc) GLIBC="$2"; shift 2 ;; @@ -117,8 +118,15 @@ if [ "$PLAN" = 1 ]; then exit 0 fi -bs_host +# the box: --box N, else the route by class (suite and bench to box 2, prove to box 3, the rest to box 1; lib.sh bs_route) +if [ -z "$BOX" ]; then + case "$SCHED_CLASS:$PRIORITY" in *:gate) BOX=1 ;; suite:*|bench:*) BOX=$(bs_route "$SCHED_CLASS") ;; *) BOX=1 ;; esac +fi +bs_host "$BOX" bs_context +# a proving crate routes to box 3 unless the caller chose (its builds and suites alike) +if [ "$BS_CRATE_REL" = proving/igneum-prove ] && [ -z "${BOX_GIVEN:-}" ] && [ "$PRIORITY" != gate ]; then b=$(bs_route prove); [ "$b" != "$BOX" ] && { BOX=$b; bs_host "$BOX"; }; fi +bs_log "box $BOX ($BS_HOST) for class $SCHED_CLASS, priority $PRIORITY" if [ "$SELFTEST" = 1 ]; then [ "$BS_KIND" = node ] || bs_die "--self-test-repro runs from a fork worktree (igneum-miner and kaspad live there)"