From 34a3df7290331629b4b341e0a1cf9db181661dd7 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:11:49 +0000 Subject: [PATCH] Box tooling from master b4214735 (build-remote routes by class and load across build-1 and build-2, --box N pins, infra/build-server with it): the pool lane's branch sits on release-0.3.20 for the ladder's igneum-pow, and a full merge of master conflicts in eight files that are the shipper's merge (bench-log, fud-ledger, testnet-go, restart-hand-nodes, ledger.html, litepaper.html, miner.html, pre-push.sh) Co-Authored-By: Claude Fable 5.1 --- infra/build-server/README.md | 37 +++++++ infra/build-server/capacity/run.sh | 5 + infra/build-server/hands/install-hands.sh | 36 +++++-- infra/build-server/hands/move-hand.sh | 86 +++++++++++++-- infra/build-server/lib.sh | 123 +++++++++++++++++++--- infra/build-server/provision.sh | 42 ++++++-- infra/build-server/remote-run.sh | 118 +++++++++++++++++++-- infra/build-server/run-from-mac.sh | 29 +++-- infra/build-server/workers/install.sh | 13 ++- tools/build-remote.sh | 111 +++++++++++++++++-- tools/cross-remote.sh | 5 + tools/workers-remote.sh | 28 +++-- 12 files changed, 563 insertions(+), 70 deletions(-) create mode 100644 infra/build-server/README.md diff --git a/infra/build-server/README.md b/infra/build-server/README.md new file mode 100644 index 000000000..49f7ba106 --- /dev/null +++ b/infra/build-server/README.md @@ -0,0 +1,37 @@ +# The build boxes (infra/build-server) + +Three Hetzner dedicated servers in Falkenstein run everything the Mac must not: builds, test suites, benchmarks, CPU proving, the +devnet hands and the observer. The Mac keeps macOS binaries, the DMG and Metal tests (CLAUDE.md, "Running agents on this Mac"). + +## No mining on any Hetzner box, ever + +the project lead's rule through main, 7 October 2026: Hetzner's policies forbid crypto mining. The boxes run nodes, builds, tests, benchmarks and +CPU proving only. The pool's fast-time network runs its miners on rented GPU pods (tools/fleet), never on a box; a box may run the +network's nodes. The capacity layer refuses a job that would start `igneum-miner mine` or a GPU worker (capacity/run.sh), and no +hands unit carries a miner. A node started with `--enable-unsynced-mining` is a node flag, not a miner; nothing feeds it blocks here. + +## The boxes and the kind map + +| Box | Host file on the Mac | Takes | Never | +|---|---|---|---| +| igneum-build-1 (188.40.146.49, AX162-1-LTD) | `~/.config/igneum/build-server` | release gates (`--priority gate`), builds and cross-builds, checks, the GPU workers' host side, the devnet hands (node 1, the observer node, the observer), the Devnet 2 seed, the CI runner, the dashboard feed | suites and benches once box 2 exists | +| igneum-build-2 (AX162-1, on order) | `~/.config/igneum/build-server-2` | suites (`cargo test`), benches (`cargo bench`), the attack rows (`--box 2`) | gates, hands | +| igneum-build-3 (AX102-1, on order) | `~/.config/igneum/build-server-3` | proving and aggregation CPU work (proving/igneum-prove builds and suites), the second prover's shadow runner, the pool's fast-time NETWORK (nodes only, `--box 3`) | miners of any kind | + +`tools/build-remote.sh` routes by class (lib.sh `bs_route`): suite and bench to box 2, the proving crate to box 3, everything else +to box 1; `--box N` overrides; a class whose box has no host file yet falls back to box 1 and says so. `--priority gate` always runs +on box 1. Each box has its own mirrors, slots, locks and JSONL log under /srv; `run-from-mac.sh --box N ` provisions a box and +writes its host file; the dashboard collector reads every box it is told about. + +## Files + +| File | What | +|---|---| +| `provision.sh` | the box itself: install mode (rescue system, Ubuntu 24.04, RAID 1, no swap) and provision mode (user build, toolchains, the pin, sccache, zig, CUDA headers, docker, Caddy, mirrors, slots, sshd, ufw) | +| `run-from-mac.sh` | ships provision.sh, writes the host file, wires the `build` remotes and pushes every branch | +| `lib.sh`, `remote-run.sh` | the Mac and box halves of a remote run: sync, checkout, slots, scheduling classes, the JSONL line | +| `hands/` | the devnet hands' units, the mover and the restart read-backs | +| `capacity/` | the capacity layer (the box-work lane's): background jobs under the build slots, never a miner | +| `repro/`, `night/`, `prover/`, `runner/`, `workers/` | other lanes' pieces that live on the boxes | + +Plan, numbers and the gotchas: docs/plans/build-server.md; the hands: docs/plans/hands-on-build-1.md. diff --git a/infra/build-server/capacity/run.sh b/infra/build-server/capacity/run.sh index d522c59b6..95eb92f61 100755 --- a/infra/build-server/capacity/run.sh +++ b/infra/build-server/capacity/run.sh @@ -58,6 +58,11 @@ run_slice() { once() { for job in $SEQUENCE; do + # No mining on any Hetzner box, ever (the project lead through main, 7 October 2026; Hetzner's policies forbid it): a job that would start a + # miner or a GPU worker is refused here, whatever SEQUENCE says. Nodes, builds, tests, benchmarks and CPU proving only. + if grep -qE 'igneum-miner[[:space:]]+mine|igneum-worker-(cuda|opencl)|igneum-app.*--mine|cargo run.*-p[[:space:]]+igneum-miner' "$JOBS_DIR/$job.sh" 2>/dev/null; then + echo "capacity: REFUSED job $job: it would start a miner or a GPU worker; no mining on a Hetzner box (infra/build-server/README.md)" >&2; continue + fi [ -f "$JOBS_DIR/$job.sh" ] || { cap_say "no job $job"; continue; } # wait out a running build before starting a slice (do not even launch during a build) while cap_build_active; do diff --git a/infra/build-server/hands/install-hands.sh b/infra/build-server/hands/install-hands.sh index d48fc3675..885e54f83 100755 --- a/infra/build-server/hands/install-hands.sh +++ b/infra/build-server/hands/install-hands.sh @@ -68,16 +68,40 @@ IFS=',' read -r -a peers <<< "${PEERS:-}"; for p in "${peers[@]}"; do [ -n "$p" [ -n "${EXTRA_ARGS_OBSERVER:-}" ] && args+=($EXTRA_ARGS_OBSERVER) exec "$IGNEUMD" "${args[@]}" RUN +# the read-only deploy key (main, 7 October 2026): made HERE as user build, the private half never leaves this box and is never +# printed; the project lead adds the public half as a read-only deploy key on github.com/igneum-network/igneum (steps in +# docs/plans/build-server.md, "Deploy key"). Until GitHub accepts it, observer-sync.sh follows the mirror. +install -d -m 700 -o $U -g $U $O/.ssh +if [ ! -f $O/.ssh/deploy_igneum ]; then su - $U -c "ssh-keygen -q -t ed25519 -N '' -C 'igneum-build-1 observer read-only' -f $O/.ssh/deploy_igneum"; log "deploy key created at $O/.ssh/deploy_igneum (public half: $O/.ssh/deploy_igneum.pub)"; else log "deploy key ok"; fi +chmod 600 $O/.ssh/deploy_igneum; chmod 644 $O/.ssh/deploy_igneum.pub +install -d -m 700 -o $U -g $U /home/$U/.ssh +if ! grep -q '^Host github-igneum-observer' /home/$U/.ssh/config 2>/dev/null; then + printf 'Host github-igneum-observer\n HostName github.com\n User git\n IdentityFile %s/.ssh/deploy_igneum\n IdentitiesOnly yes\n StrictHostKeyChecking accept-new\n' "$O" >> /home/$U/.ssh/config + chown $U:$U /home/$U/.ssh/config; chmod 600 /home/$U/.ssh/config; log "ssh alias github-igneum-observer written" +fi + cat > $H/bin/observer-sync.sh <<'RUN' #!/usr/bin/env bash -# the box's tools/observer/autosync.sh: fast-forward /srv/observer/igneum from the mirror /srv/igneum.git (fed by every -# build-remote.sh and run-from-mac.sh push from the Mac), restart igneum-observer when tools/observer or site/lib changed. +# the box's tools/observer/autosync.sh: fast-forward /srv/observer/igneum, then restart igneum-observer when tools/observer or +# site/lib changed. Source: GitHub through the read-only deploy key (ssh alias github-igneum-observer, remote `github`) once +# the project lead has added the public half; until then, or when GitHub refuses, the mirror /srv/igneum.git (fed by every build-remote.sh +# and run-from-mac.sh push from the Mac). One line says which. set -uo pipefail cd /srv/observer/igneum || exit 1 -before=$(git rev-parse HEAD:tools/observer HEAD:site/lib 2>/dev/null | tr '\n' ' ') -su - build -c "git -C /srv/observer/igneum pull -q --ff-only origin master" 2>&1 | head -2 -after=$(git rev-parse HEAD:tools/observer HEAD:site/lib 2>/dev/null | tr '\n' ' ') -if [ "$before" != "$after" ]; then echo "observer files changed ($(git rev-parse --short HEAD)); restarting igneum-observer"; systemctl restart igneum-observer; else echo "observer up to date at $(git rev-parse --short HEAD)"; fi +g() { su - build -c "git -C /srv/observer/igneum $*"; } # the clone is build's; root's git refuses it (safe.directory), so every git call runs as build +before=$(g rev-parse HEAD:tools/observer HEAD:site/lib 2>/dev/null | tr '\n' ' ') +# `ssh -T` to GitHub exits 1 after its greeting, so under this script's pipefail a `su ... | grep -q` reported failure although grep +# had matched (7 Oct 2026: the pass said "mirror" while the same line by hand said authenticated); the output is captured first +probe=$(su - build -c "ssh -o BatchMode=yes -o ConnectTimeout=10 -T git@github-igneum-observer" 2>&1 || true) +if grep -q "successfully authenticated" <<<"$probe"; then + su - build -c "git -C /srv/observer/igneum remote get-url github >/dev/null 2>&1 || git -C /srv/observer/igneum remote add github git@github-igneum-observer:igneum-network/igneum.git" + if su - build -c "git -C /srv/observer/igneum pull -q --ff-only github master" 2>&1 | head -2; then echo "source: github (deploy key accepted)"; else echo "source: github refused the pull, mirror next"; su - build -c "git -C /srv/observer/igneum pull -q --ff-only origin master" 2>&1 | head -2; fi +else + echo "source: mirror (the deploy key is not accepted by GitHub yet: docs/plans/build-server.md, Deploy key)" + su - build -c "git -C /srv/observer/igneum pull -q --ff-only origin master" 2>&1 | head -2 +fi +after=$(g rev-parse HEAD:tools/observer HEAD:site/lib 2>/dev/null | tr '\n' ' ') +if [ "$before" != "$after" ]; then echo "observer files changed ($(g rev-parse --short HEAD)); restarting igneum-observer"; systemctl restart igneum-observer; else echo "observer up to date at $(g rev-parse --short HEAD)"; fi RUN chmod 755 $H/bin/*.sh; chown $U:$U $H/bin/*.sh diff --git a/infra/build-server/hands/move-hand.sh b/infra/build-server/hands/move-hand.sh index 170a076d0..b917731fc 100755 --- a/infra/build-server/hands/move-hand.sh +++ b/infra/build-server/hands/move-hand.sh @@ -12,6 +12,13 @@ # Mac's run.sh + autosync + observer.mjs, start igneum-observer (step 3) # infra/build-server/hands/move-hand.sh node1 [--go] the same as observer-node for node 1 (step 4) # infra/build-server/hands/move-hand.sh unload [--go] bootout the Mac's two launchd agents for good (step 5, last) +# infra/build-server/hands/move-hand.sh restart observer-node|node1 [--digest ] [--go] +# a release on the box (7 Oct 2026, 0.3.17): after `binary` +# installed the new igneumd and the override, restart ONE +# unit and read it back: first executing line, commit string +# of the installed binary, digest (against --digest when +# given, else the unit's own last digest), igneum_getNodeInfo +# powEngine over the node's loopback EVM RPC # infra/build-server/hands/move-hand.sh status both sides: units, pids, tips, peers # # Needs ~/.config/igneum/build-server (build@) and the ops key; root ssh to the box for systemctl, scp of the env file and chown. @@ -23,6 +30,9 @@ REPO="$(cd "$HERE/../../.." && pwd)" BS_TOOL=move-hand # shellcheck source=../lib.sh . "$HERE/../lib.sh" + +{ # whole-body: bash parses this block entirely before running a line of it, so an edit to this file while a run is in + # flight cannot reach the running copy (7 Oct 2026: build-remote.sh was edited mid-run and died on shifted bytes after a 4-min build) bs_host IP="${BS_HOST#*@}" ROOT_SSH=(ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new "root@$IP") @@ -31,11 +41,20 @@ MAC_SNAP=/tmp/igneum-devnet/node1-copy-snapshot.bin MAC_SNAP_SHA=ac101f13576179fd7d7f5e8ee902c9a7b6cc47730e3a3c069f389f0ca46d9221 # the launchd agents' value (6 Oct 2026); recomputed below H=/srv/hands MODE="${1:-}"; shift || true -GO=0; NODE_WT=""; OV_JSON="" -while [ $# -gt 0 ]; do case "$1" in --go) GO=1; shift ;; --node) NODE_WT="$2"; shift 2 ;; --override-json) OV_JSON="$2"; shift 2 ;; *) bs_die "unknown argument $1" ;; esac; done +GO=0; NODE_WT=""; OV_JSON=""; WANT_DIGEST=""; HAND="" +while [ $# -gt 0 ]; do case "$1" in --go) GO=1; shift ;; --node) NODE_WT="$2"; shift 2 ;; --override-json) OV_JSON="$2"; shift 2 ;; --digest) WANT_DIGEST="$2"; shift 2 ;; *) if [ "$MODE" = restart ] && [ -z "$HAND" ]; then HAND="$1"; shift; else bs_die "unknown argument $1"; fi ;; esac; done say() { bs_log "$*"; } run() { if [ "$GO" = 1 ]; then "$@"; else say "DRY RUN: $*"; fi; } rssh() { "${ROOT_SSH[@]}" "$@"; } +# the digest readback (main, 6 Oct 2026 23:xx UK): the box hand's "Consensus params digest" against the Mac hand's last one +# tolerant pipelines: a missing line gives an empty string, never a failed command substitution that ends the script under set -e +# (7 Oct 2026: the restart dry run died silently because the unit's digest line was older than the 10-minute window) +mac_digest() { { grep 'Consensus params digest' "$HOME/Library/Logs/Igneum/$1.out" 2>/dev/null | tail -1 | grep -oE '[0-9a-f]{64}' | head -1; } || true; } +box_digest() { { rssh "journalctl -u $1 --no-pager -o cat --since '7 days ago' | grep 'Consensus params digest' | tail -1" 2>/dev/null | grep -oE '[0-9a-f]{64}' | head -1; } || true; } +digest_readback() { # + local b m; b=$(box_digest "$1"); m=$(mac_digest "$2") + if [ -n "$b" ] && [ "$b" = "$m" ]; then say "$1 digest ${b:0:16}... MATCHES the Mac's $2 hand"; else say "$1 digest ${b:-none} against the Mac's ${m:-none}: DIFFER (stop and read the override before moving the next hand)"; return 1; fi +} first_exec_line() { # : wait up to 180 s for the node's exec line, print it (the proof main wants) and the chain tip local u="$1" line="" for _ in $(seq 1 36); do @@ -50,7 +69,7 @@ sync_dir() { # : rsync a data dir (hot or final), keeping R } mac_stop_agent() { #