From 5523f526bab92cdef5248b31d1d5d4eab2e99a87 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 03:12:51 +0000 Subject: [PATCH] Build tools: whole-body blocks (the edited-while-running class, with its check), the igneum-pow pairing line, move-hand.sh restart with its readbacks Three classes from the 0.3.17 night. (1) bash reads a script incrementally: tools/build-remote.sh was edited while a four-minute remote build ran, the running copy continued at shifted bytes and died with a syntax error after the build had succeeded on the box; the four long-running tools (build-remote, cross-remote, workers-remote, move-hand) now keep their body in one brace block ending in exit, parsed whole before a line runs; tools/ci/whole-body-check.sh (in the gate, self-test with a block-less copy) holds the shape. (2) A fork build pairs with the igneum-pow of the igneum worktree it sits in: a fork at 12153428 under a master worktree failed in kaspa-pow four minutes in (no chain_program_shadow; master's igneum-pow predates release-0.3.17's); build-remote.sh says the pairing on its first line ('pairs with igneum 6f8d7a7e (detached): igneum-pow 0.2.0') and the JSONL line carries pairs_with. The first version of that line used '[ -n ... ] && echo' inside an assignment's $( ) and set -e ended the script on the false status; fixed. (3) move-hand.sh restart [--digest ] [--go]: after binary installed a release, restart ONE unit and read it back (first exec line, commit string in the running binary, digest against the wanted one, igneum_getNodeInfo powEngine over the node's loopback EVM RPC); the digest readers tolerate a missing line. Co-Authored-By: Claude Fable 5.1 --- infra/build-server/hands/move-hand.sh | 39 ++++++++++++++++++++++++--- infra/build-server/lib.sh | 4 +-- infra/build-server/remote-run.sh | 2 +- tools/build-remote.sh | 15 +++++++++++ tools/ci/pre-push.sh | 1 + tools/ci/whole-body-check.sh | 29 ++++++++++++++++++++ tools/cross-remote.sh | 5 ++++ tools/workers-remote.sh | 5 ++++ 8 files changed, 93 insertions(+), 7 deletions(-) create mode 100755 tools/ci/whole-body-check.sh diff --git a/infra/build-server/hands/move-hand.sh b/infra/build-server/hands/move-hand.sh index b9abfe668..d443c0d98 100755 --- a/infra/build-server/hands/move-hand.sh +++ b/infra/build-server/hands/move-hand.sh @@ -12,6 +12,13 @@ # Mac's run.sh + autosync + observer.mjs, start igneum-observer (step 3) # infra/build-server/hands/move-hand.sh node1 [--go] the same as observer-node for node 1 (step 4) # infra/build-server/hands/move-hand.sh unload [--go] bootout the Mac's two launchd agents for good (step 5, last) +# infra/build-server/hands/move-hand.sh restart observer-node|node1 [--digest ] [--go] +# a release on the box (7 Oct 2026, 0.3.17): after `binary` +# installed the new igneumd and the override, restart ONE +# unit and read it back: first executing line, commit string +# of the installed binary, digest (against --digest when +# given, else the unit's own last digest), igneum_getNodeInfo +# powEngine over the node's loopback EVM RPC # infra/build-server/hands/move-hand.sh status both sides: units, pids, tips, peers # # Needs ~/.config/igneum/build-server (build@) and the ops key; root ssh to the box for systemctl, scp of the env file and chown. @@ -23,6 +30,9 @@ REPO="$(cd "$HERE/../../.." && pwd)" BS_TOOL=move-hand # shellcheck source=../lib.sh . "$HERE/../lib.sh" + +{ # whole-body: bash parses this block entirely before running a line of it, so an edit to this file while a run is in + # flight cannot reach the running copy (7 Oct 2026: build-remote.sh was edited mid-run and died on shifted bytes after a 4-min build) bs_host IP="${BS_HOST#*@}" ROOT_SSH=(ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new "root@$IP") @@ -31,14 +41,16 @@ MAC_SNAP=/tmp/igneum-devnet/node1-copy-snapshot.bin MAC_SNAP_SHA=ac101f13576179fd7d7f5e8ee902c9a7b6cc47730e3a3c069f389f0ca46d9221 # the launchd agents' value (6 Oct 2026); recomputed below H=/srv/hands MODE="${1:-}"; shift || true -GO=0; NODE_WT=""; OV_JSON="" -while [ $# -gt 0 ]; do case "$1" in --go) GO=1; shift ;; --node) NODE_WT="$2"; shift 2 ;; --override-json) OV_JSON="$2"; shift 2 ;; *) bs_die "unknown argument $1" ;; esac; done +GO=0; NODE_WT=""; OV_JSON=""; WANT_DIGEST=""; HAND="" +while [ $# -gt 0 ]; do case "$1" in --go) GO=1; shift ;; --node) NODE_WT="$2"; shift 2 ;; --override-json) OV_JSON="$2"; shift 2 ;; --digest) WANT_DIGEST="$2"; shift 2 ;; *) if [ "$MODE" = restart ] && [ -z "$HAND" ]; then HAND="$1"; shift; else bs_die "unknown argument $1"; fi ;; esac; done say() { bs_log "$*"; } run() { if [ "$GO" = 1 ]; then "$@"; else say "DRY RUN: $*"; fi; } rssh() { "${ROOT_SSH[@]}" "$@"; } # the digest readback (main, 6 Oct 2026 23:xx UK): the box hand's "Consensus params digest" against the Mac hand's last one -mac_digest() { grep 'Consensus params digest' "$HOME/Library/Logs/Igneum/$1.out" 2>/dev/null | tail -1 | grep -oE '[0-9a-f]{64}' | head -1; } -box_digest() { rssh "journalctl -u $1 --no-pager -o cat --since '10 min ago' | grep 'Consensus params digest' | tail -1" 2>/dev/null | grep -oE '[0-9a-f]{64}' | head -1; } +# tolerant pipelines: a missing line gives an empty string, never a failed command substitution that ends the script under set -e +# (7 Oct 2026: the restart dry run died silently because the unit's digest line was older than the 10-minute window) +mac_digest() { { grep 'Consensus params digest' "$HOME/Library/Logs/Igneum/$1.out" 2>/dev/null | tail -1 | grep -oE '[0-9a-f]{64}' | head -1; } || true; } +box_digest() { { rssh "journalctl -u $1 --no-pager -o cat --since '7 days ago' | grep 'Consensus params digest' | tail -1" 2>/dev/null | grep -oE '[0-9a-f]{64}' | head -1; } || true; } digest_readback() { # local b m; b=$(box_digest "$1"); m=$(mac_digest "$2") if [ -n "$b" ] && [ "$b" = "$m" ]; then say "$1 digest ${b:0:16}... MATCHES the Mac's $2 hand"; else say "$1 digest ${b:-none} against the Mac's ${m:-none}: DIFFER (stop and read the override before moving the next hand)"; return 1; fi @@ -137,6 +149,23 @@ case "$MODE" in run rssh "systemctl start igneum-observer && sleep 5 && systemctl is-active igneum-observer && journalctl -u igneum-observer --no-pager -o cat -n 6" say "observer moved: /api/live reads Neon, which the box's observer now writes" ;; + restart) + hand="$HAND" + case "$hand" in node1) unit=igneum-node1; evm=26791 ;; observer-node) unit=igneum-observer-node; evm=26840 ;; *) bs_die "restart needs observer-node or node1" ;; esac + bin=$(rssh "readlink $H/bin/igneumd"); sha=$(printf '%s' "$bin" | sed -E 's/.*-([0-9a-f]{7,})$/\1/') + before=$(box_digest "$unit"); want="${WANT_DIGEST:-$before}" + say "$hand: restart $unit on $bin (commit $sha); digest before ${before:-none}, wanted ${want:-any}" + run rssh "systemctl restart $unit && sleep 3 && systemctl is-active $unit" + if [ "$GO" = 1 ]; then + first_exec_line "$unit" + rssh "[ \$(strings $H/bin/igneumd | grep -c '$sha') -gt 0 ] && echo 'commit string $sha present in the running binary' || { echo 'NO commit string $sha in the binary'; exit 1; }" + after=$(box_digest "$unit") + if [ -n "$after" ] && [ "$after" = "$want" ]; then say "$unit digest ${after:0:16}... MATCHES"; else say "$unit digest ${after:-none} against wanted ${want:-any}: DIFFER (stop here)"; exit 1; fi + eng=$(rssh "curl -s --max-time 10 -X POST -H 'content-type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getNodeInfo\",\"params\":[]}' http://127.0.0.1:$evm" | python3 -c 'import json,sys; d=json.load(sys.stdin); r=d.get("result") or {}; print(r.get("powEngine") or r.get("pow_engine") or ("ERROR: "+str(d.get("error")) if d.get("error") else "no powEngine field: "+str(list(r)[:8])))' 2>/dev/null || echo "igneum_getNodeInfo not answered on $evm") + say "$unit igneum_getNodeInfo powEngine: $eng" + case "$eng" in igneum-pow) ;; *) say "WARNING: powEngine is not igneum-pow" ;; esac + fi ;; + unload) say "removing the Mac's launchd agents for good (bootout and the plists moved aside); the hands are on the box" for label in network.igneum.devnet.observer network.igneum.devnet.node1; do @@ -150,3 +179,5 @@ case "$MODE" in echo "--- mac:"; launchctl print "gui/$(id -u)/network.igneum.devnet.node1" 2>/dev/null | grep -E 'state|pid' | head -2; launchctl print "gui/$(id -u)/network.igneum.devnet.observer" 2>/dev/null | grep -E 'state|pid' | head -2; pgrep -fl '[o]bserver.mjs|[o]bserver/run.sh|[a]utosync.sh' || echo "no observer processes on the Mac" ;; *) sed -n '2,20p' "$0"; exit 2 ;; esac +exit 0 +} diff --git a/infra/build-server/lib.sh b/infra/build-server/lib.sh index 5083439d7..9eb8757f7 100755 --- a/infra/build-server/lib.sh +++ b/infra/build-server/lib.sh @@ -277,9 +277,9 @@ bs_remote_run() { label="$label; agent=$agent" BR_DIR="$dir" BR_LABEL="$label" BR_CMD="$cmd" BR_TOOL="${BS_TOOL:-build-remote}" BR_WT="$BS_WT" BR_CRATE="$BS_CRATE_REL" \ BR_BRANCH="$BS_BRANCH" BR_SHA="$BS_SHA" BR_AGENT="$agent" BR_KIND="${BR_KIND:-other}" BR_COMMAND="${BR_COMMAND:-}" \ - BR_TARGET="${BR_TARGET:-}" BR_ARTEFACTS="${BR_ARTEFACTS:-}" BR_SDE="${BR_SDE:-}" \ + BR_TARGET="${BR_TARGET:-}" BR_ARTEFACTS="${BR_ARTEFACTS:-}" BR_SDE="${BR_SDE:-}" BR_PAIRS_WITH="${BR_PAIRS_WITH:-}" \ bash -c ' - for v in BR_DIR BR_LABEL BR_CMD BR_TOOL BR_WT BR_CRATE BR_BRANCH BR_SHA BR_AGENT BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS BR_SDE; do + for v in BR_DIR BR_LABEL BR_CMD BR_TOOL BR_WT BR_CRATE BR_BRANCH BR_SHA BR_AGENT BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS BR_SDE BR_PAIRS_WITH; do printf "export %s=%q\n" "$v" "${!v}" done cat "$0"' "$(dirname "${BASH_SOURCE[0]}")/remote-run.sh" | bs_ssh 'bash -s' diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh index b78b4b655..2a1cdd626 100755 --- a/infra/build-server/remote-run.sh +++ b/infra/build-server/remote-run.sh @@ -196,7 +196,7 @@ d = { "agent": e.get('BR_AGENT'), "slot": num(e['BR_SLOT']), "wait_s": num(e['BR_WAIT']), "queued_at": iso(e['BR_T0']), "start": iso(e['BR_START']), "end": iso(e['BR_END']), "secs": num(e['BR_SECS']), "exit": num(e['BR_EXIT']), "compiles": num(e['BR_COMPILES']), "jobs": num(e.get('BR_JOBS')), "measure": (e.get('BR_MEASURE') == '1') or None, - "source_date_epoch": num(e.get('BR_SDE')), + "source_date_epoch": num(e.get('BR_SDE')), "pairs_with": e.get('BR_PAIRS_WITH') or None, "class": e.get('BR_CLASS') or None, "run_log": e.get('BR_RUN_LOG') or None, } sc = {k: num(e[v]) for k, v in (("hits", "BR_HITS"), ("misses", "BR_MISSES"), ("hits_total", "BR_HITS_T"), ("misses_total", "BR_MISSES_T"))} diff --git a/tools/build-remote.sh b/tools/build-remote.sh index 2eea1bf9b..c8e72f1fa 100755 --- a/tools/build-remote.sh +++ b/tools/build-remote.sh @@ -59,6 +59,9 @@ BS_TOOL=build-remote # shellcheck source=../infra/build-server/lib.sh . "$HERE/../infra/build-server/lib.sh" +{ # whole-body: bash parses this block entirely before running a line of it, so an edit to this file while a run is in + # flight cannot reach the running copy (7 Oct 2026: build-remote.sh was edited mid-run and died on shifted bytes after a 4-min build) + # JOBS empty = the box decides: 90 alone, 45 beside another slot holder (remote-run.sh, main's ruling 6 Oct 2026) JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0; SHIP=0; SHIP_CLASS="${SHIP_CLASS:-seed}"; GLIBC="${GLIBC:-}" while [ $# -gt 0 ]; do @@ -147,6 +150,16 @@ fi [ -n "$OUT" ] || OUT="$BS_CRATE/target-remote" bs_log "$BS_KIND crate $BS_WT/$BS_CRATE_REL at $BS_SHA ($BS_BRANCH) -> $BS_HOST:$BS_REMOTE_CRATE; cargo ${CARGO_ARGS[*]} -j ${JOBS:-auto}; target dir $TARGET_DIR" +# a fork build pairs with the igneum-pow of the igneum worktree it sits in (the fork's path dependency ../../../../igneum-pow), so the +# pairing is said on the first line and recorded (7 Oct 2026, 0.3.17: a fork at 12153428 under a master worktree failed in kaspa-pow +# four minutes in, "no associated function chain_program_shadow", because master's igneum-pow predates the release branch's) +if [ "$BS_KIND" = node ]; then + pair_branch=$(git -C "$BS_WT_ROOT" branch --show-current 2>/dev/null || true); pair_dirty=$(git -C "$BS_WT_ROOT" status --porcelain -- igneum-pow 2>/dev/null || true) + # no `[ ... ] && echo` inside an assignment's $( ): its false status is the assignment's status and set -e ends the script (7 Oct 2026, 03:0x UTC) + PAIR="igneum $(git -C "$BS_WT_ROOT" rev-parse --short HEAD) (${pair_branch:-detached})${pair_dirty:+ with uncommitted igneum-pow changes}" + bs_log "pairs with $PAIR: igneum-pow $(grep -m1 '^version' "$BS_WT_ROOT/igneum-pow/Cargo.toml" | sed 's/.*"\(.*\)".*/\1/')" + export BR_PAIRS_WITH="$PAIR" +fi bs_toolchain_check t_sync0=$(date +%s) bs_sync_sources @@ -193,3 +206,5 @@ if [ "$FETCH" = 1 ] && [ -n "$ARTEFACTS" ]; then done fi bs_wt_unlock +exit 0 +} diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh index 32e4e4b7f..b84a64e94 100755 --- a/tools/ci/pre-push.sh +++ b/tools/ci/pre-push.sh @@ -72,6 +72,7 @@ tree_checks() { run "commit-string gate self-test" bash tools/ci/commit-string-check.sh --self-test run "build server remote checkout self-test" bash infra/build-server/remote-run.sh --self-test run "the remote checkout resets the mirror's tree before the branch checkout (the stale-overlay class)" bash -c 'bash tools/ci/mirror-reset-check.sh --self-test && bash tools/ci/mirror-reset-check.sh' + run "long-running tools keep their body in one parsed block (the edited-while-running class)" bash -c 'bash tools/ci/whole-body-check.sh --self-test && bash tools/ci/whole-body-check.sh' run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh' run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh' run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test diff --git a/tools/ci/whole-body-check.sh b/tools/ci/whole-body-check.sh new file mode 100755 index 000000000..983681055 --- /dev/null +++ b/tools/ci/whole-body-check.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# The edited-while-running class (7 October 2026, 03:0x UTC): bash reads a script incrementally, so editing tools/build-remote.sh while a +# four-minute remote build was in flight made the running copy continue at shifted bytes and die with a syntax error after the build +# had succeeded on the box (the artefacts were never fetched). Rule: a long-running tool's body sits in ONE brace block that ends with +# `exit`, which bash parses entirely before running a line of it, so a later edit cannot reach a running copy. This check reads the +# tools listed below and fails when the line after the library source is not `{ # whole-body` or the file does not end `exit 0` `}`. +# +# tools/ci/whole-body-check.sh # exit 1 with the file and the reason +# tools/ci/whole-body-check.sh --self-test # the real tools pass; a copy with the block removed fails +set -euo pipefail +cd "$(dirname "$0")/../.." +TOOLS=(tools/build-remote.sh tools/cross-remote.sh tools/workers-remote.sh infra/build-server/hands/move-hand.sh) +check() { # + local f="$1" src + src=$(grep -nE '^\. "\$HERE/(\.\./)+(infra/build-server/)?lib\.sh"' "$f" | head -1 | cut -d: -f1) + [ -n "$src" ] || { echo "whole-body: $f: no library source line to anchor the block"; return 1; } + local opener; opener=$(sed -n "$((src + 1)),\$p" "$f" | grep -vE '^\s*$' | head -1) # the next non-blank line + [ "$opener" = '{ # whole-body: bash parses this block entirely before running a line of it, so an edit to this file while a run is in' ] || { echo "whole-body: $f: the first line after the library source is not the whole-body block opener"; return 1; } + [ "$(tail -2 "$f" | head -1)" = 'exit 0' ] && [ "$(tail -1 "$f")" = '}' ] || { echo "whole-body: $f: the file does not end with 'exit 0' and '}'"; return 1; } + echo "whole-body: $f: body in one block (after line $src to the end)" +} +if [ "${1:-}" = --self-test ]; then + t=$(mktemp -d); trap 'rm -rf "$t"' EXIT + for f in "${TOOLS[@]}"; do check "$f" >/dev/null || { echo "whole-body self-test: the real $f FAILED"; exit 1; }; done + grep -vE '^\{ # whole-body|^ # flight cannot reach|^exit 0$|^\}$' tools/cross-remote.sh > "$t/no-block.sh" + if check "$t/no-block.sh" >/dev/null 2>&1; then echo "whole-body self-test: a copy without the block PASSED (blind)"; exit 1; fi + echo "whole-body self-test: the real tools pass; a copy without the block fails"; exit 0 +fi +fail=0; for f in "${TOOLS[@]}"; do check "$f" || fail=1; done; exit $fail diff --git a/tools/cross-remote.sh b/tools/cross-remote.sh index 431137abe..d7a58e973 100755 --- a/tools/cross-remote.sh +++ b/tools/cross-remote.sh @@ -36,6 +36,9 @@ BS_TOOL=cross-remote # shellcheck source=../infra/build-server/lib.sh . "$HERE/../infra/build-server/lib.sh" +{ # whole-body: bash parses this block entirely before running a line of it, so an edit to this file while a run is in + # flight cannot reach the running copy (7 Oct 2026: build-remote.sh was edited mid-run and died on shifted bytes after a 4-min build) + TARGET=x86_64-pc-windows-gnu # JOBS empty = the box decides: 90 alone, 45 beside another slot holder (remote-run.sh, main's ruling 6 Oct 2026). One line, no # trailing comment: a `#` on this line once swallowed every assignment after it, CARGO_ARGS was never declared and the default @@ -126,3 +129,5 @@ if printf '%s\n' "$dlls" | grep -q 'libstdc++-6.dll'; then fi bs_log "exes in $OUT/$TARGET/release" bs_wt_unlock +exit 0 +} diff --git a/tools/workers-remote.sh b/tools/workers-remote.sh index d93b95131..e29e7a859 100755 --- a/tools/workers-remote.sh +++ b/tools/workers-remote.sh @@ -18,6 +18,9 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BS_TOOL=workers-remote # shellcheck source=../infra/build-server/lib.sh . "$HERE/../infra/build-server/lib.sh" + +{ # whole-body: bash parses this block entirely before running a line of it, so an edit to this file while a run is in + # flight cannot reach the running copy (7 Oct 2026: build-remote.sh was edited mid-run and died on shifted bytes after a 4-min build) OUT=""; CLASS="${CLASS:-rig}"; while [ $# -gt 0 ]; do case "$1" in --out) OUT="$2"; shift 2 ;; --class) CLASS="$2"; shift 2 ;; *) bs_die "unknown argument $1" ;; esac; done bs_host # the context by hand (bs_context wants a Cargo.toml): the igneum worktree root is the repo; lib.sh reads these @@ -57,3 +60,5 @@ for b in igneum-worker-cuda igneum-worker-opencl; do done { printf 'igneum workers, linux x86_64, built on igneum-build-1 (glibc %s, static libstdc++) on %s from %s (%s); run-time libraries dlopen-ed\n' "$GLIBC" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$BS_SHA" "$BS_BRANCH"; } > "$OUT/version.txt" bs_wt_unlock +exit 0 +}