diff --git a/tools/ci/README.md b/tools/ci/README.md index 1eb1093a6..a2bf50e4f 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -4,3 +4,4 @@ |---|---|---| | `pgrep-self-match-check.sh` | a `pgrep -f` / `pkill -f` with a bare literal pattern, or `ps \| grep ` without a bracket or `grep -v grep`, in tools/, relay/playbooks/, infra/ or packaging/: the pattern matches the shell that runs it (the wave script of 6 October 2026 never started a node on 38 cards because `pgrep -f igneumd-0313` saw the launching shell; a kill file killed its caller the same day). Anchor to the executable's path, bracket the first letter, or use `-x`. Owed (allow-listed, finished measurements): `tools/prover-floor/pc2-*.ps1`, `tools/proving-v1/pc2-*.ps1` (their `pkill -f sp1-gpu-server` becomes `pkill -x`), `tools/repo/fresh-repo.sh:223`, `tools/observer/autosync.sh:22`, and the live devnet's operational scripts the fleet agent does not own: `infra/devnet/restart-hand-nodes.sh`, `infra/seed-nodes/addpeer-from-mac.sh`, `relay/playbooks/shard-test.ps1`, `tools/ci/fixtures/bash-body-ok.ps1`, `tools/ci/pgrep-self-match-check.sh`, `tools/ci/prover-socket-check.sh`, `tools/exec-attacks/net.sh` (their owners anchor the pattern when next touched; the hand-node and seed scripts run tonight and were not edited blind) | 6 October 2026, branch gpu-fleet | +| kill by exact command or pid file (owed as a check) | 6 October 2026, 21:09Z: a Mac-side `pkill -f ` matched nothing (the log name was a redirect, not part of the command line), the roll-everything script lived on and wiped a box it had been told to hold. Rule: a job is stopped by its pid file (`tools/fleet/fleet-bg.sh start|stop `) or by a pattern anchored on its exact command line (`^python3 -u /root/fleet/in/box-prover.py`), never by a word that may or may not appear in it. The check that flags a `pkill -f`/`pgrep -f` whose literal is a path or a name that never starts a command line is owed to the CI lane | diff --git a/tools/fleet/box-standing.sh b/tools/fleet/box-standing.sh index 353b7228b..f4d1843b9 100755 --- a/tools/fleet/box-standing.sh +++ b/tools/fleet/box-standing.sh @@ -16,7 +16,7 @@ set -uo pipefail F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; mkdir -p $OUT $F/mine/packs; exec >> $OUT/standing-supervisor.log 2>&1 stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } ROLE="${ROLE:-live}"; LABEL="${LABEL:-standing}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}"; PROVER="${PROVER:-0}"; MINE="${MINE:-1}" -HUB_PEER="${HUB_PEER:-}"; SEED="${SEED:-}"; STARTED=$(date -u +%s) +HUB_PEER="${HUB_PEER:-}"; SEED="${SEED:-}"; P2P_LISTEN="${P2P_LISTEN:-0.0.0.0:26611}"; STARTED=$(date -u +%s) # P2P_LISTEN: a box with a mapped inbound port listens on it (pool-1: 4463) if [ "$ROLE" = dn2 ]; then APP=$F/dn2; LOG=$F/dn2-node.log; OVR=$F/dn2-override.json; SUFFIX="--devnet-suffix=2"; PACK=dn2; NET=igneum-devnet-2 else APP=$F/node; LOG=$F/node.log; OVR=$F/override.json; SUFFIX=""; PACK=devnet; NET=igneum-devnet; fi echo "RESULT standing_start $(stamp) role=$ROLE label=$LABEL prover=$PROVER mine=$MINE" @@ -31,7 +31,7 @@ start_node() { [ "$ROLE" = live ] && peers="$peers --addpeer=188.245.5.161:26611" local ver; ver="$HOST_VERIFIER"; [ -x /opt/igneum-floor/bin/igneum-prove-host ] && ver=/opt/igneum-floor/bin/igneum-prove-host pkill -9 -f "$NODE_RE" 2>/dev/null; sleep 2 - IGNEUM_PROOF_VERIFIER=${ver:-} setsid nohup $nb --devnet $SUFFIX --appdir=$APP --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 $peers --override-params-file=$OVR --nodnsseed --disable-upnp --nologfiles --yes > $LOG 2>&1 & + IGNEUM_PROOF_VERIFIER=${ver:-} setsid nohup $nb --devnet $SUFFIX --appdir=$APP --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=$P2P_LISTEN $peers --override-params-file=$OVR --nodnsseed --disable-upnp --nologfiles --yes > $LOG 2>&1 & sleep 8; echo "RESULT standing_node_started $(stamp) bin=$nb pid=$(node_pid) digest=$(grep -o 'digest: [0-9a-f]*' $LOG | tail -1 | awk '{print substr($2,1,16)}')" # a standalone igneum-miner keeps a dead template subscription after its node restarts (20:24Z: templates frozen, # fetch_errors climbing, no submits), so the miner and its worker are restarted with the node; the loop brings them back diff --git a/tools/fleet/fleet-bg.sh b/tools/fleet/fleet-bg.sh new file mode 100755 index 000000000..4e0db7dc6 --- /dev/null +++ b/tools/fleet/fleet-bg.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Mac-side background jobs by NAME with a pid file, so a job is stopped by its pid and never by a pattern (21:09Z: a +# `pkill -f ` matched nothing, the roll-everything script lived on and wiped a box it had been told to hold). +# tools/fleet/fleet-bg.sh start runs the command detached, stdout+stderr to ~/Desktop/fleet/.log, pid to ~/Desktop/fleet/pids/.pid +# tools/fleet/fleet-bg.sh stop kills that pid (and its process group) if it is alive, removes the pid file +# tools/fleet/fleet-bg.sh list every job and whether its pid is alive +set -u; P="$HOME/Desktop/fleet/pids"; L="$HOME/Desktop/fleet"; mkdir -p "$P" +case "${1:-}" in + start) n="$2"; shift 2; [ -f "$P/$n.pid" ] && kill -0 "$(cat "$P/$n.pid")" 2>/dev/null && { echo "$n already running (pid $(cat "$P/$n.pid"))"; exit 1; } + nohup setsid "$@" > "$L/$n.log" 2>&1 < /dev/null & echo $! > "$P/$n.pid"; echo "$n started pid $(cat "$P/$n.pid") log $L/$n.log" ;; + stop) n="$2"; [ -f "$P/$n.pid" ] || { echo "$n: no pid file"; exit 1; }; pid=$(cat "$P/$n.pid") + if kill -0 "$pid" 2>/dev/null; then kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid"; sleep 2; kill -0 "$pid" 2>/dev/null && kill -KILL -- "-$pid" 2>/dev/null; echo "$n stopped (pid $pid)"; else echo "$n not running (pid $pid gone)"; fi; rm -f "$P/$n.pid" ;; + list) for f in "$P"/*.pid; do [ -f "$f" ] || continue; n=$(basename "$f" .pid); pid=$(cat "$f"); kill -0 "$pid" 2>/dev/null && echo "$n pid $pid alive" || echo "$n pid $pid gone"; done ;; + *) echo "usage: fleet-bg.sh start | stop | list"; exit 2 ;; +esac diff --git a/tools/fleet/publish-0315.py b/tools/fleet/publish-0315.py index fb26ea989..42d240b35 100644 --- a/tools/fleet/publish-0315.py +++ b/tools/fleet/publish-0315.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""Publish 1 of 0.3.15 on the standing live-devnet boxes (the shipper's set, 6 October 2026): the Linux igneumd 06211d55 -(release-0.3.15-node 713ef876), igneum-miner e827d9db, the generator-4 workers (cuda 97e036e2, opencl ef9fbd03); the override +"""Publish 1 of 0.3.15 on the standing live-devnet boxes (the shipper's set, 6 October 2026, 21:4xZ): the Linux igneumd 7f0bde70 +(release-0.3.15-node f1ea7a38; the first two cuts 713ef876 and 7961c5f1 failed the canary), igneum-miner e827d9db, the generator-4 workers (cuda 97e036e2, opencl ef9fbd03); the override file stays the live thirteen-field object (digest b18ed271 on the new binary). Per box, through lib.box: the node binary to /root/fleet/in/igneumd- (sha-checked), the miner and workers over /opt/igneum/pkg/bin (the old ones kept as *.0314), /root/fleet/standing.node rewritten so the supervisor restarts the node on it (data dir kept), the miner killed once so @@ -12,8 +12,8 @@ import sys, os, hashlib sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); from lib import Box, Registry, standing from concurrent.futures import ThreadPoolExecutor SHIP = "/Users/joshm/Projects/igneum-wt-ship0315" -NODE = (SHIP + "/vendor/igneum-node-0315/target-remote/release/igneumd", "06211d55b8cce1c19b31b218f5b61f42389a5ceaf68e0d27781b2e10e33b3f2a") -MINER = (SHIP + "/vendor/igneum-node-0315/target-remote/release/igneum-miner", "e827d9dbe8533ec8") +NODE = (SHIP + "/vendor/igneum-node-0315/target-remote/release/igneumd", "7f0bde70cf2e72a3a9479ba6421f2b37162c3dd4391bf413717c99e6f168668d") # release-0.3.15-node f1ea7a38 (the stamp gate, the receive-side version rule, the pruned-node sync fix) +MINER = (SHIP + "/vendor/igneum-node-0315/target-remote/release/igneum-miner", "3976c1b270761218") CUDA = (SHIP + "/infra/cross/out-workers/igneum-worker-cuda", "97e036e23f4ace66") OCL = (SHIP + "/infra/cross/out-workers/igneum-worker-opencl", "ef9fbd03e7ce14d1") def sha16(p): return hashlib.sha256(open(p, "rb").read()).hexdigest()[:16] @@ -25,7 +25,7 @@ def publish(label): b.put([MINER[0], CUDA[0], OCL[0]], "/root/fleet/in/") rc, out, err = b.run(f"""set -e; cd /opt/igneum/pkg/bin; for f in igneum-miner igneum-worker-cuda igneum-worker-opencl; do [ -e $f.0314 ] || cp $f $f.0314; cp /root/fleet/in/$f $f.new && chmod +x $f.new && mv $f.new $f; done echo {path} > /root/fleet/standing.node -pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve' 2>/dev/null; true +pkill -9 -f '^/(opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir=' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve' 2>/dev/null; true # the supervisor restarts the node on the new pointer within a minute, and the miner with it echo swapped""", 120) return label, out.strip() or err[:120] def readback(label):