From 27da32d934ec639e9668672cb37f751ff1907e67 Mon Sep 17 00:00:00 2001 From: igneum-josh <337424239+igneum-josh@users.noreply.github.com> Date: Wed, 7 Oct 2026 00:25:10 +0100 Subject: [PATCH] Fleet scripts: process patterns in master's bracket-first form (the kill-by-name check), anchors dropped Co-Authored-By: Claude Fable 5.1 --- tools/fleet/box-dn2.sh | 4 ++-- tools/fleet/box-ember.sh | 2 +- tools/fleet/box-exec-snapshot.sh | 4 ++-- tools/fleet/box-kill.sh | 8 ++++---- tools/fleet/box-matrix.sh | 2 +- tools/fleet/box-node-swap.sh | 4 ++-- tools/fleet/box-pool.sh | 2 +- tools/fleet/box-prover.py | 2 +- tools/fleet/box-prover.sh | 2 +- tools/fleet/box-rig.sh | 2 +- tools/fleet/box-standing.sh | 14 +++++++------- tools/fleet/box-wave-pool.sh | 2 +- tools/fleet/box-wave.sh | 4 ++-- tools/fleet/canary-next.sh | 2 +- tools/fleet/canary-poison-confirm.py | 2 +- tools/fleet/canary-poisoned-peer.py | 6 +++--- tools/fleet/canary.sh | 2 +- tools/fleet/devnet2-gate.sh | 4 ++-- tools/fleet/dn2-kill.sh | 4 ++-- tools/fleet/dn2-prover-restart.sh | 4 ++-- tools/fleet/fleet.py | 2 +- tools/fleet/lib/box.py | 6 +++--- tools/fleet/lib/standing.py | 4 ++-- tools/fleet/night.py | 2 +- tools/fleet/publish-0315.py | 2 +- tools/fleet/publish-2-move.py | 4 ++-- tools/fleet/rehearsal-kill.sh | 4 ++-- 27 files changed, 50 insertions(+), 50 deletions(-) diff --git a/tools/fleet/box-dn2.sh b/tools/fleet/box-dn2.sh index f4416ba1b..bc5cceb85 100755 --- a/tools/fleet/box-dn2.sh +++ b/tools/fleet/box-dn2.sh @@ -17,14 +17,14 @@ if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then curl -fsSL -o $F/pkg.tgz "https://dl.igneum.network$PKG_PATH" && echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null && mkdir -p /opt/igneum/pkg && tar -C /opt/igneum/pkg --strip-components=1 -xzf $F/pkg.tgz || { echo "RESULT dn2_failed package"; exit 2; } fi B=/opt/igneum/pkg/bin; cp $F/in/dn2-override.json $F/dn2-override.json -pkill -9 -f '^[/]root/fleet/in/igneumd-(0313|[0-9a-f]{16}) ' 2>/dev/null; pkill -9 -f "^/opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:$RPC_PORT " 2>/dev/null; sleep 2 # the Devnet 2 node only, never igneumd-v4 (the rehearsal node beside it, 19:00Z) +pkill -9 -f '[/]root/fleet/in/igneumd-(0313|[0-9a-f]{16}) ' 2>/dev/null; pkill -9 -f "^/opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:$RPC_PORT " 2>/dev/null; sleep 2 # the Devnet 2 node only, never igneumd-v4 (the rehearsal node beside it, 19:00Z) [ "${FRESH:-0}" = 1 ] && { rm -rf $F/dn2; mkdir -p $F/dn2; [ -s $F/dn2-node.log ] && mv $F/dn2-node.log $F/dn2-node.prev.log; echo "RESULT dn2_fresh $(stamp) appdir wiped for a new genesis"; } # never the script's own pattern (17:18Z: dn2-kill.sh killed its caller) PEER=""; for sd in ${SEED//,/ }; do PEER="$PEER --addpeer=$sd"; done # SEED may be a comma list (run A2: two relays per box) # the statement's program ids (proving agent, 22:1xZ): on 4c6b129d they come from the environment or the verifier host, else zero, and a zero id refuses every segment record env ${BPS:+IGNEUMD_DEVNET_BPS=$BPS} IGNEUM_PROOF_PROGRAM_IDS="${PROGRAM_IDS:-0x2b1a81cb413236cf063077b46ed3111628f6c41036bcf6e23ee4cbbf5679ef7a,0x474678f35f7545db28055d5e5bbc308231d84a5a072202087a2a8d5b09123896}" IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host nohup $NODE_BIN --devnet --devnet-suffix=2 --appdir=$F/dn2 --rpclisten=0.0.0.0:$RPC_PORT --evm-rpclisten=127.0.0.1:$EVM_PORT --listen=0.0.0.0:$P2P_PORT $PEER --override-params-file=$F/dn2-override.json --nodnsseed --disable-upnp --nologfiles --yes ${UNSYNCED:+--enable-unsynced-mining} ${NODE_EXTRA:-} >> $F/dn2-node.log 2>&1 & # (--enable-unsynced-mining: a fresh chain's nodes start unsynced and must mine anyway, Reject(IsInIBD) on the seed at 16:52Z; a comment put inside this line at 17:00Z swallowed the redirect and the ampersand, so the node ran in the foreground and the script never reached the miner) sleep 10 -echo "RESULT dn2_node $(stamp) pid=$(pgrep -f '^[/]root/fleet/in/igneumd-(0313|[0-9a-f]{16}) ' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)" +echo "RESULT dn2_node $(stamp) pid=$(pgrep -f '[/]root/fleet/in/igneumd-(0313|[0-9a-f]{16}) ' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)" for i in $(seq 1 30); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:$RPC_PORT 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [ -n "$w" ] && break; sleep 5; done echo "RESULT dn2_watch $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //')" cd $F/mine && rm -rf packs/dn2 && $B/igneum-miner export-pack grpc://127.0.0.1:$RPC_PORT packs/dn2 > $OUT/dn2-export-pack.log 2>&1 diff --git a/tools/fleet/box-ember.sh b/tools/fleet/box-ember.sh index 3d08a8a20..e278c4497 100755 --- a/tools/fleet/box-ember.sh +++ b/tools/fleet/box-ember.sh @@ -27,7 +27,7 @@ rm -rf packs/devnet; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/de nohup $B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/devnet" \ --prepare-packs packs/prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 10 > $OUT/ember-miner.log 2>&1 & MPID=$!; cd $F -cleanup() { nvidia-smi -i 0 -rgc >/dev/null 2>&1; [ "$PL_OK" = 1 ] && nvidia-smi -i 0 -pl "$PDEF" >/dev/null 2>&1; kill $MPID 2>/dev/null; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; } +cleanup() { nvidia-smi -i 0 -rgc >/dev/null 2>&1; [ "$PL_OK" = 1 ] && nvidia-smi -i 0 -pl "$PDEF" >/dev/null 2>&1; kill $MPID 2>/dev/null; pkill -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; } trap cleanup EXIT say "miner warming 90 s"; sleep 90 STEPS=$OUT/ember-steps.jsonl; : > $STEPS diff --git a/tools/fleet/box-exec-snapshot.sh b/tools/fleet/box-exec-snapshot.sh index 3977d0529..53ab9bbcc 100755 --- a/tools/fleet/box-exec-snapshot.sh +++ b/tools/fleet/box-exec-snapshot.sh @@ -16,13 +16,13 @@ echo "RESULT snap_start $(stamp)" [ "$(sha256sum "$SNAP" | cut -c1-64)" = "$SNAP_SHA" ] || { echo "RESULT snap_failed sha256 $(sha256sum "$SNAP" | cut -c1-16)"; exit 2; } chmod 644 "$SNAP" bash $F/in/box-kill.sh >/dev/null 2>&1 -pkill -f '^[/]opt/igneum/pkg/bin/igneumd-0313'; pkill -x igneumd; sleep 4; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneumd-0313' 2>/dev/null; sleep 1 +pkill -f '[/]opt/igneum/pkg/bin/igneumd-0313'; pkill -x igneumd; sleep 4; pkill -9 -f '[/]opt/igneum/pkg/bin/igneumd-0313' 2>/dev/null; sleep 1 rm -f $F/node/igneum-devnet/datadir/evm/exec-snapshot.bin $F/node/igneum-devnet/datadir/evm/exec-snapshot.prev.bin echo "RESULT snap_files_removed $(stamp) left=$(ls $F/node/igneum-devnet/datadir/evm/ 2>/dev/null | tr '\n' ' ')" IGNEUM_PROOF_VERIFIER=$HOST nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 \ --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes "--igneum-exec-snapshot=$SNAP,0x$SNAP_SHA" >> $F/node.log 2>&1 & t1=$(date +%s); sleep 15 -echo "RESULT node_started $(stamp) pid=$(pgrep -f '^[/]opt/igneum/pkg/bin/igneumd-0313' | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') snapshot_line=\"$(grep -E 'igneum-exec\].*snapshot' $F/node.log | tail -1 | cut -c1-140)\"" +echo "RESULT node_started $(stamp) pid=$(pgrep -f '[/]opt/igneum/pkg/bin/igneumd-0313' | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') snapshot_line=\"$(grep -E 'igneum-exec\].*snapshot' $F/node.log | tail -1 | cut -c1-140)\"" for i in $(seq 1 40); do sleep 10; ex="$(rpc igneum_getExecStatus | python3 -c 'import sys,json; r=sys.stdin.read(); d=json.loads(r).get("result",{}) if r.strip() else {}; print(int(d.get("executedTip","0x0"),16), "blocked" if d.get("blocked") else "ok", str(d.get("startedFrom",""))[:30])' 2>/dev/null)" w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'daa=[0-9]*' | tail -1)" diff --git a/tools/fleet/box-kill.sh b/tools/fleet/box-kill.sh index e01ef4e81..4d72134a8 100755 --- a/tools/fleet/box-kill.sh +++ b/tools/fleet/box-kill.sh @@ -3,13 +3,13 @@ # servers); never the node. Run as a FILE (bash in/box-kill.sh) so that no pattern below can match the shell that runs # it (an inline `pkill -f '[i]gneum-prove-host'` killed its own ssh shell on 6 October 2026, 12:38Z). for i in 1 2; do - pkill -9 -f '^[b]ash in/box-matrix.sh' ; pkill -9 -f '^[b]ash in/box-ember.sh'; pkill -9 -f '^[b]ash in/box-prover.sh'; pkill -9 -f '^[p]ython3 -u /root/fleet/in/box-prover.py' - pkill -9 -x igneum-miner; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -9 -x sp1-gpu-server; pkill -9 -f '^[/]opt/igneum-floor/(bin|target|prove)/.*igneum-prove-(host|export)' # pkill -x cannot match a name over 15 characters (igneum-worker-cuda, igneum-prove-host) - pkill -9 -f '^[n]vidia-smi --query' + pkill -9 -f '[b]ash in/box-matrix.sh' ; pkill -9 -f '[b]ash in/box-ember.sh'; pkill -9 -f '[b]ash in/box-prover.sh'; pkill -9 -f '[p]ython3 -u /root/fleet/in/box-prover.py' + pkill -9 -x igneum-miner; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -9 -x sp1-gpu-server; pkill -9 -f '[/]opt/igneum-floor/(bin|target|prove)/.*igneum-prove-(host|export)' # pkill -x cannot match a name over 15 characters (igneum-worker-cuda, igneum-prove-host) + pkill -9 -f '[n]vidia-smi --query' sleep 2 done rm -f /tmp/sp1-cuda-*.sock nvidia-smi -i 0 -rgc >/dev/null 2>&1 d="$(nvidia-smi --query-gpu=power.default_limit --format=csv,noheader,nounits -i 0 | tr -d ' ' | cut -d. -f1)"; [ -n "$d" ] && nvidia-smi -i 0 -pl "$d" >/dev/null 2>&1 cd /root/fleet/out 2>/dev/null && for f in matrix.log ember.log prover.log prover-launch.log rows.jsonl; do [ -f "$f" ] && mv "$f" "$f.$(date +%s).old"; done -echo "count=$(pgrep -c -f '^[b]ash in/box-|^python3 -u /root/fleet/in/box-prover')+$(pgrep -c -x igneum-miner)+$(pgrep -c -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda')+$(pgrep -c -x sp1-gpu-server)+$(pgrep -c -f '^[/]opt/igneum-floor/(bin|target|prove)/.*igneum-prove-host') mem=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 0 | tr -d ' ')" +echo "count=$(pgrep -c -f '[b]ash in/box-|^python3 -u /root/fleet/in/box-prover')+$(pgrep -c -x igneum-miner)+$(pgrep -c -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda')+$(pgrep -c -x sp1-gpu-server)+$(pgrep -c -f '[/]opt/igneum-floor/(bin|target|prove)/.*igneum-prove-host') mem=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 0 | tr -d ' ')" diff --git a/tools/fleet/box-matrix.sh b/tools/fleet/box-matrix.sh index 5ec72c173..ad336bfc0 100755 --- a/tools/fleet/box-matrix.sh +++ b/tools/fleet/box-matrix.sh @@ -55,7 +55,7 @@ miner_start() { --prepare-packs packs/prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 10 > $OUT/miner.log 2>&1 & MPID=$!; cd $F } -miner_stop() { [ -n "$MPID" ] && { kill $MPID 2>/dev/null; sleep 2; kill -9 $MPID 2>/dev/null; }; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x igneum-miner 2>/dev/null; MPID=""; sleep 3; } +miner_stop() { [ -n "$MPID" ] && { kill $MPID 2>/dev/null; sleep 2; kill -9 $MPID 2>/dev/null; }; pkill -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x igneum-miner 2>/dev/null; MPID=""; sleep 3; } miner_rate() { # mean of the STATUS now= values in the last N lines grep STATUS $OUT/miner.log | grep -o ' now=[0-9.]*' | tail -n "${1:-10}" | cut -d= -f2 | awk '{s+=$1; n++} END {if (n) printf "%.2f", s/n; else print 0}' } diff --git a/tools/fleet/box-node-swap.sh b/tools/fleet/box-node-swap.sh index 09cc93214..d07ed80c8 100755 --- a/tools/fleet/box-node-swap.sh +++ b/tools/fleet/box-node-swap.sh @@ -19,7 +19,7 @@ echo "$NODE_SHA256 $F/in/igneumd-0313" | sha256sum -c - >/dev/null || { echo "R chmod +x $F/in/igneumd-0313 t0=$(date +%s) bash $F/in/box-kill.sh >/dev/null 2>&1 # every stage process; the node is stopped on the next line -pkill -x igneumd; pkill -f '^[/]opt/igneum/pkg/bin/igneumd-0313'; sleep 4; pkill -9 -x igneumd 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneumd-0313' 2>/dev/null; sleep 1 +pkill -x igneumd; pkill -f '[/]opt/igneum/pkg/bin/igneumd-0313'; sleep 4; pkill -9 -x igneumd 2>/dev/null; pkill -9 -f '[/]opt/igneum/pkg/bin/igneumd-0313' 2>/dev/null; sleep 1 TGZ_SHA=e67cc6493cd86dc0a993c3e1005b1df43b661fc4973ef7f0095f602f1536cd85 if [ ! -f $F/node/.full-history ]; then [ "$(sha256sum $F/observer-datadir.tgz 2>/dev/null | cut -c1-64)" = "$TGZ_SHA" ] || rm -f $F/observer-datadir.tgz # a partial pre-pull is not a tarball @@ -38,7 +38,7 @@ IGNEUM_PROOF_VERIFIER=$HOST nohup $B/igneumd-0313 --devnet --appdir=$F/node --rp t1=$(date +%s); sleep 12 d="$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print $2}')" ok=DIGEST_MISMATCH; [ "$d" = "$EXPECT_DIGEST" ] && ok=digest_ok -echo "RESULT node_started $(stamp) step=$STEP pid=$(pgrep -f '^[/]opt/igneum/pkg/bin/igneumd-0313' | head -1) version=$($B/igneumd-0313 --version 2>&1 | head -1) digest=${d:0:16} expected=${EXPECT_DIGEST:0:16} $ok exec_restart=\"$(grep -o 'Exec restart from the override file.*' $F/node.log | tail -1 | cut -c1-120)\"" +echo "RESULT node_started $(stamp) step=$STEP pid=$(pgrep -f '[/]opt/igneum/pkg/bin/igneumd-0313' | head -1) version=$($B/igneumd-0313 --version 2>&1 | head -1) digest=${d:0:16} expected=${EXPECT_DIGEST:0:16} $ok exec_restart=\"$(grep -o 'Exec restart from the override file.*' $F/node.log | tail -1 | cut -c1-120)\"" [ "$ok" = digest_ok ] || { echo "RESULT swap_failed digest mismatch"; exit 2; } watch() { $B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1; } if [ "$STEP" = binary ]; then diff --git a/tools/fleet/box-pool.sh b/tools/fleet/box-pool.sh index 9c997ad9d..2980fc9d3 100755 --- a/tools/fleet/box-pool.sh +++ b/tools/fleet/box-pool.sh @@ -13,7 +13,7 @@ if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then curl -fsSL -o $F/pkg.tgz "https://dl.igneum.network$PKG_PATH" && echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null && mkdir -p /opt/igneum/pkg && tar -C /opt/igneum/pkg --strip-components=1 -xzf $F/pkg.tgz || { echo "RESULT pool_host_failed package"; exit 2; } fi B=/opt/igneum/pkg/bin; cp $F/in/igneumd-0313 $B/igneumd-0313; chmod +x $B/igneumd-0313; cp $F/in/ov13.json $F/override.json -pgrep -f '^[/]opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & +pgrep -f '[/]opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}')" for i in $(seq 1 90); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [[ "$w" == *synced=true* ]] && break; sleep 10; done echo "RESULT synced $(stamp) after $((i*10)) s" diff --git a/tools/fleet/box-prover.py b/tools/fleet/box-prover.py index 9d9e399f8..074843de8 100644 --- a/tools/fleet/box-prover.py +++ b/tools/fleet/box-prover.py @@ -86,7 +86,7 @@ def miner_start(): def miner_stop(): global MPROC if MPROC: MPROC.terminate(); time.sleep(2); MPROC.kill(); MPROC = None - subprocess.run(f"pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --device {DEV} '", shell=True) + subprocess.run(f"pkill -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda --device {DEV} '", shell=True) def miner_rate(n=6): try: vals = [float(l.split(" now=")[1].split()[0]) for l in open(f"{OUT}/prover-miner.log").read().split("\n") if "STATUS" in l and " now=" in l][-n:] diff --git a/tools/fleet/box-prover.sh b/tools/fleet/box-prover.sh index f0f008e02..2ee99fe8a 100755 --- a/tools/fleet/box-prover.sh +++ b/tools/fleet/box-prover.sh @@ -13,7 +13,7 @@ if [ -z "${THRESHOLD:-}" ]; then fi if [ -z "${MINER:-}" ]; then if [ "$TOTAL" -ge 15000 ]; then MINER=keep; else MINER=pause; fi; fi echo "RESULT launch $(stamp) total_mib=$TOTAL threshold=${THRESHOLD:-default} miner=$MINER" -pkill -x igneum-miner 2>/dev/null; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock +pkill -x igneum-miner 2>/dev/null; pkill -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock pkill -x igneumd; sleep 4; pkill -9 -x igneumd 2>/dev/null; sleep 1 HUBARG=""; [ -n "${HUB_PEER:-}" ] && HUBARG="--addpeer=$HUB_PEER" # the fleet's hub box (a second peer beside the seed, 12:35Z) NODE_BIN=$B/igneumd; [ -x $B/igneumd-0313 ] && NODE_BIN=$B/igneumd-0313 # the 0.3.13 swap's binary once box-node-swap.sh has run diff --git a/tools/fleet/box-rig.sh b/tools/fleet/box-rig.sh index 5ffcce7f4..6c2ee2adf 100755 --- a/tools/fleet/box-rig.sh +++ b/tools/fleet/box-rig.sh @@ -43,7 +43,7 @@ for d in $(seq 0 $((N-1))); do echo "RESULT miner card=$d mhs=$r watts=$w"; sum=$(awk -v a=$sum -v b=$r 'BEGIN {print a+b}') done echo "RESULT rig_miners $(stamp) cards=$N sum_mhs=$sum watts=$(awk -F', *' '{s+=$2; n++} END {printf "%.0f", s/n*'$N'}' $OUT/rig-samp-mine.csv)" -for p in "${pids[@]}"; do kill $p 2>/dev/null; done; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda'; sleep 3 +for p in "${pids[@]}"; do kill $p 2>/dev/null; done; pkill -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda'; sleep 3 fi # C + D. seven core-only provers and one compressing card, in parallel pkill -9 -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock diff --git a/tools/fleet/box-standing.sh b/tools/fleet/box-standing.sh index 17e8badcf..bed8eaddb 100755 --- a/tools/fleet/box-standing.sh +++ b/tools/fleet/box-standing.sh @@ -35,13 +35,13 @@ start_node() { sleep 8; echo "RESULT standing_node_started $(stamp) bin=$nb pid=$(node_pid) digest=$(grep -o 'digest: [0-9a-f]*' $LOG | tail -1 | awk '{print substr($2,1,16)}')" # a standalone igneum-miner keeps a dead template subscription after its node restarts (20:24Z: templates frozen, # fetch_errors climbing, no submits), so the miner and its worker are restarted with the node; the loop brings them back - pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null; true + pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null; true } HOST_VERIFIER="" miner_loop() { cd $F/mine; [ -d packs/$PACK ] || $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/$PACK > $OUT/export-pack.log 2>&1 while :; do - pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null # one miner per GPU: a restart race left two on three boxes (21:52Z) + pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null # one miner per GPU: a restart race left two on three boxes (21:52Z) $B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/$PACK" --prepare-packs packs/$PACK-prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 60 >> $OUT/miner-0.log 2>&1; rc=$? echo "RESULT standing_miner_exit $(stamp) rc=$rc"; [ $rc = 42 ] && { rm -rf packs/$PACK; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/$PACK >> $OUT/export-pack.log 2>&1; } || sleep 10 done @@ -60,17 +60,17 @@ while :; do if [ "$MINE" = 1 ]; then wl="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; wb=$(grep -oE 'blocks=[0-9]+' <<< "$wl" | cut -d= -f2); ws=$(grep -oE 'synced=[a-z]+' <<< "$wl" | cut -d= -f2) if [ "$ws" = true ] && [ "${wb:-0}" -gt 1000 ]; then - if ! pgrep -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' >/dev/null && [ -z "${MINER_LOOP_PID:-}" -o ! -e "/proc/${MINER_LOOP_PID:-0}" ]; then miner_loop & MINER_LOOP_PID=$!; echo "RESULT standing_miner_started $(stamp) loop=$MINER_LOOP_PID"; fi + if ! pgrep -f '[/]opt/igneum/pkg/bin/igneum-miner mine' >/dev/null && [ -z "${MINER_LOOP_PID:-}" -o ! -e "/proc/${MINER_LOOP_PID:-0}" ]; then miner_loop & MINER_LOOP_PID=$!; echo "RESULT standing_miner_started $(stamp) loop=$MINER_LOOP_PID"; fi else # a box never mines from a genesis-only or unsynced view (the block-rate runs' late-pod reorgs; 22:38Z: a miner on an IBD node) - if pgrep -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' >/dev/null || [ -n "${MINER_LOOP_PID:-}" -a -e "/proc/${MINER_LOOP_PID:-0}" ]; then + if pgrep -f '[/]opt/igneum/pkg/bin/igneum-miner mine' >/dev/null || [ -n "${MINER_LOOP_PID:-}" -a -e "/proc/${MINER_LOOP_PID:-0}" ]; then [ -n "${MINER_LOOP_PID:-}" ] && kill -9 "$MINER_LOOP_PID" 2>/dev/null; MINER_LOOP_PID="" - pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null + pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null echo "RESULT standing_miner_stopped $(stamp) node unsynced ($wl): the miner waits for the tip" fi fi fi - if [ "$PROVER" = 1 ] && [ -x /opt/igneum-floor/bin/igneum-prove-host ] && ! pgrep -f '^[p]ython3 -u /root/fleet/in/box-prover.py' >/dev/null; then + if [ "$PROVER" = 1 ] && [ -x /opt/igneum-floor/bin/igneum-prove-host ] && ! pgrep -f '[p]ython3 -u /root/fleet/in/box-prover.py' >/dev/null; then cd $F && LABEL="$LABEL" WALLET="$WALLET" EXPORT_FROM="${EXPORT_FROM:-0}" CHAIN_NAME=$NET MINER=none RUN_HOURS=240 setsid nohup python3 -u $F/in/box-prover.py > $OUT/prover-launch.log 2>&1 & echo "RESULT standing_prover_started $(stamp)" fi @@ -85,7 +85,7 @@ while :; do ex=$(curl -s -m 5 127.0.0.1:26790 -H 'content-type: application/json' -d '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' | grep -oE '"result":"0x[0-9a-f]+"' | cut -d'"' -f4) exn=$((${ex:-0x0})); blocks=$(printf '%s' "$w" | grep -oE 'blocks=[0-9]+' | cut -d= -f2) m="$(grep STATUS $OUT/miner-0.log 2>/dev/null | tail -1 | grep -oE 'accepted=[0-9]+ rejected=[0-9]+|\([0-9.]+ MH/s inside jobs' | tr '\n' ' ' | tr -d '(')" - echo "RESULT standing $(stamp) up=$(( $(date -u +%s) - STARTED ))s role=$ROLE node=$(node_pid) bin=$cur version=$($cur --version 2>&1 | head -1 | awk '{print $2}') $w exec=$exn miner=\"$m\" prover=$(pgrep -c -f '^[p]ython3 -u /root/fleet/in/box-prover.py') disk=$DISK gpu=$(nvidia-smi --query-gpu=utilization.gpu,power.draw --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')" >> $OUT/standing.log + echo "RESULT standing $(stamp) up=$(( $(date -u +%s) - STARTED ))s role=$ROLE node=$(node_pid) bin=$cur version=$($cur --version 2>&1 | head -1 | awk '{print $2}') $w exec=$exn miner=\"$m\" prover=$(pgrep -c -f '[p]ython3 -u /root/fleet/in/box-prover.py') disk=$DISK gpu=$(nvidia-smi --query-gpu=utilization.gpu,power.draw --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')" >> $OUT/standing.log # the dead-exec class: synced consensus, exec at 0 from snapshot -> the shipper's recovery recipe (live devnet only) if [ "$ROLE" = live ] && [ "${blocks:-0}" -gt 1000 ] && [ "$exn" = 0 ] && [ -x $F/in/box-exec-snapshot.sh ] && [ $((n / 10)) -gt 2 ]; then echo "RESULT standing_recovery $(stamp) exec=0 blocks=$blocks: running box-exec-snapshot.sh"; HUB_PEER="$HUB_PEER" bash $F/in/box-exec-snapshot.sh; sleep 30 diff --git a/tools/fleet/box-wave-pool.sh b/tools/fleet/box-wave-pool.sh index 33ba64db4..65c975808 100755 --- a/tools/fleet/box-wave-pool.sh +++ b/tools/fleet/box-wave-pool.sh @@ -6,7 +6,7 @@ set -uo pipefail F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; POOL="${POOL:?host:port}"; LABEL="${LABEL:-wave}"; WALLET="${WALLET:?}" exec >> $OUT/wave.log 2>&1 stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } -pkill -f '^[b]ash in/box-wave.sh'; pkill -x igneum-miner; sleep 2 +pkill -f '[b]ash in/box-wave.sh'; pkill -x igneum-miner; sleep 2 chmod +x $F/in/igneum-miner-pool echo "RESULT pool_mode $(stamp) pool=$POOL miner=$(sha256sum $F/in/igneum-miner-pool | cut -c1-16)" cd $F/mine diff --git a/tools/fleet/box-wave.sh b/tools/fleet/box-wave.sh index c57833c3a..6f69a6bf8 100755 --- a/tools/fleet/box-wave.sh +++ b/tools/fleet/box-wave.sh @@ -21,8 +21,8 @@ B=/opt/igneum/pkg/bin [ "$(sha256sum $F/in/igneumd-0313 | cut -c1-16)" = d6350586fe837b1f ] || { echo "RESULT wave_failed node binary"; exit 2; } cp $F/in/igneumd-0313 $B/igneumd-0313; chmod +x $B/igneumd-0313; cp $F/in/ov13.json $F/override.json ldconfig -p | grep -q libnvrtc.so.12 || echo "RESULT note no libnvrtc.so.12 on the path (the worker dlopens it)" -pgrep -f '^[/]opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & -sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') pid=$(pgrep -f '^[/]opt/igneum/pkg/bin/igneumd-0313' | head -1)" +pgrep -f '[/]opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & +sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') pid=$(pgrep -f '[/]opt/igneum/pkg/bin/igneumd-0313' | head -1)" for i in $(seq 1 90); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [[ "$w" == *synced=true* ]] && break; sleep 10; done echo "RESULT synced $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //') after $((i*10)) s" cd $F/mine && rm -rf packs/devnet && $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet > $OUT/export-pack.log 2>&1 diff --git a/tools/fleet/canary-next.sh b/tools/fleet/canary-next.sh index 8b3017332..f1f74cfdc 100755 --- a/tools/fleet/canary-next.sh +++ b/tools/fleet/canary-next.sh @@ -56,7 +56,7 @@ for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue l="$(SSH "$2" "$3" "grep -E '^RESULT (node_started|swap_failed)' /root/fleet/out/node-swap.log | tail -1")"; echo " $1: ${l:0:160}"; [[ "$l" == *digest_ok* ]] || fail "$1: the swap did not reach digest_ok (${l:0:200})" done # the provers back on (box-prover.sh keeps a running igneumd-0313 with the verifier) -for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh /dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done +for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh /dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done for p in $MINERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet/mine && pkill -x igneum-miner; sleep 2; rm -rf packs/devnet; /opt/igneum/pkg/bin/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >/dev/null 2>&1; setsid nohup /opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 $1 --worker /opt/igneum/pkg/bin/igneum-worker-cuda --worker-args '--device 0 --pack packs/devnet' --prepare-packs packs/prepare --exit-on-seed-change --evm-address $4 --payout-label $1 --status-secs 30 > /root/fleet/out/mine-only-0.log 2>&1 & echo miner" >/dev/null || fail "$1: miner relaunch"; done fi echo "CANARY running $(stamp) $MINUTES min"; sleep $((MINUTES * 60)) diff --git a/tools/fleet/canary-poison-confirm.py b/tools/fleet/canary-poison-confirm.py index a917055da..eedc8185c 100644 --- a/tools/fleet/canary-poison-confirm.py +++ b/tools/fleet/canary-poison-confirm.py @@ -11,7 +11,7 @@ def st(): return time.strftime("%H:%M:%SZ", time.gmtime()) R=Box.from_registry("pool-1"); M=Box.from_registry("p2-3090-3"); T=Box.from_registry("p2-4090-3"); hub=Box.from_registry("hub-1") _, rM = Registry.find("p2-3090-3"); _, rR = Registry.find("pool-1") W="/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\\n' ' '" -KILL="pkill -9 -f '^[b]ash in/box-standing.sh'; pkill -9 -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs'; pkill -9 -f '^[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir='; sleep 3" +KILL="pkill -9 -f '[b]ash in/box-standing.sh'; pkill -9 -f '[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs'; pkill -9 -f '[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir='; sleep 3" print(st(), "T before:", T.run("echo vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' /root/fleet/node.log | tail -1) rej1026=$(grep -c 'wrong block version' /root/fleet/node.log) got=$(grep -c 'got reject message' /root/fleet/node.log); "+W, 30)[1].replace("\n"," "), flush=True) # R: the relay on 6615571c over its 1026 datadir (kept aside as node.clean while it runs) rc,out,err=R.run(f"cd /root/fleet && {KILL}; [ -d node.clean ] || mv node node.clean; rm -rf node; cp -a node.1026-2109 node; setsid nohup in/igneumd-6615571ced35f694 --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:4463 --addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes > node.log 2>&1 & for i in $(seq 1 40); do sleep 10; w=$({W}); case \"$w\" in *synced=true*) break;; esac; done; echo R vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' node.log | tail -1) after=$((i*10))s $w", 500) diff --git a/tools/fleet/canary-poisoned-peer.py b/tools/fleet/canary-poisoned-peer.py index 6c9342446..449142a3a 100644 --- a/tools/fleet/canary-poisoned-peer.py +++ b/tools/fleet/canary-poisoned-peer.py @@ -9,10 +9,10 @@ def st(): return time.strftime("%H:%M:%SZ", time.gmtime()) NODE, SHA, MINER = sys.argv[1], sys.argv[2], sys.argv[3] assert hashlib.sha256(open(NODE,"rb").read()).hexdigest()==SHA, "node sha" WATCH="/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\\n' ' '" -KILL="pkill -9 -f '^[b]ash in/box-standing.sh'; pkill -9 -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs'; pkill -9 -f '^[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir='; sleep 3" +KILL="pkill -9 -f '[b]ash in/box-standing.sh'; pkill -9 -f '[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs'; pkill -9 -f '[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir='; sleep 3" hub=Box.from_registry("hub-1"); pool=Box.from_registry("pool-1"); A=Box.from_registry("p2-4090-3"); B=Box.from_registry("p2-3090-2") # pool-1: the poisoned peer, as it stands (6615571c, its datadir), on its inbound port -rc,out,err=pool.run("cd /root/fleet && pgrep -f '^[/]root/fleet/in/igneumd-6615571ced35f694 ' >/dev/null || { setsid nohup /root/fleet/in/igneumd-6615571ced35f694 --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:4463 --addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes > node.log 2>&1 & sleep 15; }; echo pool1 $(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' node.log | tail -1) "+WATCH, 60); print(st(), "poisoned peer:", out.replace("\n"," "), flush=True) +rc,out,err=pool.run("cd /root/fleet && pgrep -f '[/]root/fleet/in/igneumd-6615571ced35f694 ' >/dev/null || { setsid nohup /root/fleet/in/igneumd-6615571ced35f694 --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:4463 --addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes > node.log 2>&1 & sleep 15; }; echo pool1 $(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' node.log | tail -1) "+WATCH, 60); print(st(), "poisoned peer:", out.replace("\n"," "), flush=True) def hubsplit(): return hub.run("echo total=$(grep -c 'wrong block version' /root/fleet/node.log); awk '$2 >= \"'$(date -u -d '-12 min' +%H:%M:%S)'\"' /root/fleet/node.log | grep 'wrong block version' | grep -oE 'peer [0-9.]+' | sort | uniq -c | sort -rn | tr '\\n' ';'", 30)[1].replace("\n"," ").strip() print(st(), "hub before:", hubsplit(), flush=True) for box, l, peers in ((A, "p2-4090-3", "--addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --addpeer=213.173.98.36:18451"), (B, "p2-3090-2", "--addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611")): @@ -34,7 +34,7 @@ while time.time()-T0 < 720: rc,b,err=B.run(WATCH+"; echo got_reject=$(grep -c 'got reject message' /root/fleet/node.log)", 30) print(st(), f"+{int(time.time()-T0)}s A:", a.replace("\n"," | ").strip(), "| B:", b.replace("\n"," ").strip(), "| hub:", hubsplit(), flush=True) if not restarted and time.time()-T0 >= 300: - rc,out,err=B.run("cd /root/fleet && pkill -9 -f '^[/]root/fleet/in/igneumd-[0-9a-f]{16} --devnet --appdir='; sleep 20; BIN=$(cat standing.node); IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host setsid nohup $BIN --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=213.173.107.74:16516 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes > node.log 2>&1 & for i in $(seq 1 24); do sleep 5; w=$("+WATCH+"); case \"$w\" in *synced=true*) break;; esac; done; echo resync_after=$((i*5+20))s $w; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve'", 200) + rc,out,err=B.run("cd /root/fleet && pkill -9 -f '[/]root/fleet/in/igneumd-[0-9a-f]{16} --devnet --appdir='; sleep 20; BIN=$(cat standing.node); IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host setsid nohup $BIN --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=213.173.107.74:16516 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes > node.log 2>&1 & for i in $(seq 1 24); do sleep 5; w=$("+WATCH+"); case \"$w\" in *synced=true*) break;; esac; done; echo resync_after=$((i*5+20))s $w; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve'", 200) print(st(), "B restart (hub-only re-sync):", out.replace("\n"," "), flush=True); restarted=True hub_pan=hub.run("echo hub_panics=$(grep -ciE 'panicked at consensus' /root/fleet/node.log)", 20)[1].strip(); print(st(), hub_pan, flush=True) rc,out,err=hub.run(f"timeout 150 /opt/igneum/pkg/bin/igneum-miner inspect 900 grpc://127.0.0.1:26610 2>&1 | grep -c 'voteKeyHash={keyA}'; awk '$2 >= \"'$(date -u -d '-14 min' +%H:%M:%S)'\"' /root/fleet/node.log | grep -E 'wrong block version|disconnecting' | grep -c '216.232.211.246'", 170) diff --git a/tools/fleet/canary.sh b/tools/fleet/canary.sh index 41bbd6b3d..c5c063479 100755 --- a/tools/fleet/canary.sh +++ b/tools/fleet/canary.sh @@ -49,7 +49,7 @@ for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue l="$(SSH "$2" "$3" "grep -E '^RESULT (node_started|swap_failed)' /root/fleet/out/node-swap.log | tail -1 | cut -c1-140")"; echo " $1: $l"; [[ "$l" == *digest_ok* ]] || fail "$1: the swap did not reach digest_ok ($l)" done # the provers back on (box-prover.sh keeps a running igneumd-0313 with the verifier) -for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh /dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done +for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh /dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done for p in $MINERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet/mine && pkill -x igneum-miner; sleep 2; rm -rf packs/devnet; /opt/igneum/pkg/bin/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >/dev/null 2>&1; setsid nohup /opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 $1 --worker /opt/igneum/pkg/bin/igneum-worker-cuda --worker-args '--device 0 --pack packs/devnet' --prepare-packs packs/prepare --exit-on-seed-change --evm-address $4 --payout-label $1 --status-secs 30 > /root/fleet/out/mine-only-0.log 2>&1 & echo miner" >/dev/null || fail "$1: miner relaunch"; done echo "CANARY running $(stamp) $MINUTES min"; sleep $((MINUTES * 60)) echo "CANARY checks $(stamp)"; ok=1; paid_any=0; H="" diff --git a/tools/fleet/devnet2-gate.sh b/tools/fleet/devnet2-gate.sh index 23b41ecab..30561eab3 100644 --- a/tools/fleet/devnet2-gate.sh +++ b/tools/fleet/devnet2-gate.sh @@ -62,10 +62,10 @@ for l in "${BOXES[@]}"; do set -- $l done restart_all() { # with the override file already on each box as /root/fleet/in/dn2-override.json for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] || continue - SSH "$2" "$3" "cd /root/fleet && pkill -f '^[b]ash in/box-dn2.sh'; pkill -f '^[/]root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" + SSH "$2" "$3" "cd /root/fleet && pkill -f '[b]ash in/box-dn2.sh'; pkill -f '[/]root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '[p]ython3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" sleep 15; done for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] && continue - SSH "$2" "$3" "cd /root/fleet && pkill -f '^[b]ash in/box-dn2.sh'; pkill -f '^[/]root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 SEED=$SEED_PEER NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" + SSH "$2" "$3" "cd /root/fleet && pkill -f '[b]ash in/box-dn2.sh'; pkill -f '[/]root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '[p]ython3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 SEED=$SEED_PEER NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" done } SINCE="$(date -u +'%Y-%m-%d %H:%M:%S')"; echo "GATE step1 $(stamp) restart on the release binary, the current override (rejects and reorgs counted from $SINCE)"; restart_all; sleep 60 diff --git a/tools/fleet/dn2-kill.sh b/tools/fleet/dn2-kill.sh index 7905ffcf9..2e8752eeb 100755 --- a/tools/fleet/dn2-kill.sh +++ b/tools/fleet/dn2-kill.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash # Stops everything of a Devnet 2 box but sshd: run as a FILE (an inline pkill with "igneumd" in it kills the ssh shell that carries the word). -pkill -9 -f '^[b]ash in/box-dn2.sh'; pkill -9 -f '^[/]root/fleet/in/igneumd'; pkill -9 -f '^[/]root/fleet/in/igneumd-gate'; pkill -9 -x igneum-miner; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -9 -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -9 -x sp1-gpu-server -sleep 2; echo "left=$(pgrep -c -f '^[/]root/fleet/in/igneumd')+$(pgrep -c -x igneum-miner)+$(pgrep -c -f '^[b]ash in/box-dn2.sh')" +pkill -9 -f '[b]ash in/box-dn2.sh'; pkill -9 -f '[/]root/fleet/in/igneumd'; pkill -9 -f '[/]root/fleet/in/igneumd-gate'; pkill -9 -x igneum-miner; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -9 -f '[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -9 -x sp1-gpu-server +sleep 2; echo "left=$(pgrep -c -f '[/]root/fleet/in/igneumd')+$(pgrep -c -x igneum-miner)+$(pgrep -c -f '[b]ash in/box-dn2.sh')" diff --git a/tools/fleet/dn2-prover-restart.sh b/tools/fleet/dn2-prover-restart.sh index 7005d331e..1bcfc2d42 100755 --- a/tools/fleet/dn2-prover-restart.sh +++ b/tools/fleet/dn2-prover-restart.sh @@ -2,8 +2,8 @@ # Restarts the Devnet 2 prover on a box as ONE instance: run as a FILE (an inline pkill carrying these words kills the ssh shell # that carries them). Env: LABEL WALLET THRESHOLD (optional). F=/root/fleet; cd $F || exit 2 -pkill -9 -f '^[p]ython3 -u (/root/fleet/)?in/box-prover.py'; pkill -9 -x sp1-gpu-server; pkill -9 -f '^[/]opt/igneum-floor/bin/igneum-prove-host'; pkill -9 -f '^[/]opt/igneum-segal/proving/igneum-prove/target/release/igneum-prove-host' +pkill -9 -f '[p]ython3 -u (/root/fleet/)?in/box-prover.py'; pkill -9 -x sp1-gpu-server; pkill -9 -f '[/]opt/igneum-floor/bin/igneum-prove-host'; pkill -9 -f '[/]opt/igneum-segal/proving/igneum-prove/target/release/igneum-prove-host' sleep 3; rm -f /tmp/sp1-cuda-*.sock $F/out/prover.pid $F/out/prover-state.json.*.tmp LABEL="${LABEL:?}" WALLET="${WALLET:?}" THRESHOLD="${THRESHOLD:-}" EXPORT_FROM=0 CHAIN_NAME=igneum-devnet-2 MINER=none RUN_HOURS=240 setsid nohup python3 -u /root/fleet/in/box-prover.py > $F/out/dn2-prover-launch.log 2>&1 & sleep 25 -echo "RESULT dn2_prover_restart $(date -u +%FT%TZ) instances=$(pgrep -f '^[p]ython3 -u (/root/fleet/)?in/box-prover.py' | wc -l) pid=$(cat $F/out/prover.pid 2>/dev/null) server=$(pgrep -c -x sp1-gpu-server) last=\"$(tail -1 $F/out/prover.log | cut -c1-140)\"" +echo "RESULT dn2_prover_restart $(date -u +%FT%TZ) instances=$(pgrep -f '[p]ython3 -u (/root/fleet/)?in/box-prover.py' | wc -l) pid=$(cat $F/out/prover.pid 2>/dev/null) server=$(pgrep -c -x sp1-gpu-server) last=\"$(tail -1 $F/out/prover.log | cut -c1-140)\"" diff --git a/tools/fleet/fleet.py b/tools/fleet/fleet.py index 598388c08..1d8aa9fac 100755 --- a/tools/fleet/fleet.py +++ b/tools/fleet/fleet.py @@ -104,7 +104,7 @@ def setup(labels): if not b.get("ssh_ok"): print(b["label"], "no ssh yet"); continue ssh(b, "mkdir -p /root/fleet/in /root/fleet/out") if scp(b, INPUTS + [os.path.join(HERE, "box-setup.sh")], "/root/fleet/in/") != 0: print(b["label"], "scp failed"); continue - rc, out, err = ssh(b, f"cd /root/fleet && chmod +x in/box-setup.sh && if pgrep -f '^[b]ash in/box-setup.sh' >/dev/null; then echo already; else RUSTUP_TOOLCHAIN=stable ARCHS={b['archs']} LABEL={b['label']} WALLET={b['wallet']} setsid nohup in/box-setup.sh /dev/null 2>&1 & echo started; fi", timeout=60) + rc, out, err = ssh(b, f"cd /root/fleet && chmod +x in/box-setup.sh && if pgrep -f '[b]ash in/box-setup.sh' >/dev/null; then echo already; else RUSTUP_TOOLCHAIN=stable ARCHS={b['archs']} LABEL={b['label']} WALLET={b['wallet']} setsid nohup in/box-setup.sh /dev/null 2>&1 & echo started; fi", timeout=60) print(b["label"], out.strip()[:40]) patch(iid, setup_started=now()) diff --git a/tools/fleet/lib/box.py b/tools/fleet/lib/box.py index a00808b61..67889b67a 100644 --- a/tools/fleet/lib/box.py +++ b/tools/fleet/lib/box.py @@ -4,7 +4,7 @@ KEY = os.path.expanduser("~/.ssh/igneum-fleet") SSH_OPTS = ["-i", KEY, "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", "-o", "ConnectTimeout=20", "-o", "ServerAliveInterval=10", "-o", "ServerAliveCountMax=3", "-o", "BatchMode=yes"] B = "/opt/igneum/pkg/bin"; F = "/root/fleet" -NODE_PATH_PATTERN = "^[/]root/fleet/in/igneumd|^[/]opt/igneum/pkg/bin/igneumd" # the bracket form: the shell that runs a pgrep/pkill with this pattern never matches itself +NODE_PATH_PATTERN = "[/]root/fleet/in/igneumd|[/]opt/igneum/pkg/bin/igneumd" # the bracket form: the shell that runs a pgrep/pkill with this pattern never matches itself def now(): return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") class SshError(RuntimeError): pass def hexi(v): @@ -132,11 +132,11 @@ class Box: log = log or f"{F}/out/miner-{device}.log" self.check(f"cd {F}/mine 2>/dev/null || mkdir -p {F}/mine/packs && cd {F}/mine; [ -d packs/{pack} ] || {B}/igneum-miner export-pack {grpc} packs/{pack} >/dev/null 2>&1; setsid nohup {B}/igneum-miner mine {grpc} 1 100000000 {shlex.quote(label)} --worker {B}/igneum-worker-cuda --worker-args '--device {device} --pack packs/{pack}' --prepare-packs packs/prepare-{device} --exit-on-seed-change --evm-address {wallet} --payout-label {shlex.quote(label)} --status-secs 30 > {log} 2>&1 & echo $!", 90) return int(self.run("pgrep -x igneum-miner | tail -1", 20)[1].strip() or 0) - def stop_miners(self): self.run("pkill -x igneum-miner; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda'; true", 30) + def stop_miners(self): self.run("pkill -x igneum-miner; pkill -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda'; true", 30) def start_prover(self, label, wallet, hub_peer="", threshold="", miner="keep"): self.check(f"cd {F} && chmod +x in/box-prover.sh && LABEL={shlex.quote(label)} WALLET={wallet} HUB_PEER={hub_peer} THRESHOLD={threshold} MINER={miner} setsid nohup in/box-prover.sh /dev/null 2>&1 & echo started", 60) def stop_all(self): - self.run(f"bash {F}/in/box-kill.sh >/dev/null 2>&1; pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -x sp1-gpu-server; true", 60) + self.run(f"bash {F}/in/box-kill.sh >/dev/null 2>&1; pkill -f '[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; pkill -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -x sp1-gpu-server; true", 60) # ---- provider ---- def destroy(self): import sys; sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/tools/fleet/lib/standing.py b/tools/fleet/lib/standing.py index 576bd3a44..a726898d7 100644 --- a/tools/fleet/lib/standing.py +++ b/tools/fleet/lib/standing.py @@ -45,7 +45,7 @@ def install(label, role=None, prover=None, mine=None): b_.put([os.path.join(HERE, "box-standing.sh"), os.path.join(HERE, "box-exec-snapshot.sh")], f"{F}/in/") hub = next((v for v in Registry.load().values() if v.get("hub") and v.get("state") != "destroyed"), {}) seed = b.get("dn2_seed_addr", "") - rc, out, err = b_.run(f"cd {F} && chmod +x in/box-standing.sh in/box-exec-snapshot.sh; pgrep -f '^[b]ash in/box-standing.sh' >/dev/null && echo supervisor-already || {{ ROLE={role} LABEL={label} WALLET={b.get('wallet')} PROVER={prover} MINE={mine} HUB_PEER={hub.get('hub_peer','')} SEED={seed} setsid nohup bash in/box-standing.sh /dev/null 2>&1 & sleep 2; echo supervisor-started; }}", 60) + rc, out, err = b_.run(f"cd {F} && chmod +x in/box-standing.sh in/box-exec-snapshot.sh; pgrep -f '[b]ash in/box-standing.sh' >/dev/null && echo supervisor-already || {{ ROLE={role} LABEL={label} WALLET={b.get('wallet')} PROVER={prover} MINE={mine} HUB_PEER={hub.get('hub_peer','')} SEED={seed} setsid nohup bash in/box-standing.sh /dev/null 2>&1 & sleep 2; echo supervisor-started; }}", 60) Registry.patch(iid, standing=True, role=role, standing_since=b.get("standing_since") or now(), doing=f"standing {role}: node, {'miner, ' if mine == '1' else ''}{'prover' if prover == '1' else 'no prover'} (supervised)") return out.strip() DIGEST_ALERTS = os.path.join(ROOT, "digest-alerts.json") @@ -84,7 +84,7 @@ def update(label): """The version follow: the manifest's hive package onto the box, the node pointer rewritten, the supervisor restarts it.""" m = manifest(); b = Box.from_registry(label) if not m: raise SshError("no manifest") - rc, out, err = b.run(f"set -e; cd {F}; curl -fsSL -m 600 -o pkg-{m['version']}.tgz https://dl.igneum.network{m['path']}; echo '{m['sha256']} pkg-{m['version']}.tgz' | sha256sum -c - >/dev/null; rm -rf /opt/igneum/pkg.new; mkdir -p /opt/igneum/pkg.new; tar -C /opt/igneum/pkg.new --strip-components=1 -xzf pkg-{m['version']}.tgz; rm -rf /opt/igneum/pkg.prev; mv /opt/igneum/pkg /opt/igneum/pkg.prev; mv /opt/igneum/pkg.new /opt/igneum/pkg; echo /opt/igneum/pkg/bin/igneumd > {F}/standing.node; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneumd' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg.prev/bin/igneumd' 2>/dev/null; echo updated-to-{m['version']}", 900) + rc, out, err = b.run(f"set -e; cd {F}; curl -fsSL -m 600 -o pkg-{m['version']}.tgz https://dl.igneum.network{m['path']}; echo '{m['sha256']} pkg-{m['version']}.tgz' | sha256sum -c - >/dev/null; rm -rf /opt/igneum/pkg.new; mkdir -p /opt/igneum/pkg.new; tar -C /opt/igneum/pkg.new --strip-components=1 -xzf pkg-{m['version']}.tgz; rm -rf /opt/igneum/pkg.prev; mv /opt/igneum/pkg /opt/igneum/pkg.prev; mv /opt/igneum/pkg.new /opt/igneum/pkg; echo /opt/igneum/pkg/bin/igneumd > {F}/standing.node; pkill -9 -f '[/]opt/igneum/pkg/bin/igneumd' 2>/dev/null; pkill -9 -f '[/]opt/igneum/pkg.prev/bin/igneumd' 2>/dev/null; echo updated-to-{m['version']}", 900) iid, _ = Registry.find(label); Registry.patch(iid, version_wanted=m["version"], updated_at=now()); return out.strip() def rerent(label): """Same shape again on the same provider; the old row is marked destroyed. Returns the new label or None.""" diff --git a/tools/fleet/night.py b/tools/fleet/night.py index e4712eae0..697cebd7e 100644 --- a/tools/fleet/night.py +++ b/tools/fleet/night.py @@ -15,7 +15,7 @@ PROBE = r"""grep -E '^RESULT (swap_done|swap_failed)' /root/fleet/out/node-swap. curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"igneum_getExecStatus","params":[]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; r=sys.stdin.read(); d=json.loads(r).get("result",{}) if r.strip() else {}; print("exec", int(d.get("executedTip","0x0"),16), "blocked", "yes" if d.get("blocked") else "no", "from", str(d.get("startedFrom",""))[:30])' 2>/dev/null curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"igneum_getProvingStatus","params":[]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; r=sys.stdin.read(); d=json.loads(r).get("result",{}) if r.strip() else {}; v=d.get("v1",{}); w=v.get("segmentsInWindow",{}); print("tipDaa", int(d.get("tipDaa","0x0"),16), "fresh", v.get("freshRuleActive"), "paidShards", d.get("paidShards"), "pending", w.get("pending"), "proven", w.get("proven"), "unproven", w.get("unproven"), "paidSeg", v.get("paidSegments"))' 2>/dev/null /opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'daa=[0-9]*' | tail -1 -pgrep -c -f '^[p]ython3 -u /root/fleet/in/box-prover.py' +pgrep -c -f '[p]ython3 -u /root/fleet/in/box-prover.py' for f in /root/fleet/out/prover-state.json /root/fleet/card*/out/prover-state.json; do [ -f $f ] && python3 -c 'import json,sys; s=json.load(open(sys.argv[1])); print("ps", s.get("claimed",0), s.get("submitted",0), s.get("paid",0), s.get("shards_accepted",0), s.get("shards_refused",0), s.get("segment_refused",0), s.get("held",0), s.get("miner_mhs",0))' $f; done; true""" def probe(b): r, out, err = fleet.ssh(b, PROBE, timeout=45) diff --git a/tools/fleet/publish-0315.py b/tools/fleet/publish-0315.py index dcebe1dc3..4c75dc799 100644 --- a/tools/fleet/publish-0315.py +++ b/tools/fleet/publish-0315.py @@ -25,7 +25,7 @@ def publish(label): b.put([MINER[0], CUDA[0], OCL[0]], "/root/fleet/in/") rc, out, err = b.run(f"""set -e; cd /opt/igneum/pkg/bin; for f in igneum-miner igneum-worker-cuda igneum-worker-opencl; do [ -e $f.0314 ] || cp $f $f.0314; cp /root/fleet/in/$f $f.new && chmod +x $f.new && mv $f.new $f; done echo {path} > /root/fleet/standing.node -pkill -9 -f '^[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir=' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve' 2>/dev/null; true # the supervisor restarts the node on the new pointer within a minute, and the miner with it +pkill -9 -f '[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir=' 2>/dev/null; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve' 2>/dev/null; true # the supervisor restarts the node on the new pointer within a minute, and the miner with it echo swapped""", 120) return label, out.strip() or err[:120] def readback(label): diff --git a/tools/fleet/publish-2-move.py b/tools/fleet/publish-2-move.py index cb2450384..fa697c953 100644 --- a/tools/fleet/publish-2-move.py +++ b/tools/fleet/publish-2-move.py @@ -9,7 +9,7 @@ live=[r["label"] for r in standing.roster() if r["role"]=="live"] def move(l): try: b=Box.from_registry(l); b.put([F16], "/root/fleet/override-16.json") - rc,out,err=b.run("cd /root/fleet && cp override.json override-13.json && cp override-16.json override.json && python3 -c \"import json; d=json.load(open('/root/fleet/override.json')); print('fields', len(d))\" && pkill -9 -f '^[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir=' ; echo killed-for-restart $(date -u +%T)", 60) + rc,out,err=b.run("cd /root/fleet && cp override.json override-13.json && cp override-16.json override.json && python3 -c \"import json; d=json.load(open('/root/fleet/override.json')); print('fields', len(d))\" && pkill -9 -f '[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir=' ; echo killed-for-restart $(date -u +%T)", 60) return l, out.replace("\n"," ").strip()+(" ERR "+err[:60] if err.strip() else "") except Exception as e: return l, "EXC "+str(e)[:60] print(st(), "moving", len(live), "boxes", flush=True) @@ -18,7 +18,7 @@ with ThreadPoolExecutor(14) as ex: time.sleep(150) def read(l): try: - rc,out,err=Box.from_registry(l).run("echo digest=$(grep -o 'digest: [0-9a-f]*' /root/fleet/node.log | tail -1 | awk '{print substr($2,1,16)}') vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' /root/fleet/node.log | tail -1) fields=$(python3 -c \"import json; print(len(json.load(open('/root/fleet/override.json'))))\") $(/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\\n' ' ') miner=$(pgrep -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' | wc -l) rej_since=$(awk '$2 >= \"22:47:00\"' /root/fleet/node.log | grep -c 'got reject message') v4=\"$(grep -E 'Program class v4 (signal window|from the override)' /root/fleet/node.log | tail -2 | sed -E 's/^[0-9: .+-]*//' | cut -c1-70 | tr '\\n' ';')\"", 60) + rc,out,err=Box.from_registry(l).run("echo digest=$(grep -o 'digest: [0-9a-f]*' /root/fleet/node.log | tail -1 | awk '{print substr($2,1,16)}') vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' /root/fleet/node.log | tail -1) fields=$(python3 -c \"import json; print(len(json.load(open('/root/fleet/override.json'))))\") $(/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\\n' ' ') miner=$(pgrep -f '[/]opt/igneum/pkg/bin/igneum-miner mine' | wc -l) rej_since=$(awk '$2 >= \"22:47:00\"' /root/fleet/node.log | grep -c 'got reject message') v4=\"$(grep -E 'Program class v4 (signal window|from the override)' /root/fleet/node.log | tail -2 | sed -E 's/^[0-9: .+-]*//' | cut -c1-70 | tr '\\n' ';')\"", 60) return l, out.strip().split("\n")[-1] except Exception as e: return l, "EXC "+str(e)[:60] with ThreadPoolExecutor(14) as ex: res=list(ex.map(read, live)) diff --git a/tools/fleet/rehearsal-kill.sh b/tools/fleet/rehearsal-kill.sh index 7d0424d9e..1cc517a1b 100644 --- a/tools/fleet/rehearsal-kill.sh +++ b/tools/fleet/rehearsal-kill.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash # Stops a rehearsal box's miner loops, miner, worker and sampler but NOT its dn400 node: run as a FILE (an inline pkill # carrying these words kills the ssh shell that carries them). Then MINER_ONLY=1 bash in/box-rehearsal.sh restarts the miner. -pkill -9 -f '^[b]ash in/box-rehearsal.sh'; pkill -9 -f '^[/]root/fleet/in/igneum-miner-v4 '; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve .*dn400' -sleep 1; echo "left=$(pgrep -c -f '^[b]ash in/box-rehearsal.sh')+$(pgrep -c -f '^[/]root/fleet/in/igneum-miner-v4 ') node=$(pgrep -c -f '^[/]root/fleet/in/igneumd')" +pkill -9 -f '[b]ash in/box-rehearsal.sh'; pkill -9 -f '[/]root/fleet/in/igneum-miner-v4 '; pkill -9 -f '[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve .*dn400' +sleep 1; echo "left=$(pgrep -c -f '[b]ash in/box-rehearsal.sh')+$(pgrep -c -f '[/]root/fleet/in/igneum-miner-v4 ') node=$(pgrep -c -f '[/]root/fleet/in/igneumd')"