diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8fed26f91..988936a03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,8 @@ jobs: run: bash tools/ci/pinned-guests-check.sh - name: root prover playbooks kill the GPU server and unlink its socket (the root-socket class, 5 October 2026) run: bash tools/ci/prover-socket-check.sh + - name: pgrep self-match check (a process pattern never matches the shell that runs it) + run: bash tools/ci/pgrep-self-match-check.sh --self-test && bash tools/ci/pgrep-self-match-check.sh - name: no secret file names and no 64-hex secrets in the tree (self-test first, then the tree) run: bash tools/ci/no-secrets-check.sh --self-test && bash tools/ci/no-secrets-check.sh - name: faucet unit tests (validation, the daily limits, the signed transaction; keccak, RLP and secp256k1 vectors) diff --git a/tools/ci/README.md b/tools/ci/README.md new file mode 100644 index 000000000..83786a04a --- /dev/null +++ b/tools/ci/README.md @@ -0,0 +1,6 @@ +# CI checks + +| Check | What it fails | Since | +|---|---|---| + +| `pgrep-self-match-check.sh` | a `pgrep -f` / `pkill -f` with a bare literal pattern, or `ps \| grep ` without a bracket or `grep -v grep`, in tools/, relay/playbooks/, infra/ or packaging/: the pattern matches the shell that runs it (the wave script of 6 October 2026 never started a node on 38 cards because `pgrep -f igneumd-0313` saw the launching shell; a kill file killed its caller the same day). Anchor to the executable's path, bracket the first letter, or use `-x`. Owed (allow-listed, finished measurements): `tools/prover-floor/pc2-*.ps1`, `tools/proving-v1/pc2-*.ps1` (their `pkill -f sp1-gpu-server` becomes `pkill -x`), `tools/repo/fresh-repo.sh:223` | 6 October 2026, branch gpu-fleet | diff --git a/tools/ci/pgrep-self-match-check.sh b/tools/ci/pgrep-self-match-check.sh new file mode 100755 index 000000000..4a584786d --- /dev/null +++ b/tools/ci/pgrep-self-match-check.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# The self-matching process-pattern class (6 October 2026). Three times in one day a script matched its own shell: +# the shipper's recovery at 16:1xZ, the fleet's wave script at 16:25Z (`pgrep -f igneumd-0313 || start the node` matched +# the launching shell's command line, which carried the file name, so no wave pod ever started its node and 38 cards +# hashed against nothing for an hour), and the fleet's Devnet 2 kill step at 17:18Z (`pkill -f '^bash in/box-dn2.sh'` +# inside a file the same script called killed the caller). Rule: a `pgrep -f`, `pkill -f` or `ps ... | grep` whose +# pattern is a literal word matches every process whose command line carries that word, including the shell that +# runs the pattern and any ssh command that carries the script's text; the pattern must therefore exclude itself: +# anchored to the executable's path (`'^/opt/igneum/pkg/bin/igneumd'`), the bracket form (`'[i]gneumd'`), or +# `pgrep -x ` / `pkill -x ` on the binary name (15 characters at most). This check fails CI when a script +# under tools/, relay/playbooks/, infra/ or packaging/ runs pgrep -f / pkill -f with a bare literal pattern (no `^`, +# no bracket, no `$`), or pipes `ps` into `grep ` without a bracket or a `grep -v grep`. +# With file arguments it checks those files only; --self-test runs the two fixtures. +set -euo pipefail +cd "$(dirname "$0")/../.." +fail=0 +bad_pattern() { # the pattern text between the quotes after -f; prints 1 when it is a bare literal + local p="$1" + [[ "$p" == ^* || "$p" == *'['* || "$p" == *'$' || "$p" == '$'* ]] && return 1 + return 0 +} +check_file() { + local f="$1" n=0 + while IFS= read -r line; do + n=$((n + 1)) + [[ "$line" =~ ^[[:space:]]*# ]] && continue + # pgrep -f / pkill -f with a quoted or bare pattern + while read -r pat; do + [ -z "$pat" ] && continue + if bad_pattern "$pat"; then echo "pgrep-self-match: $f:$n: p(grep|kill) -f with the bare pattern '$pat' matches the shell that runs it; anchor it (^/path), bracket it ([x]rest) or use -x"; fail=1; fi + done < <(printf '%s\n' "$line" | grep -oE "p(grep|kill)( -[0-9A-Za-z]+)* -f(a|c|l)? +(\"[^\"]*\"|'[^']*'|[^ |;)]+)" | sed -E "s/^p(grep|kill)( -[0-9A-Za-z]+)* -f[acl]* +//; s/^[\"']//; s/[\"']$//") + # ps | grep word + if printf '%s\n' "$line" | grep -qE 'ps [^|]*\| *grep ' && ! printf '%s\n' "$line" | grep -qE "grep +(-[a-zA-Z]+ +)*['\"]?\[" && ! printf '%s\n' "$line" | grep -q 'grep -v grep'; then + echo "pgrep-self-match: $f:$n: ps | grep without a bracket pattern or 'grep -v grep' matches the grep itself"; fail=1 + fi + done < "$f" +} +if [ "${1:-}" = "--self-test" ]; then + t="$(mktemp -d)" + printf 'pgrep -f igneumd-0313 >/dev/null || start\npkill -f "bash in/box-x.sh"\nps aux | grep igneumd\n' > "$t/bad.sh" + printf "pgrep -f '^/opt/igneum/pkg/bin/igneumd' || start\npkill -x igneum-miner\npkill -f '[i]gneumd-0313'\nps aux | grep '[i]gneumd'\nps -eo cmd | grep igneumd | grep -v grep\n" > "$t/good.sh" + fail=0; check_file "$t/bad.sh"; [ "$fail" = 1 ] || { echo "pgrep-self-match: self-test FAILED: the bad fixture passed"; exit 1; } + fail=0; check_file "$t/good.sh"; [ "$fail" = 0 ] || { echo "pgrep-self-match: self-test FAILED: the good fixture was flagged"; exit 1; } + echo "pgrep-self-match: self-test ok (the bad fixture fails, the good one passes)"; exit 0 +fi +# Owed, not exempt: the PC 2 playbooks of 5 and 6 October use `pkill -f sp1-gpu-server` (the prover-socket check's own +# required line) inside a WSL `bash -c` whose command line carries the word, so the pkill kills that shell too when it +# runs first; they are finished measurements and get `pkill -x sp1-gpu-server` when next touched (tools/ci/README.md). +ALLOW='^(tools/prover-floor/pc2-.*\.ps1|tools/proving-v1/pc2-.*\.ps1|tools/repo/fresh-repo\.sh)$' +list_files() { if [ $# -gt 0 ]; then printf '%s\n' "$@"; else git ls-files 'tools/**' 'relay/playbooks/**' 'infra/**' 'packaging/**' | grep -E '\.(sh|bash|ps1|mjs|py)$'; fi; } +while IFS= read -r f; do [ -f "$f" ] || continue; [[ "$f" =~ $ALLOW ]] && continue; check_file "$f"; done < <(list_files "$@") +[ "$fail" = 0 ] && echo "pgrep-self-match: no script matches its own shell" +exit $fail diff --git a/tools/fleet/box-dn2.sh b/tools/fleet/box-dn2.sh index 291cb2e3c..57556cd0d 100755 --- a/tools/fleet/box-dn2.sh +++ b/tools/fleet/box-dn2.sh @@ -16,9 +16,10 @@ if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then curl -fsSL -o $F/pkg.tgz "https://dl.igneum.network$PKG_PATH" && echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null && mkdir -p /opt/igneum/pkg && tar -C /opt/igneum/pkg --strip-components=1 -xzf $F/pkg.tgz || { echo "RESULT dn2_failed package"; exit 2; } fi B=/opt/igneum/pkg/bin; cp $F/in/dn2-override.json $F/dn2-override.json -pkill -f 'igneumd.*devnet-suffix=2' 2>/dev/null; sleep 2 +pkill -9 -f '^/root/fleet/in/igneumd' 2>/dev/null; pkill -9 -x igneum-miner 2>/dev/null; sleep 2 # never the script's own pattern (17:18Z: dn2-kill.sh killed its caller) PEER=""; [ -n "$SEED" ] && PEER="--addpeer=$SEED" -IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host nohup $NODE_BIN --devnet --devnet-suffix=2 --appdir=$F/dn2 --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 $PEER --override-params-file=$F/dn2-override.json --nodnsseed --disable-upnp --nologfiles --yes ${NODE_EXTRA:-} >> $F/dn2-node.log 2>&1 & +IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host nohup $NODE_BIN --devnet --devnet-suffix=2 --appdir=$F/dn2 --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 $PEER --override-params-file=$F/dn2-override.json --nodnsseed --disable-upnp --nologfiles --yes --enable-unsynced-mining ${NODE_EXTRA:-} >> $F/dn2-node.log 2>&1 & +# (--enable-unsynced-mining: a fresh chain's nodes start unsynced and must mine anyway, Reject(IsInIBD) on the seed at 16:52Z; a comment put inside this line at 17:00Z swallowed the redirect and the ampersand, so the node ran in the foreground and the script never reached the miner) sleep 10 echo "RESULT dn2_node $(stamp) pid=$(pgrep -f 'devnet-suffix=2' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)" for i in $(seq 1 30); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [ -n "$w" ] && break; sleep 5; done diff --git a/tools/fleet/box-pool.sh b/tools/fleet/box-pool.sh index 6eed92b1a..513c64a9f 100755 --- a/tools/fleet/box-pool.sh +++ b/tools/fleet/box-pool.sh @@ -13,7 +13,7 @@ if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then curl -fsSL -o $F/pkg.tgz "https://dl.igneum.network$PKG_PATH" && echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null && mkdir -p /opt/igneum/pkg && tar -C /opt/igneum/pkg --strip-components=1 -xzf $F/pkg.tgz || { echo "RESULT pool_host_failed package"; exit 2; } fi B=/opt/igneum/pkg/bin; cp $F/in/igneumd-0313 $B/igneumd-0313; chmod +x $B/igneumd-0313; cp $F/in/ov13.json $F/override.json -pgrep -f igneumd-0313 >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & +pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}')" for i in $(seq 1 90); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [[ "$w" == *synced=true* ]] && break; sleep 10; done echo "RESULT synced $(stamp) after $((i*10)) s" diff --git a/tools/fleet/box-wave-pool.sh b/tools/fleet/box-wave-pool.sh new file mode 100755 index 000000000..91885f041 --- /dev/null +++ b/tools/fleet/box-wave-pool.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# A wave pod moved onto the fleet pool (pool-v0 on the pool pod): the solo miner loop stops, the pool fork's igneum-miner +# (pushed as /root/fleet/in/igneum-miner-pool) mines against the pool with its own node as the verifier (templates and +# seeds checked against grpc://127.0.0.1:26610), one worker, the vote key on. RESULT lines in /root/fleet/out/wave.log. +set -uo pipefail +F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; POOL="${POOL:?host:port}"; LABEL="${LABEL:-wave}"; WALLET="${WALLET:?}" +exec >> $OUT/wave.log 2>&1 +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +pkill -f '^bash in/box-wave.sh'; pkill -x igneum-miner; sleep 2 +chmod +x $F/in/igneum-miner-pool +echo "RESULT pool_mode $(stamp) pool=$POOL miner=$(sha256sum $F/in/igneum-miner-pool | cut -c1-16)" +cd $F/mine +while :; do + $F/in/igneum-miner-pool mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --pool "$POOL" --evm-address "$WALLET" --worker-name "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/devnet" --prepare-packs packs/prepare --exit-on-seed-change --status-secs 30 >> $OUT/pool-miner.log 2>&1; rc=$? + echo "RESULT pool_miner_exit $(stamp) rc=$rc"; [ $rc = 42 ] && { rm -rf packs/devnet; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >> $OUT/export-pack.log 2>&1; } || sleep 10 +done diff --git a/tools/fleet/box-wave.sh b/tools/fleet/box-wave.sh index 6b86bcb82..44b204b22 100755 --- a/tools/fleet/box-wave.sh +++ b/tools/fleet/box-wave.sh @@ -21,7 +21,7 @@ B=/opt/igneum/pkg/bin [ "$(sha256sum $F/in/igneumd-0313 | cut -c1-16)" = d6350586fe837b1f ] || { echo "RESULT wave_failed node binary"; exit 2; } cp $F/in/igneumd-0313 $B/igneumd-0313; chmod +x $B/igneumd-0313; cp $F/in/ov13.json $F/override.json ldconfig -p | grep -q libnvrtc.so.12 || echo "RESULT note no libnvrtc.so.12 on the path (the worker dlopens it)" -pgrep -f igneumd-0313 >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & +pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') pid=$(pgrep -f igneumd-0313 | head -1)" for i in $(seq 1 90); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [[ "$w" == *synced=true* ]] && break; sleep 10; done echo "RESULT synced $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //') after $((i*10)) s" diff --git a/tools/fleet/canary-next.sh b/tools/fleet/canary-next.sh new file mode 100755 index 000000000..8c68882ef --- /dev/null +++ b/tools/fleet/canary-next.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# bash 3.2 (the Mac's): plain arrays, no mapfile, no associative arrays (lookup files under $WORK) +# The 0.3.14 canary gate on the LIVE devnet (6 October 2026, 17:00Z, the coordinator's form for a publish that moves no +# digest and no height): the release's Linux igneumd goes onto the six restart-path provers and two miners within two +# minutes of the URL (the override file kept), runs TEN minutes, then one line: PASS only with zero rejected blocks on +# every box, exec state roots equal on every box and on the hub at a common height, at least one segment record paid on +# the new binary (a prover's paidSegments up), every box's node on the new version; FAIL names the box and the line. +# canary.sh --binary --sha256 [--minutes 10] +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$HOME/Desktop/fleet" +BIN=""; SHA=""; MINUTES=10 +while [[ $# -gt 0 ]]; do case "$1" in --binary) BIN="$2"; shift 2 ;; --sha256) SHA="$2"; shift 2 ;; --minutes) MINUTES="$2"; shift 2 ;; *) echo "unknown $1"; exit 2 ;; esac; done +[[ -n "$BIN" && -n "$SHA" ]] || { echo "usage: $0 --binary --sha256 [--minutes 10]"; exit 2; } +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +LOG="$ROOT/canary-$(date -u +%Y%m%dT%H%M%SZ).log"; exec > >(tee -a "$LOG") 2>&1 +PROVERS="${PROVERS:-p1-3090 p1-5090 p2-3090-1 p2-3090-2 p2-3090-3 p2-4090-1b}"; MINERS="${MINERS:-p1-3080 p1-a5000}" +fail() { echo "FAIL $(stamp) $*"; exit 1; } +LOCAL="$ROOT/canary-igneumd" +if [[ "$BIN" == http* ]]; then curl -fsSL -o "$LOCAL" "$BIN" || fail "download $BIN"; else cp "$BIN" "$LOCAL"; fi +[[ "$(shasum -a 256 "$LOCAL" | cut -c1-64)" == "$SHA" ]] || fail "sha256 of the binary is $(shasum -a 256 "$LOCAL" | cut -c1-16), not ${SHA:0:16}" +WANT="${SHA:0:16}_$(file "$LOCAL" >/dev/null; echo igneumd_2.1.0)" # the running binary's sha256 (first 16) and its --version word, per box +echo "CANARY start $(stamp) binary=${SHA:0:16} version=$WANT boxes: $PROVERS $MINERS" +ROWS=(); while IFS= read -r line; do ROWS+=("$line"); done < <(python3 - "$PROVERS $MINERS" <<'PY' +import json, os, sys; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json"))); want=sys.argv[1].split() +for lab in want: + b=next((v for v in reg.values() if v["label"]==lab and v.get("state")!="destroyed"), None) + if b: print(lab, b["ssh_host"], b["ssh_port"], b["wallet"]) +hub=next(v for v in reg.values() if v.get("hub") and v.get("state")!="destroyed" and v.get("hub_peer")); print("hub-1", hub["ssh_host"], hub["ssh_port"], hub["wallet"], hub["hub_peer"]) +PY +) +HUB="$(printf '%s\n' "${ROWS[@]}" | awk '$1=="hub-1"')"; HUB_HOST="$(awk '{print $2}' <<< "$HUB")"; HUB_PORT="$(awk '{print $3}' <<< "$HUB")"; HUB_PEER="$(awk '{print $5}' <<< "$HUB")" +SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } +read_box() { SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); ex=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getExecStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"executedTip\",\"0x0\"),16))" 2>/dev/null); v=$(pgrep -fa "^/opt/igneum/pkg/bin/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "got reject message|PoW rejected|block rejected|invalid block" /root/fleet/node.log 2>/dev/null); echo "${d:-0} ${e:-0 0} ${ex:-0} ${ver:-none} ${rej:-0}"' 2>/dev/null; } +root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } +WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; }; rej0() { cat "$WORK/rej0.$1" 2>/dev/null || echo 0; } +echo "CANARY baseline $(stamp)" +for r in "${ROWS[@]}"; do set -- $r; x="$(read_box "$2" "$3")"; echo " $1: daa=$(awk '{print $1}' <<< "$x") tip=$(awk '{print $2}' <<< "$x") paidSeg=$(awk '{print $3}' <<< "$x") exec=$(awk '{print $4}' <<< "$x") ver=$(awk '{print $5}' <<< "$x") rej=$(awk '{print $6}' <<< "$x")"; awk '{print $3}' <<< "$x" > "$WORK/paid0.$1"; awk '{print $6}' <<< "$x" > "$WORK/rej0.$1"; done +# install: the binary swap with the override file kept (box-node-swap.sh STEP=binary, which refuses on a digest change) +T0=$(date +%s) +for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue + okc=0; for try in 1 2 3; do scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -P "$3" "$LOCAL" "root@$2:/root/fleet/in/igneumd-0313" >/dev/null 2>&1 && { okc=1; break; }; sleep 5; done # Vast's ssh proxy drops a connection now and then (17:14Z); three tries before a FAIL + [[ $okc == 1 ]] || fail "$1: scp failed three times" + oks=0; for try in 1 2 3; do SSH "$2" "$3" "cd /root/fleet && chmod +x in/igneumd-0313 && [ \"\$(sha256sum in/igneumd-0313 | cut -c1-64)\" = $SHA ] && mv out/node-swap.log out/node-swap-canary-prev.log 2>/dev/null; STEP=binary NODE_SHA256=$SHA EXPECT_DIGEST=b18ed271f75dd46406d230f4156c37472127415a4c32c558bac662f6f840e61c HUB_PEER=$HUB_PEER HUB_SSH=$HUB_HOST HUB_PORT=$HUB_PORT setsid nohup in/box-node-swap.sh /dev/null 2>&1 & echo swapped" >/dev/null 2>&1 && { oks=1; break; }; sleep 5; done + [[ $oks == 1 ]] || fail "$1: install failed three times" +done +echo "CANARY installed $(stamp) on $(( ${#ROWS[@]} - 1 )) boxes in $(( $(date +%s) - T0 )) s" +sleep 75 +for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue + l="$(SSH "$2" "$3" "grep -E '^RESULT (node_started|swap_failed)' /root/fleet/out/node-swap.log | tail -1 | cut -c1-140")"; echo " $1: $l"; [[ "$l" == *digest_ok* ]] || fail "$1: the swap did not reach digest_ok ($l)" +done +# the provers back on (box-prover.sh keeps a running igneumd-0313 with the verifier) +for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh /dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done +for p in $MINERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet/mine && pkill -x igneum-miner; sleep 2; rm -rf packs/devnet; /opt/igneum/pkg/bin/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >/dev/null 2>&1; setsid nohup /opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 $1 --worker /opt/igneum/pkg/bin/igneum-worker-cuda --worker-args '--device 0 --pack packs/devnet' --prepare-packs packs/prepare --exit-on-seed-change --evm-address $4 --payout-label $1 --status-secs 30 > /root/fleet/out/mine-only-0.log 2>&1 & echo miner" >/dev/null || fail "$1: miner relaunch"; done +echo "CANARY running $(stamp) $MINUTES min"; sleep $((MINUTES * 60)) +echo "CANARY checks $(stamp)"; ok=1; paid_any=0; H="" +for r in "${ROWS[@]}"; do set -- $r; x="$(read_box "$2" "$3")"; paid="$(awk '{print $3}' <<< "$x")"; ex="$(awk '{print $4}' <<< "$x")"; ver="$(awk '{print $5}' <<< "$x")"; rej="$(awk '{print $6}' <<< "$x")" + echo " $1: tip=$(awk '{print $2}' <<< "$x") paidSeg=$paid exec=$ex ver=$ver rej=$rej (was $(rej0 $1))" + [[ "$1" == hub-1 ]] || { [[ "$ver" == "$WANT" ]] || { echo "FAIL $1: version $ver, want $WANT"; ok=0; }; } + [[ "${rej:-0}" -le "$(rej0 $1)" ]] || { echo "FAIL $1: $(( rej - $(rej0 $1) )) new rejected blocks (grep 'reject' /root/fleet/node.log)"; ok=0; } + [[ "$1" == hub-1 ]] || { for p in $PROVERS; do [[ "$1" == "$p" && "${paid:-0}" -gt "$(paid0 $1)" ]] && paid_any=1; done; } + [[ -z "$H" || "${ex:-0}" -lt "$H" ]] && H="${ex:-0}" +done +H=$((H > 20 ? H - 20 : 1)); HX="$(printf '0x%x' "$H")"; first="" +for r in "${ROWS[@]}"; do set -- $r; rt="$(root_at "$2" "$3" "$HX")"; echo " $1 root@$H ${rt:0:18}"; [[ -z "$first" ]] && first="$rt"; [[ "$rt" == "$first" ]] || { echo "FAIL $1: exec root at $H ${rt:0:18} differs from ${first:0:18}"; ok=0; }; done +[[ $paid_any == 1 ]] || { echo "FAIL no segment record paid on any prover during the $MINUTES min (paidSegments did not rise)"; ok=0; } +res=FAIL; [[ $ok == 1 ]] && res=PASS +echo "$res $(stamp) canary 0.3.14 ${SHA:0:16} version $WANT: $(( ${#ROWS[@]} - 1 )) boxes, $MINUTES min, rejects 0, roots equal at $H incl. the hub, segment paid $paid_any; log $LOG" +[[ $ok == 1 ]] diff --git a/tools/fleet/canary.sh b/tools/fleet/canary.sh new file mode 100755 index 000000000..ed8e50076 --- /dev/null +++ b/tools/fleet/canary.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# bash 3.2 (the Mac's): plain arrays, no mapfile, no associative arrays (lookup files under $WORK) +# The 0.3.14 canary gate on the LIVE devnet (6 October 2026, 17:00Z, the coordinator's form for a publish that moves no +# digest and no height): the release's Linux igneumd goes onto the six restart-path provers and two miners within two +# minutes of the URL (the override file kept), runs TEN minutes, then one line: PASS only with zero rejected blocks on +# every box, exec state roots equal on every box and on the hub at a common height, at least one segment record paid on +# the new binary (a prover's paidSegments up), every box's node on the new version; FAIL names the box and the line. +# canary.sh --binary --sha256 [--minutes 10] +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$HOME/Desktop/fleet" +BIN=""; SHA=""; MINUTES=10 +while [[ $# -gt 0 ]]; do case "$1" in --binary) BIN="$2"; shift 2 ;; --sha256) SHA="$2"; shift 2 ;; --minutes) MINUTES="$2"; shift 2 ;; *) echo "unknown $1"; exit 2 ;; esac; done +[[ -n "$BIN" && -n "$SHA" ]] || { echo "usage: $0 --binary --sha256 [--minutes 10]"; exit 2; } +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +LOG="$ROOT/canary-$(date -u +%Y%m%dT%H%M%SZ).log"; exec > >(tee -a "$LOG") 2>&1 +PROVERS="p1-3090 p1-5090 p2-3090-1 p2-3090-2 p2-3090-3 p2-4090-1b"; MINERS="p1-3080 p1-a5000" +fail() { echo "FAIL $(stamp) $*"; exit 1; } +LOCAL="$ROOT/canary-igneumd" +if [[ "$BIN" == http* ]]; then curl -fsSL -o "$LOCAL" "$BIN" || fail "download $BIN"; else cp "$BIN" "$LOCAL"; fi +[[ "$(shasum -a 256 "$LOCAL" | cut -c1-64)" == "$SHA" ]] || fail "sha256 of the binary is $(shasum -a 256 "$LOCAL" | cut -c1-16), not ${SHA:0:16}" +WANT="${SHA:0:16}_$(file "$LOCAL" >/dev/null; echo igneumd_2.1.0)" # the running binary's sha256 (first 16) and its --version word, per box +echo "CANARY start $(stamp) binary=${SHA:0:16} version=$WANT boxes: $PROVERS $MINERS" +ROWS=(); while IFS= read -r line; do ROWS+=("$line"); done < <(python3 - "$PROVERS $MINERS" <<'PY' +import json, os, sys; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json"))); want=sys.argv[1].split() +for lab in want: + b=next((v for v in reg.values() if v["label"]==lab and v.get("state")!="destroyed"), None) + if b: print(lab, b["ssh_host"], b["ssh_port"], b["wallet"]) +hub=next(v for v in reg.values() if v.get("hub") and v.get("state")!="destroyed" and v.get("hub_peer")); print("hub-1", hub["ssh_host"], hub["ssh_port"], hub["wallet"], hub["hub_peer"]) +PY +) +HUB="$(printf '%s\n' "${ROWS[@]}" | awk '$1=="hub-1"')"; HUB_HOST="$(awk '{print $2}' <<< "$HUB")"; HUB_PORT="$(awk '{print $3}' <<< "$HUB")"; HUB_PEER="$(awk '{print $5}' <<< "$HUB")" +SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } +read_box() { SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); ex=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getExecStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"executedTip\",\"0x0\"),16))" 2>/dev/null); v=$(pgrep -fa "^/opt/igneum/pkg/bin/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "got reject message|PoW rejected|block rejected|invalid block" /root/fleet/node.log 2>/dev/null); echo "${d:-0} ${e:-0 0} ${ex:-0} ${ver:-none} ${rej:-0}"' 2>/dev/null; } +root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } +WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; }; rej0() { cat "$WORK/rej0.$1" 2>/dev/null || echo 0; } +echo "CANARY baseline $(stamp)" +for r in "${ROWS[@]}"; do set -- $r; x="$(read_box "$2" "$3")"; echo " $1: daa=$(awk '{print $1}' <<< "$x") tip=$(awk '{print $2}' <<< "$x") paidSeg=$(awk '{print $3}' <<< "$x") exec=$(awk '{print $4}' <<< "$x") ver=$(awk '{print $5}' <<< "$x") rej=$(awk '{print $6}' <<< "$x")"; awk '{print $3}' <<< "$x" > "$WORK/paid0.$1"; awk '{print $6}' <<< "$x" > "$WORK/rej0.$1"; done +# install: the binary swap with the override file kept (box-node-swap.sh STEP=binary, which refuses on a digest change) +T0=$(date +%s) +for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue + okc=0; for try in 1 2 3; do scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -P "$3" "$LOCAL" "root@$2:/root/fleet/in/igneumd-0313" >/dev/null 2>&1 && { okc=1; break; }; sleep 5; done # Vast's ssh proxy drops a connection now and then (17:14Z); three tries before a FAIL + [[ $okc == 1 ]] || fail "$1: scp failed three times" + oks=0; for try in 1 2 3; do SSH "$2" "$3" "cd /root/fleet && chmod +x in/igneumd-0313 && [ \"\$(sha256sum in/igneumd-0313 | cut -c1-64)\" = $SHA ] && mv out/node-swap.log out/node-swap-canary-prev.log 2>/dev/null; STEP=binary NODE_SHA256=$SHA EXPECT_DIGEST=b18ed271f75dd46406d230f4156c37472127415a4c32c558bac662f6f840e61c HUB_PEER=$HUB_PEER HUB_SSH=$HUB_HOST HUB_PORT=$HUB_PORT setsid nohup in/box-node-swap.sh /dev/null 2>&1 & echo swapped" >/dev/null 2>&1 && { oks=1; break; }; sleep 5; done + [[ $oks == 1 ]] || fail "$1: install failed three times" +done +echo "CANARY installed $(stamp) on $(( ${#ROWS[@]} - 1 )) boxes in $(( $(date +%s) - T0 )) s" +sleep 75 +for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue + l="$(SSH "$2" "$3" "grep -E '^RESULT (node_started|swap_failed)' /root/fleet/out/node-swap.log | tail -1 | cut -c1-140")"; echo " $1: $l"; [[ "$l" == *digest_ok* ]] || fail "$1: the swap did not reach digest_ok ($l)" +done +# the provers back on (box-prover.sh keeps a running igneumd-0313 with the verifier) +for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh /dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done +for p in $MINERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet/mine && pkill -x igneum-miner; sleep 2; rm -rf packs/devnet; /opt/igneum/pkg/bin/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >/dev/null 2>&1; setsid nohup /opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 $1 --worker /opt/igneum/pkg/bin/igneum-worker-cuda --worker-args '--device 0 --pack packs/devnet' --prepare-packs packs/prepare --exit-on-seed-change --evm-address $4 --payout-label $1 --status-secs 30 > /root/fleet/out/mine-only-0.log 2>&1 & echo miner" >/dev/null || fail "$1: miner relaunch"; done +echo "CANARY running $(stamp) $MINUTES min"; sleep $((MINUTES * 60)) +echo "CANARY checks $(stamp)"; ok=1; paid_any=0; H="" +for r in "${ROWS[@]}"; do set -- $r; x="$(read_box "$2" "$3")"; paid="$(awk '{print $3}' <<< "$x")"; ex="$(awk '{print $4}' <<< "$x")"; ver="$(awk '{print $5}' <<< "$x")"; rej="$(awk '{print $6}' <<< "$x")" + echo " $1: tip=$(awk '{print $2}' <<< "$x") paidSeg=$paid exec=$ex ver=$ver rej=$rej (was $(rej0 $1))" + [[ "$1" == hub-1 ]] || { [[ "$ver" == "$WANT" ]] || { echo "FAIL $1: version $ver, want $WANT"; ok=0; }; } + [[ "${rej:-0}" -le "$(rej0 $1)" ]] || { echo "FAIL $1: $(( rej - $(rej0 $1) )) new rejected blocks (grep 'reject' /root/fleet/node.log)"; ok=0; } + [[ "$1" == hub-1 ]] || { for p in $PROVERS; do [[ "$1" == "$p" && "${paid:-0}" -gt "$(paid0 $1)" ]] && paid_any=1; done; } + [[ -z "$H" || "${ex:-0}" -lt "$H" ]] && H="${ex:-0}" +done +H=$((H > 20 ? H - 20 : 1)); HX="$(printf '0x%x' "$H")"; first="" +for r in "${ROWS[@]}"; do set -- $r; rt="$(root_at "$2" "$3" "$HX")"; echo " $1 root@$H ${rt:0:18}"; [[ -z "$first" ]] && first="$rt"; [[ "$rt" == "$first" ]] || { echo "FAIL $1: exec root at $H ${rt:0:18} differs from ${first:0:18}"; ok=0; }; done +[[ $paid_any == 1 ]] || { echo "FAIL no segment record paid on any prover during the $MINUTES min (paidSegments did not rise)"; ok=0; } +res=FAIL; [[ $ok == 1 ]] && res=PASS +echo "$res $(stamp) canary 0.3.14 ${SHA:0:16} version $WANT: $(( ${#ROWS[@]} - 1 )) boxes, $MINUTES min, rejects 0, roots equal at $H incl. the hub, segment paid $paid_any; log $LOG" +[[ $ok == 1 ]] diff --git a/tools/fleet/devnet2-gate.sh b/tools/fleet/devnet2-gate.sh index f07eb1496..b047027fb 100755 --- a/tools/fleet/devnet2-gate.sh +++ b/tools/fleet/devnet2-gate.sh @@ -60,10 +60,10 @@ for l in "${BOXES[@]}"; do set -- $l done restart_all() { # with the override file already on each box as /root/fleet/in/dn2-override.json for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] || continue - SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f 'igneumd.*devnet-suffix=2'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" + SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f '^/root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" sleep 15; done for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] && continue - SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f 'igneumd.*devnet-suffix=2'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 SEED=$SEED_PEER NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" + SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f '^/root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 SEED=$SEED_PEER NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" done } echo "GATE step1 $(stamp) restart on the release binary, the current override"; restart_all; sleep 60 diff --git a/tools/fleet/dn2-kill.sh b/tools/fleet/dn2-kill.sh new file mode 100755 index 000000000..6bb0e926f --- /dev/null +++ b/tools/fleet/dn2-kill.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +# Stops everything of a Devnet 2 box but sshd: run as a FILE (an inline pkill with "igneumd" in it kills the ssh shell that carries the word). +pkill -9 -f '^bash in/box-dn2.sh'; pkill -9 -f '^/root/fleet/in/igneumd'; pkill -9 -f '^/root/fleet/in/igneumd-gate'; pkill -9 -x igneum-miner; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -9 -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -9 -x sp1-gpu-server +sleep 2; echo "left=$(pgrep -c -f '^/root/fleet/in/igneumd')+$(pgrep -c -x igneum-miner)+$(pgrep -c -f '^bash in/box-dn2.sh')"