From 22ef7c3f97080cfdddc2270abfecb2aea461ea1a Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:31:55 +0000 Subject: [PATCH] Fleet: the last process patterns in the bracket-first form; the fleet's own pgrep check withdrawn (master's kill-by-name check covers the class) Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 2 -- tools/ci/README.md | 1 - tools/ci/pgrep-self-match-check.sh | 53 ----------------------------- tools/fleet/autorun.py | 2 +- tools/fleet/canary-next.sh | 2 +- tools/fleet/canary-poisoned-peer.py | 2 +- tools/fleet/canary.sh | 2 +- tools/fleet/devnet2-gate.sh | 2 +- tools/fleet/publish-0315.py | 2 +- 9 files changed, 6 insertions(+), 62 deletions(-) delete mode 100755 tools/ci/pgrep-self-match-check.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 988936a0..8fed26f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,8 +79,6 @@ jobs: run: bash tools/ci/pinned-guests-check.sh - name: root prover playbooks kill the GPU server and unlink its socket (the root-socket class, 5 October 2026) run: bash tools/ci/prover-socket-check.sh - - name: pgrep self-match check (a process pattern never matches the shell that runs it) - run: bash tools/ci/pgrep-self-match-check.sh --self-test && bash tools/ci/pgrep-self-match-check.sh - name: no secret file names and no 64-hex secrets in the tree (self-test first, then the tree) run: bash tools/ci/no-secrets-check.sh --self-test && bash tools/ci/no-secrets-check.sh - name: faucet unit tests (validation, the daily limits, the signed transaction; keccak, RLP and secp256k1 vectors) diff --git a/tools/ci/README.md b/tools/ci/README.md index a2bf50e4..7b1c8503 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -3,5 +3,4 @@ | Check | What it fails | Since | |---|---|---| -| `pgrep-self-match-check.sh` | a `pgrep -f` / `pkill -f` with a bare literal pattern, or `ps \| grep ` without a bracket or `grep -v grep`, in tools/, relay/playbooks/, infra/ or packaging/: the pattern matches the shell that runs it (the wave script of 6 October 2026 never started a node on 38 cards because `pgrep -f igneumd-0313` saw the launching shell; a kill file killed its caller the same day). Anchor to the executable's path, bracket the first letter, or use `-x`. Owed (allow-listed, finished measurements): `tools/prover-floor/pc2-*.ps1`, `tools/proving-v1/pc2-*.ps1` (their `pkill -f sp1-gpu-server` becomes `pkill -x`), `tools/repo/fresh-repo.sh:223`, `tools/observer/autosync.sh:22`, and the live devnet's operational scripts the fleet agent does not own: `infra/devnet/restart-hand-nodes.sh`, `infra/seed-nodes/addpeer-from-mac.sh`, `relay/playbooks/shard-test.ps1`, `tools/ci/fixtures/bash-body-ok.ps1`, `tools/ci/pgrep-self-match-check.sh`, `tools/ci/prover-socket-check.sh`, `tools/exec-attacks/net.sh` (their owners anchor the pattern when next touched; the hand-node and seed scripts run tonight and were not edited blind) | 6 October 2026, branch gpu-fleet | | kill by exact command or pid file (owed as a check) | 6 October 2026, 21:09Z: a Mac-side `pkill -f ` matched nothing (the log name was a redirect, not part of the command line), the roll-everything script lived on and wiped a box it had been told to hold. Rule: a job is stopped by its pid file (`tools/fleet/fleet-bg.sh start|stop `) or by a pattern anchored on its exact command line (`^python3 -u /root/fleet/in/box-prover.py`), never by a word that may or may not appear in it. The check that flags a `pkill -f`/`pgrep -f` whose literal is a path or a name that never starts a command line is owed to the CI lane | diff --git a/tools/ci/pgrep-self-match-check.sh b/tools/ci/pgrep-self-match-check.sh deleted file mode 100755 index 3a081910..00000000 --- a/tools/ci/pgrep-self-match-check.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env bash -# The self-matching process-pattern class (6 October 2026). Three times in one day a script matched its own shell: -# the shipper's recovery at 16:1xZ, the fleet's wave script at 16:25Z (`pgrep -f igneumd-0313 || start the node` matched -# the launching shell's command line, which carried the file name, so no wave pod ever started its node and 38 cards -# hashed against nothing for an hour), and the fleet's Devnet 2 kill step at 17:18Z (`pkill -f '^bash in/box-dn2.sh'` -# inside a file the same script called killed the caller). Rule: a `pgrep -f`, `pkill -f` or `ps ... | grep` whose -# pattern is a literal word matches every process whose command line carries that word, including the shell that -# runs the pattern and any ssh command that carries the script's text; the pattern must therefore exclude itself: -# anchored to the executable's path (`'^/opt/igneum/pkg/bin/igneumd'`), the bracket form (`'[i]gneumd'`), or -# `pgrep -x ` / `pkill -x ` on the binary name (15 characters at most). This check fails CI when a script -# under tools/, relay/playbooks/, infra/ or packaging/ runs pgrep -f / pkill -f with a bare literal pattern (no `^`, -# no bracket, no `$`), or pipes `ps` into `grep ` without a bracket or a `grep -v grep`. -# With file arguments it checks those files only; --self-test runs the two fixtures. -set -euo pipefail -cd "$(dirname "$0")/../.." -fail=0 -bad_pattern() { # the pattern text between the quotes after -f; prints 1 when it is a bare literal - local p="$1" - [[ "$p" == ^* || "$p" == *'['* || "$p" == *'$' || "$p" == '$'* ]] && return 1 - return 0 -} -check_file() { - local f="$1" n=0 - while IFS= read -r line; do - n=$((n + 1)) - [[ "$line" =~ ^[[:space:]]*# ]] && continue - # pgrep -f / pkill -f with a quoted or bare pattern - while read -r pat; do - [ -z "$pat" ] && continue - if bad_pattern "$pat"; then echo "pgrep-self-match: $f:$n: p(grep|kill) -f with the bare pattern '$pat' matches the shell that runs it; anchor it (^/path), bracket it ([x]rest) or use -x"; fail=1; fi - done < <(printf '%s\n' "$line" | grep -oE "p(grep|kill)( -[0-9A-Za-z]+)* -f(a|c|l)? +(\"[^\"]*\"|'[^']*'|[^ |;)]+)" | sed -E "s/^p(grep|kill)( -[0-9A-Za-z]+)* -f[acl]* +//; s/^[\"']//; s/[\"']$//") - # ps | grep word - if printf '%s\n' "$line" | grep -qE 'ps [^|]*\| *grep ' && ! printf '%s\n' "$line" | grep -qE "grep +(-[a-zA-Z]+ +)*['\"]?\[" && ! printf '%s\n' "$line" | grep -q 'grep -v grep'; then - echo "pgrep-self-match: $f:$n: ps | grep without a bracket pattern or 'grep -v grep' matches the grep itself"; fail=1 - fi - done < "$f" -} -if [ "${1:-}" = "--self-test" ]; then - t="$(mktemp -d)" - printf 'pgrep -f igneumd-0313 >/dev/null || start\npkill -f "bash in/box-x.sh"\nps aux | grep igneumd\n' > "$t/bad.sh" - printf "pgrep -f '^/opt/igneum/pkg/bin/igneumd' || start\npkill -x igneum-miner\npkill -f '[i]gneumd-0313'\nps aux | grep '[i]gneumd'\nps -eo cmd | grep igneumd | grep -v grep\n" > "$t/good.sh" - fail=0; check_file "$t/bad.sh"; [ "$fail" = 1 ] || { echo "pgrep-self-match: self-test FAILED: the bad fixture passed"; exit 1; } - fail=0; check_file "$t/good.sh"; [ "$fail" = 0 ] || { echo "pgrep-self-match: self-test FAILED: the good fixture was flagged"; exit 1; } - echo "pgrep-self-match: self-test ok (the bad fixture fails, the good one passes)"; exit 0 -fi -# Owed, not exempt: the PC 2 playbooks of 5 and 6 October use `pkill -f sp1-gpu-server` (the prover-socket check's own -# required line) inside a WSL `bash -c` whose command line carries the word, so the pkill kills that shell too when it -# runs first; they are finished measurements and get `pkill -x sp1-gpu-server` when next touched (tools/ci/README.md). -ALLOW='^(tools/prover-floor/pc2-.*\.ps1|tools/proving-v1/pc2-.*\.ps1|tools/repo/fresh-repo\.sh|tools/observer/autosync\.sh|infra/devnet/restart\-hand\-nodes\.sh|infra/seed\-nodes/addpeer\-from\-mac\.sh|relay/playbooks/shard\-test\.ps1|tools/ci/fixtures/bash\-body\-ok\.ps1|tools/ci/pgrep\-self\-match\-check\.sh|tools/ci/prover\-socket\-check\.sh|tools/exec\-attacks/net\.sh)$' -list_files() { if [ $# -gt 0 ]; then printf '%s\n' "$@"; else git ls-files 'tools/**' 'relay/playbooks/**' 'infra/**' 'packaging/**' | grep -E '\.(sh|bash|ps1|mjs|py)$'; fi; } -while IFS= read -r f; do [ -f "$f" ] || continue; [[ "$f" =~ $ALLOW ]] && continue; check_file "$f"; done < <(list_files "$@") -[ "$fail" = 0 ] && echo "pgrep-self-match: no script matches its own shell" -exit $fail diff --git a/tools/fleet/autorun.py b/tools/fleet/autorun.py index fdbbc0ad..b502b6bf 100644 --- a/tools/fleet/autorun.py +++ b/tools/fleet/autorun.py @@ -17,7 +17,7 @@ def log(s, page=False): line = f"{now()} {s}"; open(LOG, "a").write(line + "\n"); print(line, flush=True) if page: subprocess.run([sys.executable, os.path.join(fleet.HERE, "page.py"), "log", s]) def probe(b): - rc, out, err = fleet.ssh(b, "tail -1 /root/fleet/setup.log 2>/dev/null | grep -o '^RESULT setup_[a-z]*'; grep -c . /root/fleet/out/matrix.log 2>/dev/null; grep -o '^RESULT matrix_[a-z]*' /root/fleet/out/matrix.log 2>/dev/null | tail -1; grep -c . /root/fleet/out/ember.log 2>/dev/null; grep -o '^RESULT ember_done' /root/fleet/out/ember.log 2>/dev/null | tail -1; pgrep -c -f '^(bash in/box-|python3 -u /root/fleet/in/box-prover)'; grep -E '^RESULT (point|miner|step|choice|claim|submitted|paid|segment_refused|seg [0-9]+ (chain|shards))' /root/fleet/out/matrix.log /root/fleet/out/ember.log /root/fleet/out/prover.log 2>/dev/null | tail -1 | cut -c1-200", timeout=40) + rc, out, err = fleet.ssh(b, "tail -1 /root/fleet/setup.log 2>/dev/null | grep -o '^RESULT setup_[a-z]*'; grep -c . /root/fleet/out/matrix.log 2>/dev/null; grep -o '^RESULT matrix_[a-z]*' /root/fleet/out/matrix.log 2>/dev/null | tail -1; grep -c . /root/fleet/out/ember.log 2>/dev/null; grep -o '^RESULT ember_done' /root/fleet/out/ember.log 2>/dev/null | tail -1; pgrep -c -f '([b]ash in/box-|[p]ython3 -u /root/fleet/in/box-prover)'; grep -E '^RESULT (point|miner|step|choice|claim|submitted|paid|segment_refused|seg [0-9]+ (chain|shards))' /root/fleet/out/matrix.log /root/fleet/out/ember.log /root/fleet/out/prover.log 2>/dev/null | tail -1 | cut -c1-200", timeout=40) if rc != 0 and not out.strip(): return None l = out.split("\n") g = lambda i: l[i].strip() if i < len(l) else "" diff --git a/tools/fleet/canary-next.sh b/tools/fleet/canary-next.sh index f1f74cfd..1a773cbc 100755 --- a/tools/fleet/canary-next.sh +++ b/tools/fleet/canary-next.sh @@ -30,7 +30,7 @@ PY ) HUB="$(printf '%s\n' "${ROWS[@]}" | awk '$1=="hub-1"')"; HUB_HOST="$(awk '{print $2}' <<< "$HUB")"; HUB_PORT="$(awk '{print $3}' <<< "$HUB")"; HUB_PEER="$(awk '{print $5}' <<< "$HUB")" SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } -read_box() { SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); ex=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getExecStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"executedTip\",\"0x0\"),16))" 2>/dev/null); v=$(pgrep -fa "^/opt/igneum/pkg/bin/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "got reject message|PoW rejected|block rejected|invalid block" /root/fleet/node.log 2>/dev/null); echo "${d:-0} ${e:-0 0} ${ex:-0} ${ver:-none} ${rej:-0}"' 2>/dev/null; } +read_box() { SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); ex=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getExecStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"executedTip\",\"0x0\"),16))" 2>/dev/null); v=$(pgrep -fa "[/]opt/igneum/pkg/bin/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "got reject message|PoW rejected|block rejected|invalid block" /root/fleet/node.log 2>/dev/null); echo "${d:-0} ${e:-0 0} ${ex:-0} ${ver:-none} ${rej:-0}"' 2>/dev/null; } root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; }; rej0() { cat "$WORK/rej0.$1" 2>/dev/null || echo 0; } echo "CANARY baseline $(stamp)" diff --git a/tools/fleet/canary-poisoned-peer.py b/tools/fleet/canary-poisoned-peer.py index ad9e59de..30ac7402 100644 --- a/tools/fleet/canary-poisoned-peer.py +++ b/tools/fleet/canary-poisoned-peer.py @@ -17,7 +17,7 @@ def hubsplit(): return hub.run("echo total=$(grep -c 'wrong block version' /root print(st(), "hub before:", hubsplit(), flush=True) for box, l, peers in ((A, "p2-4090-3", "--addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --addpeer=213.173.98.36:18451"), (B, "p2-3090-2", "--addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611")): path=box.install_payload(NODE, SHA); box.put([MINER], "/root/fleet/in/igneum-miner-new"); iid,r=Registry.find(l) - already = "ALREADY=yes" in box.run(f"pgrep -f '^{path} --devnet --appdir=' >/dev/null && echo ALREADY=yes || echo ALREADY=no", 20)[1] # a marker: the Vast proxy prints a banner on stdout # the node already runs this binary (a relaunch): keep its datadir and sync + already = "ALREADY=yes" in box.run(f"pgrep -f '[{path[0]}]{path[1:]} --devnet --appdir=' >/dev/null && echo ALREADY=yes || echo ALREADY=no", 20)[1] # a marker: the Vast proxy prints a banner on stdout # the node already runs this binary (a relaunch): keep its datadir and sync if already: print(st(), l, "already on the binary, keeping its sync", flush=True) rc,out,err=(0,"kept","") if already else box.run(f"cd /root/fleet && {KILL}; [ -d node ] && mv node node.pre-$(date -u +%H%M); mkdir -p node; mv node.log node.pre-$(date -u +%H%M).log 2>/dev/null; cp in/igneum-miner-new /opt/igneum/pkg/bin/igneum-miner.new && chmod +x /opt/igneum/pkg/bin/igneum-miner.new && mv /opt/igneum/pkg/bin/igneum-miner.new /opt/igneum/pkg/bin/igneum-miner; echo {path} > standing.node; IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host setsid nohup {path} --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 {peers} --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes > node.log 2>&1 & echo started T0=$(date -u +%T)", 90) print(st(), l, out.replace("\n"," "), flush=True) diff --git a/tools/fleet/canary.sh b/tools/fleet/canary.sh index c5c06347..400efe90 100755 --- a/tools/fleet/canary.sh +++ b/tools/fleet/canary.sh @@ -30,7 +30,7 @@ PY ) HUB="$(printf '%s\n' "${ROWS[@]}" | awk '$1=="hub-1"')"; HUB_HOST="$(awk '{print $2}' <<< "$HUB")"; HUB_PORT="$(awk '{print $3}' <<< "$HUB")"; HUB_PEER="$(awk '{print $5}' <<< "$HUB")" SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } -read_box() { SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); ex=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getExecStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"executedTip\",\"0x0\"),16))" 2>/dev/null); v=$(pgrep -fa "^/opt/igneum/pkg/bin/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "got reject message|PoW rejected|block rejected|invalid block" /root/fleet/node.log 2>/dev/null); echo "${d:-0} ${e:-0 0} ${ex:-0} ${ver:-none} ${rej:-0}"' 2>/dev/null; } +read_box() { SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); ex=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getExecStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"executedTip\",\"0x0\"),16))" 2>/dev/null); v=$(pgrep -fa "[/]opt/igneum/pkg/bin/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "got reject message|PoW rejected|block rejected|invalid block" /root/fleet/node.log 2>/dev/null); echo "${d:-0} ${e:-0 0} ${ex:-0} ${ver:-none} ${rej:-0}"' 2>/dev/null; } root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; }; rej0() { cat "$WORK/rej0.$1" 2>/dev/null || echo 0; } echo "CANARY baseline $(stamp)" diff --git a/tools/fleet/devnet2-gate.sh b/tools/fleet/devnet2-gate.sh index 30561eab..a7171786 100644 --- a/tools/fleet/devnet2-gate.sh +++ b/tools/fleet/devnet2-gate.sh @@ -49,7 +49,7 @@ echo "GATE boxes=${#BOXES[@]} seed=$SEED_HOST:$SEED_PORT peer=$SEED_PEER" # a reading of every box: height, exec tip and root at a height, rejects, max reorg, version, paid segments SINCE="${SINCE:-1970-01-01 00:00:00}" # log lines at or after this UTC stamp count (set at step 1) read_box() { # host port -> "height daa exec_tip paid_seg version rejects max_reorg", rejects and reorgs since $SINCE - SSH "$1" "$2" 'SINCE="'"$SINCE"'"; B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "^/root/fleet/in/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(awk -v s="$SINCE" "substr(\$0,1,19) >= s" /root/fleet/dn2-node.log 2>/dev/null | grep -cE "reject message|PoW rejected|block rejected|invalid block"); mr=$(awk -v s="$SINCE" "substr(\$0,1,19) >= s" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "selected-chain reorg: [0-9]+ chain blocks" | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null + SSH "$1" "$2" 'SINCE="'"$SINCE"'"; B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "[/]root/fleet/in/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(awk -v s="$SINCE" "substr(\$0,1,19) >= s" /root/fleet/dn2-node.log 2>/dev/null | grep -cE "reject message|PoW rejected|block rejected|invalid block"); mr=$(awk -v s="$SINCE" "substr(\$0,1,19) >= s" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "selected-chain reorg: [0-9]+ chain blocks" | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null } root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; } diff --git a/tools/fleet/publish-0315.py b/tools/fleet/publish-0315.py index decc1a16..d99f0687 100644 --- a/tools/fleet/publish-0315.py +++ b/tools/fleet/publish-0315.py @@ -30,7 +30,7 @@ echo swapped""", 120) return label, out.strip() or err[:120] def readback(label): b = Box.from_registry(label) - rc, out, err = b.run(r"""P=$(pgrep -af '[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd[^ ]* ' | head -1 | awk '{print $2}'); echo bin=$P sha=$(sha256sum $P 2>/dev/null | cut -c1-16) version="$($P --version 2>&1 | head -1)" vline=$(grep -oE 'igneumd/2\.1\.0-[0-9a-f]+' /root/fleet/node.log | tail -1) digest=$(grep -o 'digest: [0-9a-f]*' /root/fleet/node.log | tail -1 | awk '{print substr($2,1,16)}') miner=$(sha256sum /opt/igneum/pkg/bin/igneum-miner | cut -c1-16) cuda=$(sha256sum /opt/igneum/pkg/bin/igneum-worker-cuda | cut -c1-16) ocl=$(sha256sum /opt/igneum/pkg/bin/igneum-worker-opencl | cut -c1-16) $(/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|synced=[a-z]+' | tr '\n' ' ') miner_up=$(pgrep -fc '^/opt/igneum/pkg/bin/igneum-miner mine') prover_up=$(pgrep -fc '^python3 -u /root/fleet/in/box-prover.py') rej=$(grep -c 'PoW rejected' /root/fleet/node.log)""", 60) + rc, out, err = b.run(r"""P=$(pgrep -af '[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd[^ ]* ' | head -1 | awk '{print $2}'); echo bin=$P sha=$(sha256sum $P 2>/dev/null | cut -c1-16) version="$($P --version 2>&1 | head -1)" vline=$(grep -oE 'igneumd/2\.1\.0-[0-9a-f]+' /root/fleet/node.log | tail -1) digest=$(grep -o 'digest: [0-9a-f]*' /root/fleet/node.log | tail -1 | awk '{print substr($2,1,16)}') miner=$(sha256sum /opt/igneum/pkg/bin/igneum-miner | cut -c1-16) cuda=$(sha256sum /opt/igneum/pkg/bin/igneum-worker-cuda | cut -c1-16) ocl=$(sha256sum /opt/igneum/pkg/bin/igneum-worker-opencl | cut -c1-16) $(/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|synced=[a-z]+' | tr '\n' ' ') miner_up=$(pgrep -fc '[/]opt/igneum/pkg/bin/igneum-miner mine') prover_up=$(pgrep -fc '[p]ython3 -u /root/fleet/in/box-prover.py') rej=$(grep -c 'PoW rejected' /root/fleet/node.log)""", 60) return label, out.strip() or err[:120] if __name__ == "__main__": a = sys.argv[1:]; rb = a and a[0] == "--readback"; a = a[1:] if rb else a