diff --git a/tools/ci/README.md b/tools/ci/README.md index 83786a04a..c717caad7 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -3,4 +3,4 @@ | Check | What it fails | Since | |---|---|---| -| `pgrep-self-match-check.sh` | a `pgrep -f` / `pkill -f` with a bare literal pattern, or `ps \| grep ` without a bracket or `grep -v grep`, in tools/, relay/playbooks/, infra/ or packaging/: the pattern matches the shell that runs it (the wave script of 6 October 2026 never started a node on 38 cards because `pgrep -f igneumd-0313` saw the launching shell; a kill file killed its caller the same day). Anchor to the executable's path, bracket the first letter, or use `-x`. Owed (allow-listed, finished measurements): `tools/prover-floor/pc2-*.ps1`, `tools/proving-v1/pc2-*.ps1` (their `pkill -f sp1-gpu-server` becomes `pkill -x`), `tools/repo/fresh-repo.sh:223` | 6 October 2026, branch gpu-fleet | +| `pgrep-self-match-check.sh` | a `pgrep -f` / `pkill -f` with a bare literal pattern, or `ps \| grep ` without a bracket or `grep -v grep`, in tools/, relay/playbooks/, infra/ or packaging/: the pattern matches the shell that runs it (the wave script of 6 October 2026 never started a node on 38 cards because `pgrep -f igneumd-0313` saw the launching shell; a kill file killed its caller the same day). Anchor to the executable's path, bracket the first letter, or use `-x`. Owed (allow-listed, finished measurements): `tools/prover-floor/pc2-*.ps1`, `tools/proving-v1/pc2-*.ps1` (their `pkill -f sp1-gpu-server` becomes `pkill -x`), `tools/repo/fresh-repo.sh:223`, `tools/observer/autosync.sh:22` | 6 October 2026, branch gpu-fleet | diff --git a/tools/ci/pgrep-self-match-check.sh b/tools/ci/pgrep-self-match-check.sh index 4a584786d..ac07afd7f 100755 --- a/tools/ci/pgrep-self-match-check.sh +++ b/tools/ci/pgrep-self-match-check.sh @@ -46,7 +46,7 @@ fi # Owed, not exempt: the PC 2 playbooks of 5 and 6 October use `pkill -f sp1-gpu-server` (the prover-socket check's own # required line) inside a WSL `bash -c` whose command line carries the word, so the pkill kills that shell too when it # runs first; they are finished measurements and get `pkill -x sp1-gpu-server` when next touched (tools/ci/README.md). -ALLOW='^(tools/prover-floor/pc2-.*\.ps1|tools/proving-v1/pc2-.*\.ps1|tools/repo/fresh-repo\.sh)$' +ALLOW='^(tools/prover-floor/pc2-.*\.ps1|tools/proving-v1/pc2-.*\.ps1|tools/repo/fresh-repo\.sh|tools/observer/autosync\.sh)$' list_files() { if [ $# -gt 0 ]; then printf '%s\n' "$@"; else git ls-files 'tools/**' 'relay/playbooks/**' 'infra/**' 'packaging/**' | grep -E '\.(sh|bash|ps1|mjs|py)$'; fi; } while IFS= read -r f; do [ -f "$f" ] || continue; [[ "$f" =~ $ALLOW ]] && continue; check_file "$f"; done < <(list_files "$@") [ "$fail" = 0 ] && echo "pgrep-self-match: no script matches its own shell" diff --git a/tools/fleet/box-dn2.sh b/tools/fleet/box-dn2.sh index 57556cd0d..ee1abb800 100755 --- a/tools/fleet/box-dn2.sh +++ b/tools/fleet/box-dn2.sh @@ -21,7 +21,7 @@ PEER=""; [ -n "$SEED" ] && PEER="--addpeer=$SEED" IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host nohup $NODE_BIN --devnet --devnet-suffix=2 --appdir=$F/dn2 --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 $PEER --override-params-file=$F/dn2-override.json --nodnsseed --disable-upnp --nologfiles --yes --enable-unsynced-mining ${NODE_EXTRA:-} >> $F/dn2-node.log 2>&1 & # (--enable-unsynced-mining: a fresh chain's nodes start unsynced and must mine anyway, Reject(IsInIBD) on the seed at 16:52Z; a comment put inside this line at 17:00Z swallowed the redirect and the ampersand, so the node ran in the foreground and the script never reached the miner) sleep 10 -echo "RESULT dn2_node $(stamp) pid=$(pgrep -f 'devnet-suffix=2' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)" +echo "RESULT dn2_node $(stamp) pid=$(pgrep -f '^/root/fleet/in/igneumd' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)" for i in $(seq 1 30); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [ -n "$w" ] && break; sleep 5; done echo "RESULT dn2_watch $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //')" cd $F/mine && rm -rf packs/dn2 && $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/dn2 > $OUT/dn2-export-pack.log 2>&1 diff --git a/tools/fleet/box-prover.sh b/tools/fleet/box-prover.sh index 323858dec..902660544 100755 --- a/tools/fleet/box-prover.sh +++ b/tools/fleet/box-prover.sh @@ -13,7 +13,7 @@ if [ -z "${THRESHOLD:-}" ]; then fi if [ -z "${MINER:-}" ]; then if [ "$TOTAL" -ge 15000 ]; then MINER=keep; else MINER=pause; fi; fi echo "RESULT launch $(stamp) total_mib=$TOTAL threshold=${THRESHOLD:-default} miner=$MINER" -pkill -f "igneum-miner mine" 2>/dev/null; pkill -f igneum-worker-cuda 2>/dev/null; pkill -f sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock +pkill -f "igneum-miner mine" 2>/dev/null; pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock pkill -x igneumd; sleep 4; pkill -9 -x igneumd 2>/dev/null; sleep 1 HUBARG=""; [ -n "${HUB_PEER:-}" ] && HUBARG="--addpeer=$HUB_PEER" # the fleet's hub box (a second peer beside the seed, 12:35Z) NODE_BIN=$B/igneumd; [ -x $B/igneumd-0313 ] && NODE_BIN=$B/igneumd-0313 # the 0.3.13 swap's binary once box-node-swap.sh has run diff --git a/tools/fleet/box-wave.sh b/tools/fleet/box-wave.sh index 44b204b22..435cc1508 100755 --- a/tools/fleet/box-wave.sh +++ b/tools/fleet/box-wave.sh @@ -22,7 +22,7 @@ B=/opt/igneum/pkg/bin cp $F/in/igneumd-0313 $B/igneumd-0313; chmod +x $B/igneumd-0313; cp $F/in/ov13.json $F/override.json ldconfig -p | grep -q libnvrtc.so.12 || echo "RESULT note no libnvrtc.so.12 on the path (the worker dlopens it)" pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & -sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') pid=$(pgrep -f igneumd-0313 | head -1)" +sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') pid=$(pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' | head -1)" for i in $(seq 1 90); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [[ "$w" == *synced=true* ]] && break; sleep 10; done echo "RESULT synced $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //') after $((i*10)) s" cd $F/mine && rm -rf packs/devnet && $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet > $OUT/export-pack.log 2>&1 diff --git a/tools/fleet/devnet2-gate.sh b/tools/fleet/devnet2-gate.sh index b047027fb..edcfc8e7b 100755 --- a/tools/fleet/devnet2-gate.sh +++ b/tools/fleet/devnet2-gate.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # The Devnet 2 release gate (6 October 2026). Given a release's Linux igneumd (a download URL or a local file) and its # manifest (for the activation field and value, or --activation name=value), it: (1) installs the binary on every # Devnet 2 box and restarts each node on the CURRENT override, checking the version; (2) sets the activation at tip + MARGIN @@ -28,9 +28,9 @@ exit 1; } LOCAL="$ROOT/dn2-gate-igneumd" if [[ "$BIN" == http* ]]; then curl -fsSL -o "$LOCAL" "$BIN" || fail "download $BIN"; else cp "$BIN" "$LOCAL"; fi [[ "$(shasum -a 256 "$LOCAL" | cut -c1-64)" == "$SHA" ]] || fail "sha256 of the binary is $(shasum -a 256 "$LOCAL" | cut -c1-16), not ${SHA:0:16}" -VERSION_WANT="$(python3 "$HERE/fleet.py" dn2-version "$LOCAL" 2>/dev/null || true)" +VERSION_WANT="${SHA:0:16}_igneumd_2.1.0" # the running binary's sha256 (first 16) and its --version word # the boxes -mapfile -t BOXES < <(python3 - <<'PY' +BOXES=(); while IFS= read -r line; do BOXES+=("$line"); done < <(python3 - <<'PY' import json, os; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json"))) for iid,b in reg.items(): if b.get("devnet2") and b.get("state")!="destroyed" and b.get("ssh_host"): print(f"{b['label']} {b['ssh_host']} {b['ssh_port']} {int(bool(b.get('dn2_seed')))} {b.get('wallet')} {int(bool(b.get('dn2_prover')))}") @@ -48,15 +48,16 @@ PY echo "GATE boxes=${#BOXES[@]} seed=$SEED_HOST:$SEED_PORT peer=$SEED_PEER" # a reading of every box: height, exec tip and root at a height, rejects, max reorg, version, paid segments read_box() { # host port -> "height daa exec_tip paid_seg version rejects max_reorg" - SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "devnet-suffix=2" | head -1 | awk "{print \$2}"); ver=$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "reject message|PoW rejected|block rejected|invalid block" /root/fleet/dn2-node.log 2>/dev/null); mr=$(grep -oE "selected-chain reorg: [0-9]+ chain blocks" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null + SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "^/root/fleet/in/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "reject message|PoW rejected|block rejected|invalid block" /root/fleet/dn2-node.log 2>/dev/null); mr=$(grep -oE "selected-chain reorg: [0-9]+ chain blocks" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null } root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } -echo "GATE baseline $(stamp)"; declare -A PAID0 -for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; PAID0[$1]="$(awk '{print $4}' <<< "$r")"; done +WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; } +echo "GATE baseline $(stamp)" +for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; awk '{print $4}' <<< "$r" > "$WORK/paid0.$1"; done # 1. install the binary on every box, restart on the current override for l in "${BOXES[@]}"; do set -- $l SCP "$2" "$3" "$LOCAL" || fail "$1: scp of the binary" - SSH "$2" "$3" "mv /root/fleet/in/dn2-gate-igneumd /root/fleet/in/igneumd-gate && chmod +x /root/fleet/in/igneumd-gate && [ \"\$(sha256sum /root/fleet/in/igneumd-gate | cut -c1-64)\" = $SHA" || fail "$1: the binary's sha256 on the box" + SSH "$2" "$3" "mv /root/fleet/in/dn2-gate-igneumd /root/fleet/in/igneumd-gate && chmod +x /root/fleet/in/igneumd-gate && [ \"\$(sha256sum /root/fleet/in/igneumd-gate | cut -c1-64)\" = $SHA ]" || fail "$1: the binary's sha256 on the box" done restart_all() { # with the override file already on each box as /root/fleet/in/dn2-override.json for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] || continue @@ -81,18 +82,17 @@ fi # 3. run echo "GATE step3 $(stamp) running $MINUTES min"; sleep $((MINUTES * 60)) # 4. the checks -echo "GATE step4 $(stamp) checks"; H=""; declare -A ROOTS; ok=1; paid_any=0 +echo "GATE step4 $(stamp) checks"; H=""; ok=1; paid_any=0 for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r" rej="$(awk '{print $6}' <<< "$r")"; mr="$(awk '{print $7}' <<< "$r")"; v="$(awk '{print $5}' <<< "$r")"; paid="$(awk '{print $4}' <<< "$r")"; et="$(awk '{print $3}' <<< "$r")" [[ "$rej" == 0 ]] || { echo "FAIL $1: $rej rejected blocks (grep 'reject' /root/fleet/dn2-node.log)"; ok=0; } [[ "${mr:-0}" -le 3 ]] || { echo "FAIL $1: a selected-chain reorg of depth $mr"; ok=0; } [[ -z "$VERSION_WANT" || "$v" == "$VERSION_WANT" ]] || { echo "FAIL $1: version $v"; ok=0; } - [[ "${paid:-0}" -gt "${PAID0[$1]:-0}" ]] && paid_any=1 + [[ "${paid:-0}" -gt "$(paid0 $1)" ]] && paid_any=1 [[ -z "$H" || "$et" -lt "$H" ]] && H="$et" done H=$((H > 20 ? H - 20 : 1)); HX="$(printf '0x%x' "$H")" -for l in "${BOXES[@]}"; do set -- $l; ROOTS[$1]="$(root_at "$2" "$3" "$HX")"; echo " $1 root@$H ${ROOTS[$1]:0:18}"; done -first=""; for k in "${!ROOTS[@]}"; do [[ -z "$first" ]] && first="${ROOTS[$k]}"; [[ "${ROOTS[$k]}" == "$first" ]] || { echo "FAIL $k: exec root at height $H ${ROOTS[$k]:0:18} differs from ${first:0:18}"; ok=0; }; done +first=""; for l in "${BOXES[@]}"; do set -- $l; rt="$(root_at "$2" "$3" "$HX")"; echo " $1 root@$H ${rt:0:18}"; [[ -z "$first" ]] && first="$rt"; [[ "$rt" == "$first" ]] || { echo "FAIL $1: exec root at height $H ${rt:0:18} differs from ${first:0:18}"; ok=0; }; done [[ $paid_any == 1 ]] || { echo "FAIL no segment record paid on any box during the run"; ok=0; } res="FAIL"; [[ $ok == 1 ]] && res="PASS" echo "$res $(stamp) release ${SHA:0:16} activation ${ACT:-none} boxes ${#BOXES[@]} minutes $MINUTES log $LOG"