fleet scripts: every process pattern anchored (the pgrep-self-match check passes on tools/fleet); the observer's autosync on the owed list
This commit is contained in:
igneum-labs
parent
c40af44e0f
commit
d30dc44213
6 changed files with 16 additions and 16 deletions
|
|
@ -3,4 +3,4 @@
|
|||
| Check | What it fails | Since |
|
||||
|---|---|---|
|
||||
|
||||
| `pgrep-self-match-check.sh` | a `pgrep -f` / `pkill -f` with a bare literal pattern, or `ps \| grep <word>` without a bracket or `grep -v grep`, in tools/, relay/playbooks/, infra/ or packaging/: the pattern matches the shell that runs it (the wave script of 6 October 2026 never started a node on 38 cards because `pgrep -f igneumd-0313` saw the launching shell; a kill file killed its caller the same day). Anchor to the executable's path, bracket the first letter, or use `-x`. Owed (allow-listed, finished measurements): `tools/prover-floor/pc2-*.ps1`, `tools/proving-v1/pc2-*.ps1` (their `pkill -f sp1-gpu-server` becomes `pkill -x`), `tools/repo/fresh-repo.sh:223` | 6 October 2026, branch gpu-fleet |
|
||||
| `pgrep-self-match-check.sh` | a `pgrep -f` / `pkill -f` with a bare literal pattern, or `ps \| grep <word>` without a bracket or `grep -v grep`, in tools/, relay/playbooks/, infra/ or packaging/: the pattern matches the shell that runs it (the wave script of 6 October 2026 never started a node on 38 cards because `pgrep -f igneumd-0313` saw the launching shell; a kill file killed its caller the same day). Anchor to the executable's path, bracket the first letter, or use `-x`. Owed (allow-listed, finished measurements): `tools/prover-floor/pc2-*.ps1`, `tools/proving-v1/pc2-*.ps1` (their `pkill -f sp1-gpu-server` becomes `pkill -x`), `tools/repo/fresh-repo.sh:223`, `tools/observer/autosync.sh:22` | 6 October 2026, branch gpu-fleet |
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ fi
|
|||
# Owed, not exempt: the PC 2 playbooks of 5 and 6 October use `pkill -f sp1-gpu-server` (the prover-socket check's own
|
||||
# required line) inside a WSL `bash -c` whose command line carries the word, so the pkill kills that shell too when it
|
||||
# runs first; they are finished measurements and get `pkill -x sp1-gpu-server` when next touched (tools/ci/README.md).
|
||||
ALLOW='^(tools/prover-floor/pc2-.*\.ps1|tools/proving-v1/pc2-.*\.ps1|tools/repo/fresh-repo\.sh)$'
|
||||
ALLOW='^(tools/prover-floor/pc2-.*\.ps1|tools/proving-v1/pc2-.*\.ps1|tools/repo/fresh-repo\.sh|tools/observer/autosync\.sh)$'
|
||||
list_files() { if [ $# -gt 0 ]; then printf '%s\n' "$@"; else git ls-files 'tools/**' 'relay/playbooks/**' 'infra/**' 'packaging/**' | grep -E '\.(sh|bash|ps1|mjs|py)$'; fi; }
|
||||
while IFS= read -r f; do [ -f "$f" ] || continue; [[ "$f" =~ $ALLOW ]] && continue; check_file "$f"; done < <(list_files "$@")
|
||||
[ "$fail" = 0 ] && echo "pgrep-self-match: no script matches its own shell"
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ PEER=""; [ -n "$SEED" ] && PEER="--addpeer=$SEED"
|
|||
IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host nohup $NODE_BIN --devnet --devnet-suffix=2 --appdir=$F/dn2 --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 $PEER --override-params-file=$F/dn2-override.json --nodnsseed --disable-upnp --nologfiles --yes --enable-unsynced-mining ${NODE_EXTRA:-} >> $F/dn2-node.log 2>&1 &
|
||||
# (--enable-unsynced-mining: a fresh chain's nodes start unsynced and must mine anyway, Reject(IsInIBD) on the seed at 16:52Z; a comment put inside this line at 17:00Z swallowed the redirect and the ampersand, so the node ran in the foreground and the script never reached the miner)
|
||||
sleep 10
|
||||
echo "RESULT dn2_node $(stamp) pid=$(pgrep -f 'devnet-suffix=2' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)"
|
||||
echo "RESULT dn2_node $(stamp) pid=$(pgrep -f '^/root/fleet/in/igneumd' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)"
|
||||
for i in $(seq 1 30); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [ -n "$w" ] && break; sleep 5; done
|
||||
echo "RESULT dn2_watch $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //')"
|
||||
cd $F/mine && rm -rf packs/dn2 && $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/dn2 > $OUT/dn2-export-pack.log 2>&1
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ if [ -z "${THRESHOLD:-}" ]; then
|
|||
fi
|
||||
if [ -z "${MINER:-}" ]; then if [ "$TOTAL" -ge 15000 ]; then MINER=keep; else MINER=pause; fi; fi
|
||||
echo "RESULT launch $(stamp) total_mib=$TOTAL threshold=${THRESHOLD:-default} miner=$MINER"
|
||||
pkill -f "igneum-miner mine" 2>/dev/null; pkill -f igneum-worker-cuda 2>/dev/null; pkill -f sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock
|
||||
pkill -f "igneum-miner mine" 2>/dev/null; pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock
|
||||
pkill -x igneumd; sleep 4; pkill -9 -x igneumd 2>/dev/null; sleep 1
|
||||
HUBARG=""; [ -n "${HUB_PEER:-}" ] && HUBARG="--addpeer=$HUB_PEER" # the fleet's hub box (a second peer beside the seed, 12:35Z)
|
||||
NODE_BIN=$B/igneumd; [ -x $B/igneumd-0313 ] && NODE_BIN=$B/igneumd-0313 # the 0.3.13 swap's binary once box-node-swap.sh has run
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ B=/opt/igneum/pkg/bin
|
|||
cp $F/in/igneumd-0313 $B/igneumd-0313; chmod +x $B/igneumd-0313; cp $F/in/ov13.json $F/override.json
|
||||
ldconfig -p | grep -q libnvrtc.so.12 || echo "RESULT note no libnvrtc.so.12 on the path (the worker dlopens it)"
|
||||
pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 &
|
||||
sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') pid=$(pgrep -f igneumd-0313 | head -1)"
|
||||
sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') pid=$(pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' | head -1)"
|
||||
for i in $(seq 1 90); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [[ "$w" == *synced=true* ]] && break; sleep 10; done
|
||||
echo "RESULT synced $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //') after $((i*10)) s"
|
||||
cd $F/mine && rm -rf packs/devnet && $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet > $OUT/export-pack.log 2>&1
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env bash
|
||||
#!/bin/bash
|
||||
# The Devnet 2 release gate (6 October 2026). Given a release's Linux igneumd (a download URL or a local file) and its
|
||||
# manifest (for the activation field and value, or --activation name=value), it: (1) installs the binary on every
|
||||
# Devnet 2 box and restarts each node on the CURRENT override, checking the version; (2) sets the activation at tip + MARGIN
|
||||
|
|
@ -28,9 +28,9 @@ exit 1; }
|
|||
LOCAL="$ROOT/dn2-gate-igneumd"
|
||||
if [[ "$BIN" == http* ]]; then curl -fsSL -o "$LOCAL" "$BIN" || fail "download $BIN"; else cp "$BIN" "$LOCAL"; fi
|
||||
[[ "$(shasum -a 256 "$LOCAL" | cut -c1-64)" == "$SHA" ]] || fail "sha256 of the binary is $(shasum -a 256 "$LOCAL" | cut -c1-16), not ${SHA:0:16}"
|
||||
VERSION_WANT="$(python3 "$HERE/fleet.py" dn2-version "$LOCAL" 2>/dev/null || true)"
|
||||
VERSION_WANT="${SHA:0:16}_igneumd_2.1.0" # the running binary's sha256 (first 16) and its --version word
|
||||
# the boxes
|
||||
mapfile -t BOXES < <(python3 - <<'PY'
|
||||
BOXES=(); while IFS= read -r line; do BOXES+=("$line"); done < <(python3 - <<'PY'
|
||||
import json, os; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json")))
|
||||
for iid,b in reg.items():
|
||||
if b.get("devnet2") and b.get("state")!="destroyed" and b.get("ssh_host"): print(f"{b['label']} {b['ssh_host']} {b['ssh_port']} {int(bool(b.get('dn2_seed')))} {b.get('wallet')} {int(bool(b.get('dn2_prover')))}")
|
||||
|
|
@ -48,15 +48,16 @@ PY
|
|||
echo "GATE boxes=${#BOXES[@]} seed=$SEED_HOST:$SEED_PORT peer=$SEED_PEER"
|
||||
# a reading of every box: height, exec tip and root at a height, rejects, max reorg, version, paid segments
|
||||
read_box() { # host port -> "height daa exec_tip paid_seg version rejects max_reorg"
|
||||
SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "devnet-suffix=2" | head -1 | awk "{print \$2}"); ver=$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "reject message|PoW rejected|block rejected|invalid block" /root/fleet/dn2-node.log 2>/dev/null); mr=$(grep -oE "selected-chain reorg: [0-9]+ chain blocks" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null
|
||||
SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "^/root/fleet/in/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "reject message|PoW rejected|block rejected|invalid block" /root/fleet/dn2-node.log 2>/dev/null); mr=$(grep -oE "selected-chain reorg: [0-9]+ chain blocks" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null
|
||||
}
|
||||
root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; }
|
||||
echo "GATE baseline $(stamp)"; declare -A PAID0
|
||||
for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; PAID0[$1]="$(awk '{print $4}' <<< "$r")"; done
|
||||
WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; }
|
||||
echo "GATE baseline $(stamp)"
|
||||
for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; awk '{print $4}' <<< "$r" > "$WORK/paid0.$1"; done
|
||||
# 1. install the binary on every box, restart on the current override
|
||||
for l in "${BOXES[@]}"; do set -- $l
|
||||
SCP "$2" "$3" "$LOCAL" || fail "$1: scp of the binary"
|
||||
SSH "$2" "$3" "mv /root/fleet/in/dn2-gate-igneumd /root/fleet/in/igneumd-gate && chmod +x /root/fleet/in/igneumd-gate && [ \"\$(sha256sum /root/fleet/in/igneumd-gate | cut -c1-64)\" = $SHA" || fail "$1: the binary's sha256 on the box"
|
||||
SSH "$2" "$3" "mv /root/fleet/in/dn2-gate-igneumd /root/fleet/in/igneumd-gate && chmod +x /root/fleet/in/igneumd-gate && [ \"\$(sha256sum /root/fleet/in/igneumd-gate | cut -c1-64)\" = $SHA ]" || fail "$1: the binary's sha256 on the box"
|
||||
done
|
||||
restart_all() { # with the override file already on each box as /root/fleet/in/dn2-override.json
|
||||
for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] || continue
|
||||
|
|
@ -81,18 +82,17 @@ fi
|
|||
# 3. run
|
||||
echo "GATE step3 $(stamp) running $MINUTES min"; sleep $((MINUTES * 60))
|
||||
# 4. the checks
|
||||
echo "GATE step4 $(stamp) checks"; H=""; declare -A ROOTS; ok=1; paid_any=0
|
||||
echo "GATE step4 $(stamp) checks"; H=""; ok=1; paid_any=0
|
||||
for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"
|
||||
rej="$(awk '{print $6}' <<< "$r")"; mr="$(awk '{print $7}' <<< "$r")"; v="$(awk '{print $5}' <<< "$r")"; paid="$(awk '{print $4}' <<< "$r")"; et="$(awk '{print $3}' <<< "$r")"
|
||||
[[ "$rej" == 0 ]] || { echo "FAIL $1: $rej rejected blocks (grep 'reject' /root/fleet/dn2-node.log)"; ok=0; }
|
||||
[[ "${mr:-0}" -le 3 ]] || { echo "FAIL $1: a selected-chain reorg of depth $mr"; ok=0; }
|
||||
[[ -z "$VERSION_WANT" || "$v" == "$VERSION_WANT" ]] || { echo "FAIL $1: version $v"; ok=0; }
|
||||
[[ "${paid:-0}" -gt "${PAID0[$1]:-0}" ]] && paid_any=1
|
||||
[[ "${paid:-0}" -gt "$(paid0 $1)" ]] && paid_any=1
|
||||
[[ -z "$H" || "$et" -lt "$H" ]] && H="$et"
|
||||
done
|
||||
H=$((H > 20 ? H - 20 : 1)); HX="$(printf '0x%x' "$H")"
|
||||
for l in "${BOXES[@]}"; do set -- $l; ROOTS[$1]="$(root_at "$2" "$3" "$HX")"; echo " $1 root@$H ${ROOTS[$1]:0:18}"; done
|
||||
first=""; for k in "${!ROOTS[@]}"; do [[ -z "$first" ]] && first="${ROOTS[$k]}"; [[ "${ROOTS[$k]}" == "$first" ]] || { echo "FAIL $k: exec root at height $H ${ROOTS[$k]:0:18} differs from ${first:0:18}"; ok=0; }; done
|
||||
first=""; for l in "${BOXES[@]}"; do set -- $l; rt="$(root_at "$2" "$3" "$HX")"; echo " $1 root@$H ${rt:0:18}"; [[ -z "$first" ]] && first="$rt"; [[ "$rt" == "$first" ]] || { echo "FAIL $1: exec root at height $H ${rt:0:18} differs from ${first:0:18}"; ok=0; }; done
|
||||
[[ $paid_any == 1 ]] || { echo "FAIL no segment record paid on any box during the run"; ok=0; }
|
||||
res="FAIL"; [[ $ok == 1 ]] && res="PASS"
|
||||
echo "$res $(stamp) release ${SHA:0:16} activation ${ACT:-none} boxes ${#BOXES[@]} minutes $MINUTES log $LOG"
|
||||
|
|
|
|||
Loading…
Reference in a new issue