68 lines
9.3 KiB
Bash
Executable file
68 lines
9.3 KiB
Bash
Executable file
#!/bin/bash
|
|
# bash 3.2 (the Mac's): plain arrays, no mapfile, no associative arrays (lookup files under $WORK)
|
|
# The 0.3.14 canary gate on the LIVE devnet (6 October 2026, 17:00Z, the coordinator's form for a publish that moves no
|
|
# digest and no height): the release's Linux igneumd goes onto the six restart-path provers and two miners within two
|
|
# minutes of the URL (the override file kept), runs TEN minutes, then one line: PASS only with zero rejected blocks on
|
|
# every box, exec state roots equal on every box and on the hub at a common height, at least one segment record paid on
|
|
# the new binary (a prover's paidSegments up), every box's node on the new version; FAIL names the box and the line.
|
|
# canary.sh --binary <url|file> --sha256 <hex> [--minutes 10]
|
|
set -uo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$HOME/Desktop/fleet"
|
|
BIN=""; SHA=""; MINUTES=10
|
|
while [[ $# -gt 0 ]]; do case "$1" in --binary) BIN="$2"; shift 2 ;; --sha256) SHA="$2"; shift 2 ;; --minutes) MINUTES="$2"; shift 2 ;; *) echo "unknown $1"; exit 2 ;; esac; done
|
|
[[ -n "$BIN" && -n "$SHA" ]] || { echo "usage: $0 --binary <url|file> --sha256 <hex> [--minutes 10]"; exit 2; }
|
|
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
LOG="$ROOT/canary-$(date -u +%Y%m%dT%H%M%SZ).log"; exec > >(tee -a "$LOG") 2>&1
|
|
PROVERS="p1-3090 p1-5090 p2-3090-1 p2-3090-2 p2-3090-3 p2-4090-1b"; MINERS="p1-3080 p1-a5000"
|
|
fail() { echo "FAIL $(stamp) $*"; exit 1; }
|
|
LOCAL="$ROOT/canary-igneumd"
|
|
if [[ "$BIN" == http* ]]; then curl -fsSL -o "$LOCAL" "$BIN" || fail "download $BIN"; else cp "$BIN" "$LOCAL"; fi
|
|
[[ "$(shasum -a 256 "$LOCAL" | cut -c1-64)" == "$SHA" ]] || fail "sha256 of the binary is $(shasum -a 256 "$LOCAL" | cut -c1-16), not ${SHA:0:16}"
|
|
WANT="${SHA:0:16}_$(file "$LOCAL" >/dev/null; echo igneumd_2.1.0)" # the running binary's sha256 (first 16) and its --version word, per box
|
|
echo "CANARY start $(stamp) binary=${SHA:0:16} version=$WANT boxes: $PROVERS $MINERS"
|
|
ROWS=(); while IFS= read -r line; do ROWS+=("$line"); done < <(python3 - "$PROVERS $MINERS" <<'PY'
|
|
import json, os, sys; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json"))); want=sys.argv[1].split()
|
|
for lab in want:
|
|
b=next((v for v in reg.values() if v["label"]==lab and v.get("state")!="destroyed"), None)
|
|
if b: print(lab, b["ssh_host"], b["ssh_port"], b["wallet"])
|
|
hub=next(v for v in reg.values() if v.get("hub") and v.get("state")!="destroyed" and v.get("hub_peer")); print("hub-1", hub["ssh_host"], hub["ssh_port"], hub["wallet"], hub["hub_peer"])
|
|
PY
|
|
)
|
|
HUB="$(printf '%s\n' "${ROWS[@]}" | awk '$1=="hub-1"')"; HUB_HOST="$(awk '{print $2}' <<< "$HUB")"; HUB_PORT="$(awk '{print $3}' <<< "$HUB")"; HUB_PEER="$(awk '{print $5}' <<< "$HUB")"
|
|
SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; }
|
|
read_box() { SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); ex=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getExecStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"executedTip\",\"0x0\"),16))" 2>/dev/null); v=$(pgrep -fa "^/opt/igneum/pkg/bin/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "got reject message|PoW rejected|block rejected|invalid block" /root/fleet/node.log 2>/dev/null); echo "${d:-0} ${e:-0 0} ${ex:-0} ${ver:-none} ${rej:-0}"' 2>/dev/null; }
|
|
root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; }
|
|
WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; }; rej0() { cat "$WORK/rej0.$1" 2>/dev/null || echo 0; }
|
|
echo "CANARY baseline $(stamp)"
|
|
for r in "${ROWS[@]}"; do set -- $r; x="$(read_box "$2" "$3")"; echo " $1: daa=$(awk '{print $1}' <<< "$x") tip=$(awk '{print $2}' <<< "$x") paidSeg=$(awk '{print $3}' <<< "$x") exec=$(awk '{print $4}' <<< "$x") ver=$(awk '{print $5}' <<< "$x") rej=$(awk '{print $6}' <<< "$x")"; awk '{print $3}' <<< "$x" > "$WORK/paid0.$1"; awk '{print $6}' <<< "$x" > "$WORK/rej0.$1"; done
|
|
# install: the binary swap with the override file kept (box-node-swap.sh STEP=binary, which refuses on a digest change)
|
|
T0=$(date +%s)
|
|
for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue
|
|
okc=0; for try in 1 2 3; do scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -P "$3" "$LOCAL" "root@$2:/root/fleet/in/igneumd-0313" >/dev/null 2>&1 && { okc=1; break; }; sleep 5; done # Vast's ssh proxy drops a connection now and then (17:14Z); three tries before a FAIL
|
|
[[ $okc == 1 ]] || fail "$1: scp failed three times"
|
|
oks=0; for try in 1 2 3; do SSH "$2" "$3" "cd /root/fleet && chmod +x in/igneumd-0313 && [ \"\$(sha256sum in/igneumd-0313 | cut -c1-64)\" = $SHA ] && mv out/node-swap.log out/node-swap-canary-prev.log 2>/dev/null; STEP=binary NODE_SHA256=$SHA EXPECT_DIGEST=b18ed271f75dd46406d230f4156c37472127415a4c32c558bac662f6f840e61c HUB_PEER=$HUB_PEER HUB_SSH=$HUB_HOST HUB_PORT=$HUB_PORT setsid nohup in/box-node-swap.sh </dev/null >/dev/null 2>&1 & echo swapped" >/dev/null 2>&1 && { oks=1; break; }; sleep 5; done
|
|
[[ $oks == 1 ]] || fail "$1: install failed three times"
|
|
done
|
|
echo "CANARY installed $(stamp) on $(( ${#ROWS[@]} - 1 )) boxes in $(( $(date +%s) - T0 )) s"
|
|
sleep 75
|
|
for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue
|
|
l="$(SSH "$2" "$3" "grep -E '^RESULT (node_started|swap_failed)' /root/fleet/out/node-swap.log | tail -1 | cut -c1-140")"; echo " $1: $l"; [[ "$l" == *digest_ok* ]] || fail "$1: the swap did not reach digest_ok ($l)"
|
|
done
|
|
# the provers back on (box-prover.sh keeps a running igneumd-0313 with the verifier)
|
|
for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh </dev/null >/dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done
|
|
for p in $MINERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet/mine && pkill -x igneum-miner; sleep 2; rm -rf packs/devnet; /opt/igneum/pkg/bin/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >/dev/null 2>&1; setsid nohup /opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 $1 --worker /opt/igneum/pkg/bin/igneum-worker-cuda --worker-args '--device 0 --pack packs/devnet' --prepare-packs packs/prepare --exit-on-seed-change --evm-address $4 --payout-label $1 --status-secs 30 </dev/null >> /root/fleet/out/mine-only-0.log 2>&1 & echo miner" >/dev/null || fail "$1: miner relaunch"; done
|
|
echo "CANARY running $(stamp) $MINUTES min"; sleep $((MINUTES * 60))
|
|
echo "CANARY checks $(stamp)"; ok=1; paid_any=0; H=""
|
|
for r in "${ROWS[@]}"; do set -- $r; x="$(read_box "$2" "$3")"; paid="$(awk '{print $3}' <<< "$x")"; ex="$(awk '{print $4}' <<< "$x")"; ver="$(awk '{print $5}' <<< "$x")"; rej="$(awk '{print $6}' <<< "$x")"
|
|
echo " $1: tip=$(awk '{print $2}' <<< "$x") paidSeg=$paid exec=$ex ver=$ver rej=$rej (was $(rej0 $1))"
|
|
[[ "$1" == hub-1 ]] || { [[ "$ver" == "$WANT" ]] || { echo "FAIL $1: version $ver, want $WANT"; ok=0; }; }
|
|
[[ "${rej:-0}" -le "$(rej0 $1)" ]] || { echo "FAIL $1: $(( rej - $(rej0 $1) )) new rejected blocks (grep 'reject' /root/fleet/node.log)"; ok=0; }
|
|
[[ "$1" == hub-1 ]] || { for p in $PROVERS; do [[ "$1" == "$p" && "${paid:-0}" -gt "$(paid0 $1)" ]] && paid_any=1; done; }
|
|
[[ -z "$H" || "${ex:-0}" -lt "$H" ]] && H="${ex:-0}"
|
|
done
|
|
H=$((H > 20 ? H - 20 : 1)); HX="$(printf '0x%x' "$H")"; first=""
|
|
for r in "${ROWS[@]}"; do set -- $r; rt="$(root_at "$2" "$3" "$HX")"; echo " $1 root@$H ${rt:0:18}"; [[ -z "$first" ]] && first="$rt"; [[ "$rt" == "$first" ]] || { echo "FAIL $1: exec root at $H ${rt:0:18} differs from ${first:0:18}"; ok=0; }; done
|
|
[[ $paid_any == 1 ]] || { echo "FAIL no segment record paid on any prover during the $MINUTES min (paidSegments did not rise)"; ok=0; }
|
|
res=FAIL; [[ $ok == 1 ]] && res=PASS
|
|
echo "$res $(stamp) canary 0.3.14 ${SHA:0:16} version $WANT: $(( ${#ROWS[@]} - 1 )) boxes, $MINUTES min, rejects 0, roots equal at $H incl. the hub, segment paid $paid_any; log $LOG"
|
|
[[ $ok == 1 ]]
|