106 lines
10 KiB
Bash
Executable file
106 lines
10 KiB
Bash
Executable file
#!/bin/bash
|
|
# The Devnet 2 release gate (6 October 2026). Given a release's Linux igneumd (a download URL or a local file) and its
|
|
# manifest (for the activation field and value, or --activation name=value), it: (1) installs the binary on every
|
|
# Devnet 2 box and restarts each node on the CURRENT override, checking the version; (2) sets the activation at tip + MARGIN
|
|
# in a new override, restarts every box on it (the seed first) and waits for the crossing; (3) runs N minutes past it;
|
|
# (4) prints PASS only when: zero rejected blocks on every box, no selected-chain reorg over depth 3, exec state roots
|
|
# equal on every box at a common height, at least one segment record paid since the start, every node's version equal
|
|
# to the release's. FAIL names the box and the line. The one-line status goes to ~/Desktop/fleet/devnet2-status.json.
|
|
#
|
|
# devnet2-gate.sh --binary <url|file> --sha256 <hex> [--activation proving_v1_fresh_rule_daa=NNN | --activation-field F]
|
|
# [--margin 600] [--minutes 20] [--dry-run] (dry-run: steps 1 and 4 against the current object, no activation)
|
|
set -uo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$HOME/Desktop/fleet"
|
|
BIN=""; SHA=""; ACT=""; ACT_FIELD=""; MARGIN=600; MINUTES=20; DRY=0
|
|
while [[ $# -gt 0 ]]; do case "$1" in
|
|
--binary) BIN="$2"; shift 2 ;; --sha256) SHA="$2"; shift 2 ;; --activation) ACT="$2"; shift 2 ;; --activation-field) ACT_FIELD="$2"; shift 2 ;;
|
|
--margin) MARGIN="$2"; shift 2 ;; --minutes) MINUTES="$2"; shift 2 ;; --dry-run) DRY=1; shift ;; *) echo "unknown $1"; exit 2 ;; esac; done
|
|
[[ -n "$BIN" && -n "$SHA" ]] || { echo "usage: $0 --binary <url|file> --sha256 <hex> [--activation name=value] [--minutes N] [--dry-run]"; exit 2; }
|
|
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
LOG="$ROOT/devnet2-gate-$(date -u +%Y%m%dT%H%M%SZ).log"; exec > >(tee -a "$LOG") 2>&1
|
|
echo "GATE start $(stamp) binary=$BIN sha256=${SHA:0:16} activation=${ACT:-none} minutes=$MINUTES dry_run=$DRY"
|
|
fail() { echo "FAIL $(stamp) $*"; python3 - "$ROOT" "FAIL: $*" <<'PY'
|
|
import json, sys, os, datetime; p=os.path.join(sys.argv[1], "devnet2-status.json"); d=json.load(open(p)) if os.path.exists(p) else {}
|
|
d.update({"last_gate": sys.argv[2][:200], "last_gate_at": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")}); json.dump(d, open(p, "w"))
|
|
PY
|
|
exit 1; }
|
|
# the binary, local
|
|
LOCAL="$ROOT/dn2-gate-igneumd"
|
|
if [[ "$BIN" == http* ]]; then curl -fsSL -o "$LOCAL" "$BIN" || fail "download $BIN"; else cp "$BIN" "$LOCAL"; fi
|
|
[[ "$(shasum -a 256 "$LOCAL" | cut -c1-64)" == "$SHA" ]] || fail "sha256 of the binary is $(shasum -a 256 "$LOCAL" | cut -c1-16), not ${SHA:0:16}"
|
|
VERSION_WANT="${SHA:0:16}_igneumd_2.1.0" # the running binary's sha256 (first 16) and its --version word
|
|
# the boxes
|
|
BOXES=(); while IFS= read -r line; do BOXES+=("$line"); done < <(python3 - <<'PY'
|
|
import json, os; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json")))
|
|
for iid,b in reg.items():
|
|
if b.get("devnet2") and b.get("state")!="destroyed" and b.get("ssh_host"): print(f"{b['label']} {b['ssh_host']} {b['ssh_port']} {int(bool(b.get('dn2_seed')))} {b.get('wallet')} {int(bool(b.get('dn2_prover')))}")
|
|
PY
|
|
)
|
|
[[ ${#BOXES[@]} -gt 0 ]] || fail "no Devnet 2 boxes in the registry"
|
|
SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; }
|
|
SCP() { scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -P "$2" "${@:3}" "root@$1:/root/fleet/in/" >/dev/null; }
|
|
seed_line="$(printf '%s\n' "${BOXES[@]}" | awk '$4==1 {print; exit}')"; SEED_HOST="$(awk '{print $2}' <<< "$seed_line")"; SEED_PORT="$(awk '{print $3}' <<< "$seed_line")"
|
|
SEED_PEER="$(python3 - <<'PY'
|
|
import json, os; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json")))
|
|
print(next((b.get("dn2_peer","") for b in reg.values() if b.get("dn2_seed") and b.get("state")!="destroyed"), ""))
|
|
PY
|
|
)"
|
|
echo "GATE boxes=${#BOXES[@]} seed=$SEED_HOST:$SEED_PORT peer=$SEED_PEER"
|
|
# a reading of every box: height, exec tip and root at a height, rejects, max reorg, version, paid segments
|
|
read_box() { # host port -> "height daa exec_tip paid_seg version rejects max_reorg"
|
|
SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "^/root/fleet/in/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "reject message|PoW rejected|block rejected|invalid block" /root/fleet/dn2-node.log 2>/dev/null); mr=$(grep -oE "selected-chain reorg: [0-9]+ chain blocks" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null
|
|
}
|
|
root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; }
|
|
WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; }
|
|
echo "GATE baseline $(stamp)"
|
|
for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; awk '{print $4}' <<< "$r" > "$WORK/paid0.$1"; done
|
|
# 1. install the binary on every box, restart on the current override
|
|
for l in "${BOXES[@]}"; do set -- $l
|
|
SCP "$2" "$3" "$LOCAL" || fail "$1: scp of the binary"
|
|
SSH "$2" "$3" "mv /root/fleet/in/dn2-gate-igneumd /root/fleet/in/igneumd-gate && chmod +x /root/fleet/in/igneumd-gate && [ \"\$(sha256sum /root/fleet/in/igneumd-gate | cut -c1-64)\" = $SHA ]" || fail "$1: the binary's sha256 on the box"
|
|
done
|
|
restart_all() { # with the override file already on each box as /root/fleet/in/dn2-override.json
|
|
for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] || continue
|
|
SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f '^/root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh </dev/null >/dev/null 2>&1 & echo restarted" || fail "$1: restart"
|
|
sleep 15; done
|
|
for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] && continue
|
|
SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f '^/root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 SEED=$SEED_PEER NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh </dev/null >/dev/null 2>&1 & echo restarted" || fail "$1: restart"
|
|
done
|
|
}
|
|
echo "GATE step1 $(stamp) restart on the release binary, the current override"; restart_all; sleep 60
|
|
for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; v="$(awk '{print $5}' <<< "$r")"; [[ -z "$VERSION_WANT" || "$v" == "$VERSION_WANT" ]] || fail "$1: version $v, want $VERSION_WANT"; done
|
|
# 2. the activation
|
|
if [[ $DRY == 0 && -n "$ACT" ]]; then
|
|
tip="$(read_box "$SEED_HOST" "$SEED_PORT" | awk '{print $2}')"; name="${ACT%%=*}"; val="${ACT#*=}"; [[ "$val" == auto ]] && val=$((tip + MARGIN))
|
|
python3 - "$HERE/devnet2-override.json" "$name" "$val" "$ROOT/dn2-override-next.json" <<'PY'
|
|
import json, sys; d=json.load(open(sys.argv[1])); d[sys.argv[2]]=int(sys.argv[3]); json.dump(d, open(sys.argv[4], "w"))
|
|
PY
|
|
for l in "${BOXES[@]}"; do set -- $l; SCP "$2" "$3" "$ROOT/dn2-override-next.json" && SSH "$2" "$3" "mv /root/fleet/in/dn2-override-next.json /root/fleet/in/dn2-override.json" || fail "$1: override push"; done
|
|
echo "GATE step2 $(stamp) activation $name=$val at tip $tip (margin $MARGIN); restart on the new object"; restart_all
|
|
for i in $(seq 1 120); do
|
|
sleep 15; d="$(read_box "$SEED_HOST" "$SEED_PORT" | awk '{print $2}')"
|
|
if [[ "${d:-0}" -ge "$val" ]]; then echo "GATE crossed $name=$val at $(stamp), daa $d"; break; fi
|
|
done
|
|
fi
|
|
# 3. run
|
|
echo "GATE step3 $(stamp) running $MINUTES min"; sleep $((MINUTES * 60))
|
|
# 4. the checks
|
|
echo "GATE step4 $(stamp) checks"; H=""; ok=1; paid_any=0
|
|
for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"
|
|
rej="$(awk '{print $6}' <<< "$r")"; mr="$(awk '{print $7}' <<< "$r")"; v="$(awk '{print $5}' <<< "$r")"; paid="$(awk '{print $4}' <<< "$r")"; et="$(awk '{print $3}' <<< "$r")"
|
|
[[ "$rej" == 0 ]] || { echo "FAIL $1: $rej rejected blocks (grep 'reject' /root/fleet/dn2-node.log)"; ok=0; }
|
|
[[ "${mr:-0}" -le 3 ]] || { echo "FAIL $1: a selected-chain reorg of depth $mr"; ok=0; }
|
|
[[ -z "$VERSION_WANT" || "$v" == "$VERSION_WANT" ]] || { echo "FAIL $1: version $v"; ok=0; }
|
|
[[ "${paid:-0}" -gt "$(paid0 $1)" ]] && paid_any=1
|
|
[[ -z "$H" || "$et" -lt "$H" ]] && H="$et"
|
|
done
|
|
H=$((H > 20 ? H - 20 : 1)); HX="$(printf '0x%x' "$H")"
|
|
first=""; for l in "${BOXES[@]}"; do set -- $l; rt="$(root_at "$2" "$3" "$HX")"; echo " $1 root@$H ${rt:0:18}"; [[ -z "$first" ]] && first="$rt"; [[ "$rt" == "$first" ]] || { echo "FAIL $1: exec root at height $H ${rt:0:18} differs from ${first:0:18}"; ok=0; }; done
|
|
[[ $paid_any == 1 ]] || { echo "FAIL no segment record paid on any box during the run"; ok=0; }
|
|
res="FAIL"; [[ $ok == 1 ]] && res="PASS"
|
|
echo "$res $(stamp) release ${SHA:0:16} activation ${ACT:-none} boxes ${#BOXES[@]} minutes $MINUTES log $LOG"
|
|
python3 - "$ROOT" "$res: ${SHA:0:16} ${ACT:-current object}, ${#BOXES[@]} boxes, $MINUTES min" <<'PY'
|
|
import json, sys, os, datetime; p=os.path.join(sys.argv[1], "devnet2-status.json"); d=json.load(open(p)) if os.path.exists(p) else {}
|
|
d.update({"last_gate": sys.argv[2], "last_gate_at": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")}); json.dump(d, open(p, "w"))
|
|
PY
|
|
[[ $ok == 1 ]]
|