bb3e676f95
Main, 7 October 2026: a 100 s hook gate on the merge commit against a master that moves every minute lost six pushes in a row. Self-test: a fixture repository (unstamped branch, stamped branch, stale stamp, wrong tip, plain commit, dirty tree). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
128 lines
12 KiB
Bash
Executable file
128 lines
12 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# The 0.3.20 FIRST wave (the shipper, 7 October 2026): the publish carries a floor-moved override file whose digest differs from
|
|
# the live one, and a node on the old file refuses a node on the new one as a peer, so the three testnet seeds and the hands move
|
|
# AT the publish minute, in parallel with the fleet's wave. Dry run by default (every line printed, no box touched); --go executes.
|
|
#
|
|
# infra/build-server/wave1-0320.sh seeds [--override <file>] --igneumd <seed-class igneumd> --sha256 <hex> [--miner <seed-class igneum-miner>] \
|
|
# --digest <hex> [--commit <sha8>] [--wipe-genesis --genesis <hex>] [--go]
|
|
# the three seeds in PARALLEL (seed1/2/3.testnet, root over the ops key, infra/seed-nodes/seeds-testnet.tsv): the binary put
|
|
# as /opt/igneum/bin/igneumd.new with its sha256 asserted on the box, the override written to /etc/igneum/override-params.json,
|
|
# EXTRA_ARGS in /etc/igneum/seed.env set to --override-params-file=<that>, the unit igneumd stopped, the binary swapped (the old
|
|
# one kept as igneumd.prev), the unit started on its kept datadir, then the read-back: commit string in the installed binary,
|
|
# the "Consensus params digest" line of the new run against --digest, eth_syncing over the loopback EVM RPC, peers; one line
|
|
# per seed, logs in the scratch directory. Without --override the seeds' env is left as it is (no override today).
|
|
# --wipe-genesis (the testnet go, docs/plans/testnet-go.md runbook step 2; the shipper, 7 Oct 2026): the genesis re-cut changes
|
|
# the genesis itself, so a binary put onto the kept datadir would refuse its own genesis; with the unit stopped the datadir
|
|
# /var/lib/igneum/igneum-testnet-1 (height 0, nothing but genesis) is moved aside as igneum-testnet-1.prev-<stamp> and the node
|
|
# starts fresh; the read-back adds the "[igneum-exec] genesis <hash> executed" line against --genesis (MATCH/MISMATCH)
|
|
# infra/build-server/wave1-0320.sh hands --node <fork worktree on the Mac> --override-json '<object>' --digest <hex> [--go]
|
|
# the hands through infra/build-server/hands/move-hand.sh: binary (build + install + override), restart observer-node, restart
|
|
# node1, each read back by that tool (first executing line, commit string, digest, powEngine)
|
|
# infra/build-server/wave1-0320.sh lift-rpc-filter [--go]
|
|
# on seed1 only, AFTER the three read-backs: BLOCKED_UNTIL_FIXED_NODE in /opt/igneum/bin/rpc-filter.py becomes the empty set
|
|
# (the N7 class is fixed from 8097d600; backup kept as rpc-filter.py.bak-<stamp>), python3 -m py_compile, the unit
|
|
# igneum-rpc-filter restarted, read back: unit active and eth_blockNumber answered through https://rpc.testnet.igneum.network
|
|
set -euo pipefail
|
|
HERE=$(cd "$(dirname "$0")" && pwd); ROOT=$(cd "$HERE/../.." && pwd)
|
|
SEEDS_TSV="$ROOT/infra/seed-nodes/seeds-testnet.tsv"; KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
|
|
S="${WAVE_LOG_DIR:-/private/tmp/claude-501/-Users-joshm/cd75457f-4858-4f86-9634-7481ee056b7b/scratchpad/wave1-0320}"; mkdir -p "$S"
|
|
SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15)
|
|
now() { TZ=Europe/London date '+%H:%M:%S %Z'; }
|
|
say() { echo "$(now) wave1: $*" >&2; }
|
|
die() { say "ERROR: $*"; exit 1; }
|
|
seed_ip() { awk -F'\t' -v n="$1" '$1==n {print $4}' "$SEEDS_TSV"; }
|
|
mode="${1:-}"; [ -n "$mode" ] || { sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }; shift
|
|
GO=0; OVERRIDE=""; BIN=""; SHA=""; MINER=""; DIGEST=""; COMMIT="c4459193"; NODE=""; OVJSON=""; WIPE=0; GENESIS=""
|
|
while [ $# -gt 0 ]; do case "$1" in
|
|
--go) GO=1; shift ;; --override) OVERRIDE="$2"; shift 2 ;; --igneumd) BIN="$2"; shift 2 ;; --sha256) SHA="$2"; shift 2 ;;
|
|
--miner) MINER="$2"; shift 2 ;; --digest) DIGEST="$2"; shift 2 ;; --commit) COMMIT="$2"; shift 2 ;; --node) NODE="$2"; shift 2 ;;
|
|
--override-json) OVJSON="$2"; shift 2 ;; --wipe-genesis) WIPE=1; shift ;; --genesis) GENESIS="$2"; shift 2 ;; *) die "unknown option $1" ;; esac; done
|
|
|
|
one_seed() { # <name>: runs in the background; prints one RESULT line at the end
|
|
local name="$1" ip log t0 line
|
|
ip=$(seed_ip "$name"); [ -n "$ip" ] || { echo "RESULT $name: no ip in $SEEDS_TSV"; return 1; }
|
|
log="$S/$name.log"; : > "$log"; t0=$(date +%s)
|
|
{
|
|
if [ "$GO" = 0 ]; then
|
|
echo "DRY: scp $BIN root@$ip:/opt/igneum/bin/igneumd.new; sha256 asserted = $SHA"
|
|
[ -n "$MINER" ] && echo "DRY: scp $MINER root@$ip:/opt/igneum/bin/igneum-miner.new (swapped with the node)"
|
|
[ -n "$OVERRIDE" ] && echo "DRY: scp $OVERRIDE root@$ip:/etc/igneum/override-params.json; seed.env EXTRA_ARGS=--override-params-file=/etc/igneum/override-params.json" || echo "DRY: no override: seed.env untouched"
|
|
[ "$WIPE" = 1 ] && echo "DRY: with the unit stopped: mv /var/lib/igneum/igneum-testnet-1 /var/lib/igneum/igneum-testnet-1.prev-<stamp> (kept); genesis read back against $GENESIS"
|
|
echo "DRY: systemctl stop igneumd; mv igneumd igneumd.prev; mv igneumd.new igneumd; systemctl start igneumd"
|
|
echo "DRY: read back: grep -c $COMMIT in the binary; journal 'Consensus params digest' == $DIGEST; eth_syncing on 127.0.0.1:26890; peers"
|
|
echo "DRY RUN, nothing touched; box $ip answers as $("${SSH[@]}" "root@$ip" 'hostname; systemctl is-active igneumd; sha256sum /opt/igneum/bin/igneumd | cut -c1-12' 2>/dev/null | tr '\n' ' ')"
|
|
else
|
|
scp -q -i "$KEY" -o BatchMode=yes "$BIN" "root@$ip:/opt/igneum/bin/igneumd.new"
|
|
[ -n "$MINER" ] && scp -q -i "$KEY" -o BatchMode=yes "$MINER" "root@$ip:/opt/igneum/bin/igneum-miner.new"
|
|
[ -n "$OVERRIDE" ] && scp -q -i "$KEY" -o BatchMode=yes "$OVERRIDE" "root@$ip:/etc/igneum/override-params.json"
|
|
"${SSH[@]}" "root@$ip" bash -s -- "$SHA" "$COMMIT" "$DIGEST" "$([ -n "$MINER" ] && echo 1 || echo 0)" "$([ -n "$OVERRIDE" ] && echo 1 || echo 0)" "$WIPE" "$GENESIS" <<'REMOTE'
|
|
set -euo pipefail
|
|
SHA="$1"; COMMIT="$2"; DIGEST="$3"; MINER="$4"; OV="$5"; WIPE="$6"; GENESIS="$7"; cd /opt/igneum/bin
|
|
got=$(sha256sum igneumd.new | cut -d' ' -f1); [ "$got" = "$SHA" ] || { echo "sha256 MISMATCH on the box: $got"; exit 1; }
|
|
if [ "$OV" = 1 ]; then
|
|
python3 -c 'import json,sys; json.load(open("/etc/igneum/override-params.json"))' || { echo "override file does not parse"; exit 1; }
|
|
grep -qE '^EXTRA_ARGS=' /etc/igneum/seed.env && sed -i -E 's#^EXTRA_ARGS=.*#EXTRA_ARGS="--override-params-file=/etc/igneum/override-params.json"#' /etc/igneum/seed.env || echo 'EXTRA_ARGS="--override-params-file=/etc/igneum/override-params.json"' >> /etc/igneum/seed.env
|
|
fi
|
|
chmod 755 igneumd.new; [ "$MINER" = 1 ] && chmod 755 igneum-miner.new
|
|
t0=$(date +%s); systemctl stop igneumd
|
|
wiped=""
|
|
if [ "$WIPE" = 1 ] && [ -d /var/lib/igneum/igneum-testnet-1 ]; then stamp=$(date -u +%Y%m%dT%H%M%SZ); mv /var/lib/igneum/igneum-testnet-1 "/var/lib/igneum/igneum-testnet-1.prev-$stamp"; wiped="datadir moved aside as igneum-testnet-1.prev-$stamp; "; fi
|
|
mv -f igneumd igneumd.prev; mv -f igneumd.new igneumd; [ "$MINER" = 1 ] && { mv -f igneum-miner igneum-miner.prev 2>/dev/null || true; mv -f igneum-miner.new igneum-miner; }
|
|
systemctl start igneumd; sleep 6
|
|
gline=""; if [ "$WIPE" = 1 ]; then for i in $(seq 1 30); do gline=$(journalctl -u igneumd --since "-90 s" --no-pager 2>/dev/null | grep -oE "genesis [0-9a-f]{8,} executed" | tail -1); [ -n "$gline" ] && break; sleep 2; done; fi
|
|
gmatch=""; if [ "$WIPE" = 1 ]; then gh=$(echo "$gline" | awk '{print $2}'); if [ -z "$gh" ]; then gmatch="genesis line not seen in 60 s; "; elif [ -n "$GENESIS" ] && [ "${gh#${GENESIS:0:8}}" = "$gh" ]; then gmatch="genesis MISMATCH($gh); "; else gmatch="genesis $gh MATCH; "; fi
|
|
# the re-cut's binary prints "Base unit: 10^18" at start and the 5 October one never does (the testnet lane, 7 Oct 2026)
|
|
if journalctl -u igneumd --since "-90 s" --no-pager 2>/dev/null | grep -q "Base unit: 10^18"; then gmatch="${gmatch}base unit 10^18 line seen; "; else gmatch="${gmatch}NO base unit line; "; fi; fi
|
|
down=$(( $(date +%s) - t0 ))
|
|
commits=$(grep -a -c "$COMMIT" igneumd || true)
|
|
first=$(journalctl -u igneumd --since "-40 s" --no-pager 2>/dev/null | grep -vE "Started|Stopped|Stopping|Deactivated|Consumed" | head -1 | cut -c1-120)
|
|
dig=$(journalctl -u igneumd --since "-40 s" --no-pager 2>/dev/null | grep -oE "Consensus params digest: [0-9a-f]+" | tail -1 | awk '{print $4}')
|
|
syncing=$(curl -s -m 5 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_syncing","params":[]}' http://127.0.0.1:26890 | cut -c1-80)
|
|
peers=$(curl -s -m 5 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"net_peerCount","params":[]}' http://127.0.0.1:26890 | grep -oE '"result":"[^"]*"' | cut -d'"' -f4)
|
|
dmatch=no; [ -n "$dig" ] && [ "$dig" = "$DIGEST" ] && dmatch=MATCH; [ -n "$dig" ] && [ "$dig" != "$DIGEST" ] && dmatch="MISMATCH($dig)"
|
|
echo "unit $(systemctl is-active igneumd) down ${down}s; ${wiped}${gmatch}commit strings $commits; digest $dmatch; eth_syncing $syncing; peers $peers; first: $first"
|
|
REMOTE
|
|
fi
|
|
} >> "$log" 2>&1; rc=$?
|
|
line=$(tail -1 "$log" | cut -c1-300)
|
|
echo "RESULT $name ($ip) rc=$rc after $(( $(date +%s) - t0 )) s: $line"
|
|
}
|
|
|
|
case "$mode" in
|
|
seeds)
|
|
[ -n "$BIN" ] && [ -n "$SHA" ] && [ -n "$DIGEST" ] || die "seeds needs --igneumd --sha256 --digest (and --override when the seeds take one)"
|
|
[ -f "$BIN" ] || die "binary file missing"; [ -z "$OVERRIDE" ] || [ -f "$OVERRIDE" ] || die "override file missing"
|
|
[ "$WIPE" = 0 ] || [ -n "$GENESIS" ] || die "--wipe-genesis needs --genesis <hex> for the read-back"
|
|
local_sha=$(shasum -a 256 "$BIN" | cut -d' ' -f1); [ "$local_sha" = "$SHA" ] || die "the binary's sha256 is $local_sha, not $SHA"
|
|
[ -z "$OVERRIDE" ] || python3 -c 'import json,sys; json.load(open(sys.argv[1]))' "$OVERRIDE" || die "override file does not parse"
|
|
say "seeds: $( [ "$GO" = 1 ] && echo GO || echo DRY RUN ); igneumd $SHA; override ${OVERRIDE:-none}; digest $DIGEST; commit $COMMIT; wipe-genesis $WIPE${GENESIS:+ (genesis $GENESIS)}"
|
|
for n in seed1.testnet seed2.testnet seed3.testnet; do one_seed "$n" & done; wait
|
|
say "seeds done; logs in $S" ;;
|
|
hands)
|
|
[ -n "$NODE" ] && [ -n "$OVJSON" ] && [ -n "$DIGEST" ] || die "hands needs --node --override-json --digest"
|
|
g=""; [ "$GO" = 1 ] && g="--go"
|
|
say "hands: $( [ "$GO" = 1 ] && echo GO || echo DRY RUN ) through move-hand.sh"
|
|
"$HERE/hands/move-hand.sh" binary --node "$NODE" --override-json "$OVJSON" $g
|
|
"$HERE/hands/move-hand.sh" restart observer-node --digest "$DIGEST" $g
|
|
"$HERE/hands/move-hand.sh" restart node1 --digest "$DIGEST" $g ;;
|
|
lift-rpc-filter)
|
|
ip=$(seed_ip seed1.testnet)
|
|
if [ "$GO" = 0 ]; then
|
|
say "DRY RUN: on root@$ip: back up /opt/igneum/bin/rpc-filter.py, set BLOCKED_UNTIL_FIXED_NODE = set(), py_compile, systemctl restart igneum-rpc-filter, read back"
|
|
"${SSH[@]}" "root@$ip" 'echo "current: $(grep -c "^BLOCKED_UNTIL_FIXED_NODE = {" /opt/igneum/bin/rpc-filter.py) block set(s), unit $(systemctl is-active igneum-rpc-filter.service)"'; exit 0; fi
|
|
"${SSH[@]}" "root@$ip" bash -s <<'REMOTE'
|
|
set -euo pipefail
|
|
f=/opt/igneum/bin/rpc-filter.py; cp -a "$f" "$f.bak-$(date -u +%Y%m%dT%H%M%SZ)"
|
|
python3 - <<'PY'
|
|
import re
|
|
p='/opt/igneum/bin/rpc-filter.py'; s=open(p).read()
|
|
new, n = re.subn(r'BLOCKED_UNTIL_FIXED_NODE = \{.*?\n\}', 'BLOCKED_UNTIL_FIXED_NODE = set() # lifted 7 Oct 2026: every seed runs the fixed node (N7 class fixed from 8097d600)', s, count=1, flags=re.S)
|
|
assert n == 1, "block set not found"
|
|
open(p,'w').write(new)
|
|
PY
|
|
python3 -m py_compile "$f"; systemctl restart igneum-rpc-filter.service; sleep 2
|
|
echo "unit $(systemctl is-active igneum-rpc-filter.service); blocked set now: $(grep -E '^BLOCKED_UNTIL_FIXED_NODE' "$f" | cut -c1-60)"
|
|
REMOTE
|
|
say "public read-back: $(curl -s -m 8 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' https://rpc.testnet.igneum.network | cut -c1-120)" ;;
|
|
*) die "unknown mode $mode" ;;
|
|
esac
|