From fa9fc92b78e445c53dcd481839df2e4f5000336f Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:32:35 +0000 Subject: [PATCH] fleet: box-node-swap.sh rewritten clean (the edited copy called stamp before its definition and misquoted the sha line; no node was touched) --- tools/fleet/box-node-swap.sh | 54 ++++++++++++++++++++++-------------- tools/fleet/swap.py | 3 +- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/tools/fleet/box-node-swap.sh b/tools/fleet/box-node-swap.sh index e0c81db60..7c26bf07d 100755 --- a/tools/fleet/box-node-swap.sh +++ b/tools/fleet/box-node-swap.sh @@ -1,45 +1,57 @@ #!/usr/bin/env bash -# The 0.3.13 node swap for a phase-2 box (6 October 2026, the coordinator's procedure): the fixed Linux igneumd (pushed -# to /root/fleet/in/igneumd-0313, sha256 in NODE_SHA256) replaces the running one over the observer's full-history -# datadir (pulled from the hub unless already in place), with the hub and the seed as peers and the proof verifier; -# then the exec follower's replay is timed as box-datadir.sh does: eth_blockNumber every 15 s until it reaches the tip. +# The 0.3.13 node swap for a phase-2 box (6 October 2026, the shipper's two-word procedure). +# STEP=binary (publish 1): the fixed igneumd (/root/fleet/in/igneumd-0313, sha256 NODE_SHA256) replaces the running +# node over the observer's full-history datadir (pulled from the hub if not in place), the TEN-field file kept; +# the digest must read EXPECT_DIGEST (7bd98cc4...); the exec layer stays blocked by design; one status read, done. +# STEP=file (publish 2): ov13.json becomes the override, the node restarts, the digest must read b18ed271...; +# then eth_blockNumber is polled every 15 s until it reaches the tip (the replay time). set -uo pipefail F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; FLOOR=/opt/igneum-floor; HOST=$FLOOR/bin/igneum-prove-host HUB_SSH="${HUB_SSH:-213.173.107.74}"; HUB_PORT="${HUB_PORT:-16515}"; HUB_PEER="${HUB_PEER:-213.173.107.74:16516}" -NODE_SHA256="${NODE_SHA256:?the fixed igneumd's sha256}" -STEP="${STEP:-binary}" # binary = publish 1 (the new igneumd, the ten-field file kept, digest 7bd98cc4); file = publish 2 (ov13.json becomes the override, digest b18ed271) +NODE_SHA256="${NODE_SHA256:?the fixed igneumd sha256}" +STEP="${STEP:-binary}" EXPECT_DIGEST="${EXPECT_DIGEST:-7bd98cc4118616455709d5e32a30b799e6e67caa42d2b5d09875cd49848a7ed7}" mkdir -p $OUT; exec >> $OUT/node-swap.log 2>&1 stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } -echo "RESULT swap_start $(stamp)" +rpc() { curl -s -m 8 -X POST -H 'Content-Type: application/json' --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"$1\",\"params\":[]}" http://127.0.0.1:26790/; } +echo "RESULT swap_start $(stamp) step=$STEP" echo "$NODE_SHA256 $F/in/igneumd-0313" | sha256sum -c - >/dev/null || { echo "RESULT swap_failed sha256 mismatch: $(sha256sum $F/in/igneumd-0313 | cut -c1-16)"; exit 2; } chmod +x $F/in/igneumd-0313 t0=$(date +%s) -if [ ! -d $F/node/igneum-devnet ] || [ ! -f $F/node/.full-history ]; then +bash $F/in/box-kill.sh >/dev/null 2>&1 # every stage process; the node is stopped on the next line +pkill -x igneumd; pkill -f igneumd-0313; sleep 4; pkill -9 -x igneumd 2>/dev/null; pkill -9 -f igneumd-0313 2>/dev/null; sleep 1 +if [ ! -f $F/node/.full-history ]; then [ -s $F/observer-datadir.tgz ] || scp -i $F/in/fleet-internal -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P "$HUB_PORT" "root@$HUB_SSH:/root/fleet/share/observer-datadir.tgz" $F/observer-datadir.tgz || { echo "RESULT swap_failed pull"; exit 2; } - bash $F/in/box-kill.sh >/dev/null 2>&1; pkill -x igneumd; sleep 4; pkill -9 -x igneumd 2>/dev/null rm -rf $F/node.proof && mv $F/node $F/node.proof && mkdir -p $F/node && tar -C $F/node -xzf $F/observer-datadir.tgz 2>/dev/null && touch $F/node/.full-history || { echo "RESULT swap_failed untar"; exit 2; } - echo "RESULT datadir_in_place $(stamp) s=$(( $(date +%s) - t0 ))" -else - bash $F/in/box-kill.sh >/dev/null 2>&1; pkill -x igneumd; sleep 4; pkill -9 -x igneumd 2>/dev/null + echo "RESULT datadir_in_place $(stamp) s=$(( $(date +%s) - t0 )) $(du -sh $F/node | cut -f1)" +fi +if [ "$STEP" = file ]; then + cp $F/override.json $F/override-10.json; cp $F/in/ov13.json $F/override.json + echo "RESULT override_swapped $(stamp) fields=$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))))' $F/override.json)" fi -if [ "$STEP" = file ]; then cp $F/override.json $F/override-10.json; cp $F/in/ov13.json $F/override.json; echo "RESULT override_swapped $(stamp) fields=$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))))' $F/override.json)"; fi cp $F/in/igneumd-0313 $B/igneumd-0313 IGNEUM_PROOF_VERIFIER=$HOST nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 \ --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes ${NODE_EXTRA:-} >> $F/node.log 2>&1 & -t1=$(date +%s); sleep 10 +t1=$(date +%s); sleep 12 d="$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print $2}')" -echo "RESULT node_started $(stamp) step=$STEP pid=$(pgrep -f igneumd-0313 | head -1) version=$($B/igneumd-0313 --version 2>&1 | head -1) digest=${d:0:16} expected=${EXPECT_DIGEST:0:16} $( [ "$d" = "$EXPECT_DIGEST" ] && echo digest_ok || echo DIGEST_MISMATCH ) exec_restart=\"$(grep -o 'Exec restart from the override file.*' $F/node.log | tail -1 | cut -c1-120)\"" -[ "$d" = "$EXPECT_DIGEST" ] || { echo "RESULT swap_failed digest mismatch"; exit 2; } -bn() { curl -s -m 8 -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' http://127.0.0.1:26790/ | grep -o '"result":"[^"]*"' | cut -d'"' -f4; } +ok=DIGEST_MISMATCH; [ "$d" = "$EXPECT_DIGEST" ] && ok=digest_ok +echo "RESULT node_started $(stamp) step=$STEP pid=$(pgrep -f igneumd-0313 | head -1) version=$($B/igneumd-0313 --version 2>&1 | head -1) digest=${d:0:16} expected=${EXPECT_DIGEST:0:16} $ok exec_restart=\"$(grep -o 'Exec restart from the override file.*' $F/node.log | tail -1 | cut -c1-120)\"" +[ "$ok" = digest_ok ] || { echo "RESULT swap_failed digest mismatch"; exit 2; } +watch() { $B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1; } +if [ "$STEP" = binary ]; then + sleep 45 + echo "RESULT exec_status $(rpc igneum_getExecStatus | cut -c1-240)" + echo "RESULT swap_done $(stamp) step=binary $(watch)"; exit 0 +fi for i in $(seq 1 240); do - sleep 15; h="$(bn)"; n=$(( ${h:-0x0} )); w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)" + sleep 15 + h="$(rpc eth_blockNumber | grep -o '"result":"[^"]*"' | cut -d'"' -f4)"; n=$(( ${h:-0x0} )); w="$(watch)" echo "RESULT replay $(stamp) s=$(( $(date +%s) - t1 )) evm_block=$n $w" if [ "$n" -gt 0 ] && [[ "$w" == *synced=true* ]]; then - st="$(curl -s -m 8 -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"igneum_getProvingStatus","params":[]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; d=json.load(sys.stdin).get("result",{}); print("tipDaa", int(d.get("tipDaa","0x0"),16), "active", d.get("v1",{}).get("active"), "fresh", d.get("v1",{}).get("freshRuleActive"))' 2>/dev/null)" - ex="$(curl -s -m 8 -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"igneum_getExecStatus","params":[]}' http://127.0.0.1:26790/ | cut -c1-200)"; echo "RESULT exec_status $ex" + echo "RESULT exec_status $(rpc igneum_getExecStatus | cut -c1-240)" + st="$(rpc igneum_getProvingStatus | python3 -c 'import sys,json; d=json.load(sys.stdin).get("result",{}); print("tipDaa", int(d.get("tipDaa","0x0"),16), "active", d.get("v1",{}).get("active"), "fresh", d.get("v1",{}).get("freshRuleActive"))' 2>/dev/null)" daa=$(printf '%s' "$w" | grep -o 'daa=[0-9]*' | cut -d= -f2); tip=$(printf '%s' "$st" | awk '{print $2}') - [ -n "$tip" ] && [ "$tip" -ge $(( ${daa:-0} - 20 )) ] && { echo "RESULT swap_done $(stamp) replay_s=$(( $(date +%s) - t1 )) $st"; exit 0; } + if [ -n "$tip" ] && [ "$tip" -ge $(( ${daa:-0} - 20 )) ]; then echo "RESULT swap_done $(stamp) step=file replay_s=$(( $(date +%s) - t1 )) $st"; exit 0; fi fi done echo "RESULT swap_failed replay did not reach the tip in 60 min" diff --git a/tools/fleet/swap.py b/tools/fleet/swap.py index 0a28443e7..af6aeb987 100644 --- a/tools/fleet/swap.py +++ b/tools/fleet/swap.py @@ -18,7 +18,8 @@ def start(step): env = f"STEP={step} NODE_SHA256={SHA} EXPECT_DIGEST={DIG[step]} HUB_PEER={hub['hub_peer']} HUB_SSH={hub['ssh_host']} HUB_PORT={hub['ssh_port']}" if hub else f"STEP={step} NODE_SHA256={SHA} EXPECT_DIGEST={DIG[step]}" def one(x): iid, b = x - r, out, err = fleet.ssh(b, f"cd /root/fleet && mv out/node-swap.log out/node-swap-{step}-prev.log 2>/dev/null; {env} setsid nohup in/box-node-swap.sh /dev/null 2>&1 & sleep 1; echo started", timeout=60) + fleet.scp(b, [os.path.join(fleet.HERE, "box-node-swap.sh"), os.path.join(fleet.HERE, "box-kill.sh")], "/root/fleet/in/") + r, out, err = fleet.ssh(b, f"cd /root/fleet && chmod +x in/box-node-swap.sh && mv out/node-swap.log out/node-swap-{step}-prev.log 2>/dev/null; {env} setsid nohup in/box-node-swap.sh /dev/null 2>&1 & sleep 1; echo started", timeout=60) fleet.patch(iid, stage="swap-" + step, doing=f"0.3.13 swap, step {step}: restart, digest check, the exec replay timed") return b["label"], out.strip() or err[:60] ts = targets()