3cc5e5c2f0
The shipper's read-back for 0.3.18 (7 October 2026): igneum_getNodeInfo exists again and must answer powEngine igneum-pow (a stub is a FAIL) with a blockrate object, which the mover prints; a tree before 0.3.18 answers -32601 and the engine is still read from the binary's igneum-pow source paths. The parser is checked against the three answer shapes. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
201 lines
18 KiB
Bash
Executable file
201 lines
18 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Move one devnet hand from this Mac to igneum-build-1, one at a time so one hand always serves (the project lead, 6 October 2026: the Mac
|
|
# runs nothing the network depends on). Plan and order: docs/plans/hands-on-build-1.md. Dry run by default; --go executes.
|
|
#
|
|
# infra/build-server/hands/move-hand.sh binary --node <fork worktree> [--override-json '<object>']
|
|
# build 0.3.15's Linux igneumd on the box, install it to
|
|
# /srv/hands/bin, write the override (the shipper's exact
|
|
# object, else the Mac's file) and the snapshot (step 1)
|
|
# infra/build-server/hands/move-hand.sh observer-node [--go] hot rsync, stop the Mac's observer node (launchd), final rsync,
|
|
# start igneum-observer-node, print its first executing line (step 2)
|
|
# infra/build-server/hands/move-hand.sh observer [--go] copy ~/.config/igneum/env to /srv/observer/env (600), stop the
|
|
# Mac's run.sh + autosync + observer.mjs, start igneum-observer (step 3)
|
|
# infra/build-server/hands/move-hand.sh node1 [--go] the same as observer-node for node 1 (step 4)
|
|
# infra/build-server/hands/move-hand.sh unload [--go] bootout the Mac's two launchd agents for good (step 5, last)
|
|
# infra/build-server/hands/move-hand.sh restart observer-node|node1 [--digest <hex>] [--go]
|
|
# a release on the box (7 Oct 2026, 0.3.17): after `binary`
|
|
# installed the new igneumd and the override, restart ONE
|
|
# unit and read it back: first executing line, commit string
|
|
# of the installed binary, digest (against --digest when
|
|
# given, else the unit's own last digest), igneum_getNodeInfo
|
|
# powEngine over the node's loopback EVM RPC
|
|
# infra/build-server/hands/move-hand.sh status both sides: units, pids, tips, peers
|
|
#
|
|
# Needs ~/.config/igneum/build-server (build@<ip>) and the ops key; root ssh to the box for systemctl, scp of the env file and chown.
|
|
# Nothing here prints a secret: the env file travels by scp and is only ever stat'ed.
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO="$(cd "$HERE/../../.." && pwd)"
|
|
# shellcheck disable=SC2034
|
|
BS_TOOL=move-hand
|
|
# shellcheck source=../lib.sh
|
|
. "$HERE/../lib.sh"
|
|
|
|
{ # whole-body: bash parses this block entirely before running a line of it, so an edit to this file while a run is in
|
|
# flight cannot reach the running copy (7 Oct 2026: build-remote.sh was edited mid-run and died on shifted bytes after a 4-min build)
|
|
bs_host
|
|
IP="${BS_HOST#*@}"
|
|
ROOT_SSH=(ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new "root@$IP")
|
|
MAC_OV=/tmp/igneum-devnet/override-v3.json
|
|
MAC_SNAP=/tmp/igneum-devnet/node1-copy-snapshot.bin
|
|
MAC_SNAP_SHA=ac101f13576179fd7d7f5e8ee902c9a7b6cc47730e3a3c069f389f0ca46d9221 # the launchd agents' value (6 Oct 2026); recomputed below
|
|
H=/srv/hands
|
|
MODE="${1:-}"; shift || true
|
|
GO=0; NODE_WT=""; OV_JSON=""; WANT_DIGEST=""; HAND=""
|
|
while [ $# -gt 0 ]; do case "$1" in --go) GO=1; shift ;; --node) NODE_WT="$2"; shift 2 ;; --override-json) OV_JSON="$2"; shift 2 ;; --digest) WANT_DIGEST="$2"; shift 2 ;; *) if [ "$MODE" = restart ] && [ -z "$HAND" ]; then HAND="$1"; shift; else bs_die "unknown argument $1"; fi ;; esac; done
|
|
say() { bs_log "$*"; }
|
|
run() { if [ "$GO" = 1 ]; then "$@"; else say "DRY RUN: $*"; fi; }
|
|
rssh() { "${ROOT_SSH[@]}" "$@"; }
|
|
# the digest readback (main, 6 Oct 2026 23:xx UK): the box hand's "Consensus params digest" against the Mac hand's last one
|
|
# tolerant pipelines: a missing line gives an empty string, never a failed command substitution that ends the script under set -e
|
|
# (7 Oct 2026: the restart dry run died silently because the unit's digest line was older than the 10-minute window)
|
|
mac_digest() { { grep 'Consensus params digest' "$HOME/Library/Logs/Igneum/$1.out" 2>/dev/null | tail -1 | grep -oE '[0-9a-f]{64}' | head -1; } || true; }
|
|
box_digest() { { rssh "journalctl -u $1 --no-pager -o cat --since '7 days ago' | grep 'Consensus params digest' | tail -1" 2>/dev/null | grep -oE '[0-9a-f]{64}' | head -1; } || true; }
|
|
digest_readback() { # <unit> <mac hand log name>
|
|
local b m; b=$(box_digest "$1"); m=$(mac_digest "$2")
|
|
if [ -n "$b" ] && [ "$b" = "$m" ]; then say "$1 digest ${b:0:16}... MATCHES the Mac's $2 hand"; else say "$1 digest ${b:-none} against the Mac's ${m:-none}: DIFFER (stop and read the override before moving the next hand)"; return 1; fi
|
|
}
|
|
first_exec_line() { # <unit>: wait up to 180 s for the node's exec line, print it (the proof main wants) and the chain tip
|
|
local u="$1" line=""
|
|
for _ in $(seq 1 36); do
|
|
line=$(rssh "journalctl -u $u --no-pager -o cat --since '5 min ago' | grep -m1 -E 'igneum-exec\] exec (sync: resumed|state loaded)'" 2>/dev/null || true)
|
|
[ -n "$line" ] && break; sleep 5
|
|
done
|
|
if [ -n "$line" ]; then say "$u first executing line: ${line:0:220}"; else say "$u: no exec line in 180 s; last lines:"; rssh "journalctl -u $u --no-pager -o cat -n 5" | sed 's/^/ /'; fi
|
|
rssh "journalctl -u $u --no-pager -o cat --since '5 min ago' | grep -E 'Accepted [0-9]+ blocks|PoW accepted|IBD|Consensus params digest' | tail -3" | cut -c1-200 | sed 's/^/ /' || true
|
|
}
|
|
sync_dir() { # <mac dir> <box dir>: rsync a data dir (hot or final), keeping RocksDB files whole
|
|
bs_rsync -a --delete --exclude '*.out' "$1/" "$BS_HOST:$2/"
|
|
}
|
|
mac_stop_agent() { # <label>: bootout the launchd agent (KeepAlive would restart a killed process), wait for the pid to end
|
|
local label="$1" pid
|
|
pid=$(launchctl print "gui/$(id -u)/$label" 2>/dev/null | grep -oE 'pid = [0-9]+' | head -1 | grep -oE '[0-9]+' || true) # macOS awk has no \s: the first run printed "pid none" though the bootout worked
|
|
launchctl bootout "gui/$(id -u)/$label" 2>/dev/null || true
|
|
if [ -n "$pid" ]; then while kill -0 "$pid" 2>/dev/null; do sleep 1; done; fi
|
|
say "launchd $label unloaded (igneumd pid ${pid:-none} ended)"
|
|
}
|
|
|
|
case "$MODE" in
|
|
binary)
|
|
if [ -z "$NODE_WT" ]; then
|
|
# default: the fork tree that built the hand running on the Mac now (the launchd agent's binary path), so the box runs
|
|
# the same commit the Mac did (6 Oct 2026: igneum-wt-ship0315/vendor/igneum-node-0315 at f1ea7a38)
|
|
macbin=$(plutil -p "$HOME/Library/LaunchAgents/network.igneum.devnet.node1.plist" 2>/dev/null | grep -oE '/[^"]*igneumd' | head -1)
|
|
NODE_WT=$(dirname "$macbin" | sed -E 's|/target[^/]*/release$||'); [ -f "$NODE_WT/Cargo.toml" ] || bs_die "no --node and no fork tree behind the Mac's node1 agent ($macbin)"
|
|
say "node tree from the Mac's node1 agent: $NODE_WT ($(git -C "$NODE_WT" rev-parse --short HEAD) on $(git -C "$NODE_WT" branch --show-current))"
|
|
fi
|
|
[ -f "$NODE_WT/Cargo.toml" ] || bs_die "no Cargo.toml in $NODE_WT"
|
|
ver=$(grep -m1 '^version' "$NODE_WT/Cargo.toml" | sed 's/.*"\(.*\)".*/\1/'); sha=$(git -C "$NODE_WT" rev-parse --short HEAD)
|
|
say "building igneumd $ver ($sha) on the box from $NODE_WT"
|
|
out=$(mktemp -d)
|
|
(cd "$NODE_WT" && IGNEUM_AGENT="${IGNEUM_AGENT:-hands}" "$REPO/tools/build-remote.sh" --out "$out" --artefacts target/release/igneumd -- build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow) || bs_die "the box build failed"
|
|
bin_sha=$(bs_sha256 "$out/release/igneumd"); rm -rf "$out"
|
|
# the binary is already on the box; copy it there, never back and forth
|
|
ctx=$(cd "$NODE_WT" && BS_TOOL=move-hand bash -c '. "$0"; bs_host >/dev/null; bs_context; echo "$BS_REMOTE_CRATE"' "$HERE/../lib.sh")
|
|
run rssh "install -m 755 -o build -g build '$ctx/target/release/igneumd' '$H/bin/igneumd-$ver-$sha' && ln -sfn '$H/bin/igneumd-$ver-$sha' '$H/bin/igneumd' && sha256sum '$H/bin/igneumd-$ver-$sha' | cut -c1-16 && { '$H/bin/igneumd' --version || true; }" # igneumd --version prints and exits 1 (7 Oct 2026: it ended this step under set -e before the override was written)
|
|
say "box binary sha256 $bin_sha; checking its commit string on the box"
|
|
run rssh "[ \$(strings '$H/bin/igneumd' | grep -c '$sha') -gt 0 ] && echo 'commit $sha in the binary' || { echo 'NO commit string in the binary'; exit 1; }"
|
|
if [ -n "$OV_JSON" ]; then
|
|
# the shipper's exact object for the cut (6 Oct 2026: the sixteen-field file at publish 2); checked as a JSON object with the
|
|
# fee switch field, as infra/devnet/restart-hand-nodes.sh checks its argument
|
|
printf '%s' "$OV_JSON" | python3 -c 'import json,sys; o=json.load(sys.stdin); assert isinstance(o, dict) and "fees_v1_activation_daa" in o, "the object must carry fees_v1_activation_daa"; print("override (%d fields):" % len(o), json.dumps(o, sort_keys=True)[:400])'
|
|
ovfile=$(mktemp); printf '%s\n' "$OV_JSON" > "$ovfile"
|
|
else
|
|
[ -f "$MAC_OV" ] || bs_die "no override file at $MAC_OV and no --override-json"
|
|
nf=$(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))))' "$MAC_OV")
|
|
say "override from the Mac's file ($nf fields): $(cut -c1-200 "$MAC_OV")"; ovfile="$MAC_OV"
|
|
[ "$nf" -ge 16 ] || say "WARNING: the Mac's override has $nf fields; publish 2 writes the sixteen-field object when it restarts the hands. Move only after the shipper's 'publish 2 live: digest <x>' line, or pass --override-json"
|
|
fi
|
|
run bs_rsync -p "$ovfile" "$BS_HOST:$H/override.json"
|
|
[ "$ovfile" = "$MAC_OV" ] || rm -f "$ovfile"
|
|
if [ -f "$MAC_SNAP" ]; then
|
|
snap_sha=$(bs_sha256 "$MAC_SNAP"); [ "$snap_sha" = "$MAC_SNAP_SHA" ] || say "WARNING: snapshot sha256 is $snap_sha, the agents say $MAC_SNAP_SHA; the box gets the file's own"
|
|
say "snapshot: $MAC_SNAP ($(bs_size "$MAC_SNAP") bytes) -> $H/node1-copy-snapshot.bin,$snap_sha"
|
|
run bs_rsync -p "$MAC_SNAP" "$BS_HOST:$H/node1-copy-snapshot.bin"
|
|
run bs_ssh "sed -i 's|^SNAPSHOT=.*|SNAPSHOT=$H/node1-copy-snapshot.bin,$snap_sha|; s|^IGNEUMD=.*|IGNEUMD=$H/bin/igneumd|' $H/hands.env && grep -E '^(IGNEUMD|SNAPSHOT|OVERRIDE)=' $H/hands.env"
|
|
fi
|
|
say "binary step done; next: move-hand.sh observer-node --go" ;;
|
|
|
|
observer-node|node1)
|
|
if [ "$MODE" = node1 ]; then mac_dir=/tmp/igneum-devnet/node1; label=network.igneum.devnet.node1; unit=igneum-node1; else mac_dir=/tmp/igneum-devnet/observer-v4; label=network.igneum.devnet.observer; unit=igneum-observer-node; fi
|
|
[ -d "$mac_dir" ] || bs_die "no data dir $mac_dir on the Mac"
|
|
bs_ssh "[ -x $H/bin/igneumd ] && [ -f $H/override.json ]" || bs_die "the box has no binary or override yet: run move-hand.sh binary --node <wt> first"
|
|
say "$MODE: hot rsync of $mac_dir ($(du -sh "$mac_dir" | cut -f1)) while the Mac node runs"
|
|
run sync_dir "$mac_dir" "$H/$MODE"
|
|
say "$MODE: stopping the Mac's $label, then the final rsync (the delta, seconds), then the unit on the box"
|
|
run mac_stop_agent "$label"
|
|
run sync_dir "$mac_dir" "$H/$MODE"
|
|
run rssh "systemctl start $unit && sleep 3 && systemctl is-active $unit"
|
|
if [ "$GO" = 1 ]; then
|
|
first_exec_line "$unit"
|
|
rssh "[ \$(strings $H/bin/igneumd | grep -c \"\$(readlink $H/bin/igneumd | sed -E 's/.*-([0-9a-f]{7,})\$/\\1/')\") -gt 0 ] && echo 'commit string present in the box binary' || echo 'WARNING: no commit string in the box binary'"
|
|
digest_readback "$unit" "$( [ "$MODE" = node1 ] && echo node1 || echo observer )" || true
|
|
fi
|
|
say "$MODE moved; the Mac's agent stays unloaded (step 5 removes the plist from the login)" ;;
|
|
|
|
observer)
|
|
[ -f "$HOME/.config/igneum/env" ] || bs_die "no ~/.config/igneum/env on the Mac"
|
|
grep -q '^DATABASE_URL=' "$HOME/.config/igneum/env" || bs_die "$HOME/.config/igneum/env has no DATABASE_URL line"
|
|
tmp=$(mktemp); chmod 600 "$tmp"
|
|
{ grep -E '^(DATABASE_URL|LIVE_RETAIN_HOURS|LIVE_TABLE_PREFIX)=' "$HOME/.config/igneum/env"; printf 'IGNEUM_RPC=ws://127.0.0.1:28640\nIGNEUM_EVM_RPC=http://127.0.0.1:26840\n'; } > "$tmp"
|
|
say "observer env: $(grep -c . "$tmp") lines (names: $(cut -d= -f1 "$tmp" | tr '\n' ' ')) -> root@$IP:/srv/observer/env mode 600 owner build"
|
|
run scp -q -i "$BS_KEY" -o BatchMode=yes "$tmp" "root@$IP:/srv/observer/env.new"; rm -f "$tmp"
|
|
run rssh "chown build:build /srv/observer/env.new && chmod 600 /srv/observer/env.new && mv /srv/observer/env.new /srv/observer/env && stat -c '%U %a %s bytes' /srv/observer/env"
|
|
run rssh "systemctl is-active igneum-observer-node" || bs_die "igneum-observer-node is not active on the box; move it first"
|
|
say "stopping the Mac's observer: autosync.sh, run.sh, observer.mjs (two writers to Neon would duplicate rows, so the Mac stops first)"
|
|
for pat in 'tools/observer/autosync.sh' 'tools/observer/run.sh' 'tools/observer/observer.mjs'; do
|
|
for p in $(pgrep -f "$pat" || true); do run kill -TERM "$p"; done
|
|
done
|
|
run rssh "systemctl start igneum-observer && sleep 5 && systemctl is-active igneum-observer && journalctl -u igneum-observer --no-pager -o cat -n 6"
|
|
say "observer moved: /api/live reads Neon, which the box's observer now writes" ;;
|
|
|
|
restart)
|
|
hand="$HAND"
|
|
case "$hand" in node1) unit=igneum-node1; evm=26791 ;; observer-node) unit=igneum-observer-node; evm=26840 ;; *) bs_die "restart needs observer-node or node1" ;; esac
|
|
bin=$(rssh "readlink $H/bin/igneumd"); sha=$(printf '%s' "$bin" | sed -E 's/.*-([0-9a-f]{7,})$/\1/')
|
|
before=$(box_digest "$unit"); want="${WANT_DIGEST:-$before}"
|
|
say "$hand: restart $unit on $bin (commit $sha); digest before ${before:-none}, wanted ${want:-any}"
|
|
run rssh "systemctl restart $unit && sleep 3 && systemctl is-active $unit"
|
|
if [ "$GO" = 1 ]; then
|
|
first_exec_line "$unit"
|
|
rssh "[ \$(strings $H/bin/igneumd | grep -c '$sha') -gt 0 ] && echo 'commit string $sha present in the running binary' || { echo 'NO commit string $sha in the binary'; exit 1; }"
|
|
after=$(box_digest "$unit")
|
|
if [ -n "$after" ] && [ "$after" = "$want" ]; then say "$unit digest ${after:0:16}... MATCHES"; else say "$unit digest ${after:-none} against wanted ${want:-any}: DIFFER (stop here)"; exit 1; fi
|
|
# 0.3.18 and later (the shipper, 7 Oct 2026): igneum_getNodeInfo answers with powEngine (igneum-pow; a stub is a FAIL that stops
|
|
# the sequence) and a blockrate object; a tree before it answers -32601 and the engine is read from the binary below
|
|
eng=$(rssh "curl -s --max-time 10 -X POST -H 'content-type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getNodeInfo\",\"params\":[]}' http://127.0.0.1:$evm" | python3 -c '
|
|
import json, sys
|
|
d = json.load(sys.stdin); r = d.get("result") or {}
|
|
if d.get("error"): print("ERROR: " + str(d["error"])); sys.exit()
|
|
eng = r.get("powEngine") or r.get("pow_engine") or "none"
|
|
br = r.get("blockrate") or r.get("blockRate")
|
|
print(eng + "|" + (json.dumps(br, separators=(",", ":"))[:160] if isinstance(br, dict) else "NO blockrate object"))' 2>/dev/null || echo "igneum_getNodeInfo not answered on $evm")
|
|
case "$eng" in
|
|
igneum-pow\|*)
|
|
say "$unit igneum_getNodeInfo powEngine: igneum-pow; blockrate: ${eng#igneum-pow|}"
|
|
case "$eng" in *"NO blockrate"*) say "WARNING: $unit igneum_getNodeInfo carries no blockrate object" ;; esac ;;
|
|
stub\|*|none\|*) say "$unit igneum_getNodeInfo powEngine: ${eng%%|*}: FAIL (stop here)"; exit 1 ;;
|
|
*-32601*|*"not found"*|*"not answered"*)
|
|
# a tree before 0.3.18 has no igneum_getNodeInfo (the shipper, 7 Oct 2026): the engine is read from the binary instead; a stub
|
|
# build carries none of the igneum-pow crate's source paths
|
|
n=$(rssh "strings $H/bin/igneumd | grep -c 'igneum-pow/src/'" || echo 0)
|
|
if [ "${n:-0}" -gt 0 ]; then say "$unit engine from the binary: igneum-pow ($n igneum-pow/src/ paths; igneum_getNodeInfo is not on this tree)"; else say "WARNING: $unit binary carries no igneum-pow/src/ path: a stub build?"; fi ;;
|
|
*) say "WARNING: $unit igneum_getNodeInfo powEngine: $eng" ;;
|
|
esac
|
|
fi ;;
|
|
|
|
unload)
|
|
say "removing the Mac's launchd agents for good (bootout and the plists moved aside); the hands are on the box"
|
|
for label in network.igneum.devnet.observer network.igneum.devnet.node1; do
|
|
run launchctl bootout "gui/$(id -u)/$label" 2>/dev/null || true
|
|
run mv -f "$HOME/Library/LaunchAgents/$label.plist" "$HOME/Library/LaunchAgents/$label.plist.moved-to-build-1-$(date -u +%Y%m%d)"
|
|
done
|
|
say "Mac igneumd processes now: $(pgrep -fl '[i]gneumd --' | grep -v Wallet | wc -l | tr -d ' ') (the wallet's own node is not a hand)" ;;
|
|
|
|
status)
|
|
echo "--- box:"; rssh "for u in igneum-node1 igneum-observer-node igneum-observer igneum-observer-sync.timer; do printf '%-26s %s\n' \$u \$(systemctl is-active \$u); done; journalctl -u igneum-node1 -o cat -n 2 --no-pager 2>/dev/null | cut -c1-160; journalctl -u igneum-observer -o cat -n 2 --no-pager 2>/dev/null | cut -c1-160"
|
|
echo "--- mac:"; launchctl print "gui/$(id -u)/network.igneum.devnet.node1" 2>/dev/null | grep -E 'state|pid' | head -2; launchctl print "gui/$(id -u)/network.igneum.devnet.observer" 2>/dev/null | grep -E 'state|pid' | head -2; pgrep -fl '[o]bserver.mjs|[o]bserver/run.sh|[a]utosync.sh' || echo "no observer processes on the Mac" ;;
|
|
*) sed -n '2,20p' "$0"; exit 2 ;;
|
|
esac
|
|
exit 0
|
|
}
|