11c4426a96
The fleet's 22:09 UK incident (a Mac-side pkill -f <log file name> matched nothing, the roll-everything script lived on and wiped a held box) and the day's two pgrep self-matches are one class. The check flags pgrep -f / pkill -f with a plain literal (every one on a line), any pgrep/pkill on a file-name shape, and ps | grep with a literal; it allows the bracket form, -x, -F pidfile, kill $(cat pidfile), a variable and a full path; 11 banned and 16 allowed shapes in its self-test; 0.15 s over the tree. The 25 pkill -f sp1-gpu-server inside bash -c bodies (which matched the calling bash) are pkill -x; the other 11 literals take the bracket form; prover-socket-check accepts both. Row R in the record; the CLAUDE.md rule names the check and covers pkill and file names. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
133 lines
11 KiB
Bash
Executable file
133 lines
11 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Move one devnet hand from this Mac to igneum-build-1, one at a time so one hand always serves (the project lead, 6 October 2026: the Mac
|
|
# runs nothing the network depends on). Plan and order: docs/plans/hands-on-build-1.md. Dry run by default; --go executes.
|
|
#
|
|
# infra/build-server/hands/move-hand.sh binary --node <fork worktree> [--override-json '<object>']
|
|
# build 0.3.15's Linux igneumd on the box, install it to
|
|
# /srv/hands/bin, write the override (the shipper's exact
|
|
# object, else the Mac's file) and the snapshot (step 1)
|
|
# infra/build-server/hands/move-hand.sh observer-node [--go] hot rsync, stop the Mac's observer node (launchd), final rsync,
|
|
# start igneum-observer-node, print its first executing line (step 2)
|
|
# infra/build-server/hands/move-hand.sh observer [--go] copy ~/.config/igneum/env to /srv/observer/env (600), stop the
|
|
# Mac's run.sh + autosync + observer.mjs, start igneum-observer (step 3)
|
|
# infra/build-server/hands/move-hand.sh node1 [--go] the same as observer-node for node 1 (step 4)
|
|
# infra/build-server/hands/move-hand.sh unload [--go] bootout the Mac's two launchd agents for good (step 5, last)
|
|
# infra/build-server/hands/move-hand.sh status both sides: units, pids, tips, peers
|
|
#
|
|
# Needs ~/.config/igneum/build-server (build@<ip>) and the ops key; root ssh to the box for systemctl, scp of the env file and chown.
|
|
# Nothing here prints a secret: the env file travels by scp and is only ever stat'ed.
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO="$(cd "$HERE/../../.." && pwd)"
|
|
# shellcheck disable=SC2034
|
|
BS_TOOL=move-hand
|
|
# shellcheck source=../lib.sh
|
|
. "$HERE/../lib.sh"
|
|
bs_host
|
|
IP="${BS_HOST#*@}"
|
|
ROOT_SSH=(ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new "root@$IP")
|
|
MAC_OV=/tmp/igneum-devnet/override-v3.json
|
|
MAC_SNAP=/tmp/igneum-devnet/node1-copy-snapshot.bin
|
|
MAC_SNAP_SHA=ac101f13576179fd7d7f5e8ee902c9a7b6cc47730e3a3c069f389f0ca46d9221 # the launchd agents' value (6 Oct 2026); recomputed below
|
|
H=/srv/hands
|
|
MODE="${1:-}"; shift || true
|
|
GO=0; NODE_WT=""; OV_JSON=""
|
|
while [ $# -gt 0 ]; do case "$1" in --go) GO=1; shift ;; --node) NODE_WT="$2"; shift 2 ;; --override-json) OV_JSON="$2"; shift 2 ;; *) bs_die "unknown argument $1" ;; esac; done
|
|
say() { bs_log "$*"; }
|
|
run() { if [ "$GO" = 1 ]; then "$@"; else say "DRY RUN: $*"; fi; }
|
|
rssh() { "${ROOT_SSH[@]}" "$@"; }
|
|
first_exec_line() { # <unit>: wait up to 180 s for the node's exec line, print it (the proof main wants) and the chain tip
|
|
local u="$1" line=""
|
|
for _ in $(seq 1 36); do
|
|
line=$(rssh "journalctl -u $u --no-pager -o cat --since '5 min ago' | grep -m1 -E 'igneum-exec\] exec (sync: resumed|state loaded)'" 2>/dev/null || true)
|
|
[ -n "$line" ] && break; sleep 5
|
|
done
|
|
if [ -n "$line" ]; then say "$u first executing line: ${line:0:220}"; else say "$u: no exec line in 180 s; last lines:"; rssh "journalctl -u $u --no-pager -o cat -n 5" | sed 's/^/ /'; fi
|
|
rssh "journalctl -u $u --no-pager -o cat --since '5 min ago' | grep -E 'Accepted [0-9]+ blocks|PoW accepted|IBD|Consensus params digest' | tail -3" | cut -c1-200 | sed 's/^/ /' || true
|
|
}
|
|
sync_dir() { # <mac dir> <box dir>: rsync a data dir (hot or final), keeping RocksDB files whole
|
|
bs_rsync -a --delete --exclude '*.out' "$1/" "$BS_HOST:$2/"
|
|
}
|
|
mac_stop_agent() { # <label>: bootout the launchd agent (KeepAlive would restart a killed process), wait for the pid to end
|
|
local label="$1" pid
|
|
pid=$(launchctl print "gui/$(id -u)/$label" 2>/dev/null | awk '/^\s*pid = /{ print $3 }' | head -1 || true)
|
|
launchctl bootout "gui/$(id -u)/$label" 2>/dev/null || true
|
|
if [ -n "$pid" ]; then while kill -0 "$pid" 2>/dev/null; do sleep 1; done; fi
|
|
say "launchd $label unloaded (igneumd pid ${pid:-none} ended)"
|
|
}
|
|
|
|
case "$MODE" in
|
|
binary)
|
|
[ -n "$NODE_WT" ] || bs_die "binary needs --node <fork worktree> (the release-0.3.15-node tree)"
|
|
[ -f "$NODE_WT/Cargo.toml" ] || bs_die "no Cargo.toml in $NODE_WT"
|
|
ver=$(grep -m1 '^version' "$NODE_WT/Cargo.toml" | sed 's/.*"\(.*\)".*/\1/'); sha=$(git -C "$NODE_WT" rev-parse --short HEAD)
|
|
say "building igneumd $ver ($sha) on the box from $NODE_WT"
|
|
out=$(mktemp -d)
|
|
(cd "$NODE_WT" && IGNEUM_AGENT="${IGNEUM_AGENT:-hands}" "$REPO/tools/build-remote.sh" --out "$out" --artefacts target/release/igneumd -- build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow) || bs_die "the box build failed"
|
|
bin_sha=$(bs_sha256 "$out/release/igneumd"); rm -rf "$out"
|
|
# the binary is already on the box; copy it there, never back and forth
|
|
ctx=$(cd "$NODE_WT" && BS_TOOL=move-hand bash -c '. "$0"; bs_host >/dev/null; bs_context; echo "$BS_REMOTE_CRATE"' "$HERE/../lib.sh")
|
|
run rssh "install -m 755 -o build -g build '$ctx/target/release/igneumd' '$H/bin/igneumd-$ver-$sha' && ln -sfn '$H/bin/igneumd-$ver-$sha' '$H/bin/igneumd' && sha256sum '$H/bin/igneumd-$ver-$sha' | cut -c1-16 && '$H/bin/igneumd' --version"
|
|
say "box binary sha256 $bin_sha; checking its commit string on the box"
|
|
run rssh "[ \$(strings '$H/bin/igneumd' | grep -c '$sha') -gt 0 ] && echo 'commit $sha in the binary' || { echo 'NO commit string in the binary'; exit 1; }"
|
|
if [ -n "$OV_JSON" ]; then
|
|
# the shipper's exact object for the cut (6 Oct 2026: the sixteen-field file at publish 2); checked as a JSON object with the
|
|
# fee switch field, as infra/devnet/restart-hand-nodes.sh checks its argument
|
|
printf '%s' "$OV_JSON" | python3 -c 'import json,sys; o=json.load(sys.stdin); assert isinstance(o, dict) and "fees_v1_activation_daa" in o, "the object must carry fees_v1_activation_daa"; print("override (%d fields):" % len(o), json.dumps(o, sort_keys=True)[:400])'
|
|
ovfile=$(mktemp); printf '%s\n' "$OV_JSON" > "$ovfile"
|
|
else
|
|
[ -f "$MAC_OV" ] || bs_die "no override file at $MAC_OV and no --override-json"
|
|
say "override from the Mac's file: $(cut -c1-200 "$MAC_OV")"; ovfile="$MAC_OV"
|
|
fi
|
|
run bs_rsync -p "$ovfile" "$BS_HOST:$H/override.json"
|
|
[ "$ovfile" = "$MAC_OV" ] || rm -f "$ovfile"
|
|
if [ -f "$MAC_SNAP" ]; then
|
|
snap_sha=$(bs_sha256 "$MAC_SNAP"); [ "$snap_sha" = "$MAC_SNAP_SHA" ] || say "WARNING: snapshot sha256 is $snap_sha, the agents say $MAC_SNAP_SHA; the box gets the file's own"
|
|
say "snapshot: $MAC_SNAP ($(bs_size "$MAC_SNAP") bytes) -> $H/node1-copy-snapshot.bin,$snap_sha"
|
|
run bs_rsync -p "$MAC_SNAP" "$BS_HOST:$H/node1-copy-snapshot.bin"
|
|
run bs_ssh "sed -i 's|^SNAPSHOT=.*|SNAPSHOT=$H/node1-copy-snapshot.bin,$snap_sha|; s|^IGNEUMD=.*|IGNEUMD=$H/bin/igneumd|' $H/hands.env && grep -E '^(IGNEUMD|SNAPSHOT|OVERRIDE)=' $H/hands.env"
|
|
fi
|
|
say "binary step done; next: move-hand.sh observer-node --go" ;;
|
|
|
|
observer-node|node1)
|
|
if [ "$MODE" = node1 ]; then mac_dir=/tmp/igneum-devnet/node1; label=network.igneum.devnet.node1; unit=igneum-node1; else mac_dir=/tmp/igneum-devnet/observer-v4; label=network.igneum.devnet.observer; unit=igneum-observer-node; fi
|
|
[ -d "$mac_dir" ] || bs_die "no data dir $mac_dir on the Mac"
|
|
bs_ssh "[ -x $H/bin/igneumd ] && [ -f $H/override.json ]" || bs_die "the box has no binary or override yet: run move-hand.sh binary --node <wt> first"
|
|
say "$MODE: hot rsync of $mac_dir ($(du -sh "$mac_dir" | cut -f1)) while the Mac node runs"
|
|
run sync_dir "$mac_dir" "$H/$MODE"
|
|
say "$MODE: stopping the Mac's $label, then the final rsync (the delta, seconds), then the unit on the box"
|
|
run mac_stop_agent "$label"
|
|
run sync_dir "$mac_dir" "$H/$MODE"
|
|
run rssh "systemctl start $unit && sleep 3 && systemctl is-active $unit"
|
|
[ "$GO" = 1 ] && first_exec_line "$unit"
|
|
say "$MODE moved; the Mac's agent stays unloaded (step 5 removes the plist from the login)" ;;
|
|
|
|
observer)
|
|
[ -f "$HOME/.config/igneum/env" ] || bs_die "no ~/.config/igneum/env on the Mac"
|
|
grep -q '^DATABASE_URL=' "$HOME/.config/igneum/env" || bs_die "$HOME/.config/igneum/env has no DATABASE_URL line"
|
|
tmp=$(mktemp); chmod 600 "$tmp"
|
|
{ grep -E '^(DATABASE_URL|LIVE_RETAIN_HOURS|LIVE_TABLE_PREFIX)=' "$HOME/.config/igneum/env"; printf 'IGNEUM_RPC=ws://127.0.0.1:28640\nIGNEUM_EVM_RPC=http://127.0.0.1:26840\n'; } > "$tmp"
|
|
say "observer env: $(grep -c . "$tmp") lines (names: $(cut -d= -f1 "$tmp" | tr '\n' ' ')) -> root@$IP:/srv/observer/env mode 600 owner build"
|
|
run scp -q -i "$BS_KEY" -o BatchMode=yes "$tmp" "root@$IP:/srv/observer/env.new"; rm -f "$tmp"
|
|
run rssh "chown build:build /srv/observer/env.new && chmod 600 /srv/observer/env.new && mv /srv/observer/env.new /srv/observer/env && stat -c '%U %a %s bytes' /srv/observer/env"
|
|
run rssh "systemctl is-active igneum-observer-node" || bs_die "igneum-observer-node is not active on the box; move it first"
|
|
say "stopping the Mac's observer: autosync.sh, run.sh, observer.mjs (two writers to Neon would duplicate rows, so the Mac stops first)"
|
|
for pat in 'tools/observer/autosync.sh' 'tools/observer/run.sh' 'tools/observer/observer.mjs'; do
|
|
for p in $(pgrep -f "$pat" || true); do run kill -TERM "$p"; done
|
|
done
|
|
run rssh "systemctl start igneum-observer && sleep 5 && systemctl is-active igneum-observer && journalctl -u igneum-observer --no-pager -o cat -n 6"
|
|
say "observer moved: /api/live reads Neon, which the box's observer now writes" ;;
|
|
|
|
unload)
|
|
say "removing the Mac's launchd agents for good (bootout and the plists moved aside); the hands are on the box"
|
|
for label in network.igneum.devnet.observer network.igneum.devnet.node1; do
|
|
run launchctl bootout "gui/$(id -u)/$label" 2>/dev/null || true
|
|
run mv -f "$HOME/Library/LaunchAgents/$label.plist" "$HOME/Library/LaunchAgents/$label.plist.moved-to-build-1-$(date -u +%Y%m%d)"
|
|
done
|
|
say "Mac igneumd processes now: $(pgrep -fl '[i]gneumd --' | grep -v Wallet | wc -l | tr -d ' ') (the wallet's own node is not a hand)" ;;
|
|
|
|
status)
|
|
echo "--- box:"; rssh "for u in igneum-node1 igneum-observer-node igneum-observer igneum-observer-sync.timer; do printf '%-26s %s\n' \$u \$(systemctl is-active \$u); done; journalctl -u igneum-node1 -o cat -n 2 --no-pager 2>/dev/null | cut -c1-160; journalctl -u igneum-observer -o cat -n 2 --no-pager 2>/dev/null | cut -c1-160"
|
|
echo "--- mac:"; launchctl print "gui/$(id -u)/network.igneum.devnet.node1" 2>/dev/null | grep -E 'state|pid' | head -2; launchctl print "gui/$(id -u)/network.igneum.devnet.observer" 2>/dev/null | grep -E 'state|pid' | head -2; pgrep -fl '[o]bserver.mjs|[o]bserver/run.sh|[a]utosync.sh' || echo "no observer processes on the Mac" ;;
|
|
*) sed -n '2,20p' "$0"; exit 2 ;;
|
|
esac
|