From 7943f83c178e97326fe05b2ea3adc1c2f1c8a57b Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:52:33 +0000 Subject: [PATCH] Devnet hands under launchd: hands-launchd.sh (plists, kickstart, status), restart-hand-nodes.sh hands the start to it; CLAUDE.md rule 6 October 2026, 18:2x UK: the desktop app crashed and both hand nodes, nohup children of agent shells, died with it for about 70 minutes. A LaunchAgent per hand (KeepAlive, RunAtLoad, ThrottleInterval 10, logs under ~/Library/Logs/Igneum) keeps them up; every restart goes through launchctl and prints the pids with parent 1, versions, digests and exec tips. Co-Authored-By: Claude Fable 5.1 --- infra/devnet/hands-launchd.sh | 75 ++++++++++++++++++++++++++++++ infra/devnet/restart-hand-nodes.sh | 22 +++++---- 2 files changed, 87 insertions(+), 10 deletions(-) create mode 100755 infra/devnet/hands-launchd.sh diff --git a/infra/devnet/hands-launchd.sh b/infra/devnet/hands-launchd.sh new file mode 100755 index 00000000..1e1b91df --- /dev/null +++ b/infra/devnet/hands-launchd.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# The two hand-run devnet nodes on this Mac (the observer and node 1) run under launchd, never as children of a shell. +# 6 October 2026, 18:2x UK: the Claude desktop app crashed and both hands died with it (they were nohup children of +# agent shells); the devnet lost its two hand nodes for about 70 minutes. From here a LaunchAgent per hand keeps them up +# (KeepAlive, RunAtLoad), and every restart goes through launchctl, so a closed terminal or a crashed app changes nothing. +# +# infra/devnet/hands-launchd.sh install [,] write both plists for this binary (and +# the exec recovery flag), (re)start the +# observer, wait for it, then node 1 +# infra/devnet/hands-launchd.sh restart kickstart both as they are, observer first +# infra/devnet/hands-launchd.sh status pids, parents, versions, digests, exec tips +# +# The override object stays in /tmp/igneum-devnet/override-v3.json (written by restart-hand-nodes.sh, which calls `restart` +# here after it). Plists: ~/Library/LaunchAgents/network.igneum.devnet.observer.plist and network.igneum.devnet.node1.plist. +# Logs: ~/Library/Logs/Igneum/observer.out and node1.out. Ports as before: observer rpc 26640 / p2p 26641 / evm 26790 (the +# default), node 1 rpc 26610 / p2p 26611 / evm 26791. The two hands never share a port. +set -euo pipefail +UID_="$(id -u)"; LA="$HOME/Library/LaunchAgents"; LOGS="$HOME/Library/Logs/Igneum"; OV=/tmp/igneum-devnet/override-v3.json +LABELS=(network.igneum.devnet.observer network.igneum.devnet.node1) +mkdir -p "$LA" "$LOGS" + +write_plists() { + local bin="$1" snap="${2:-}" + [ -x "$bin" ] || { echo "no binary at $bin"; exit 1; } + python3 - "$bin" "$snap" "$OV" "$LA" "$LOGS" <<'PY' +import sys, plistlib +bin_, snap, ov, la, logs = sys.argv[1:6] +common = ["--devnet", "--nodnsseed", "--disable-upnp", f"--override-params-file={ov}"] +if snap: common.append(f"--igneum-exec-snapshot={snap}") +common += ["--nologfiles", "--yes"] +hands = { + "network.igneum.devnet.observer": [bin_, "--appdir=/tmp/igneum-devnet/observer-v4", "--rpclisten=127.0.0.1:26640", "--rpclisten-json=127.0.0.1:28640", + "--listen=127.0.0.1:26641", "--addpeer=127.0.0.1:26611", "--addpeer=188.245.5.161:26611", "--addpeer=192.168.68.67:26611"] + common, + "network.igneum.devnet.node1": ["/usr/bin/caffeinate", "-dims", bin_, "--enable-unsynced-mining", "--appdir=/tmp/igneum-devnet/node1", "--rpclisten=0.0.0.0:26610", + "--listen=0.0.0.0:26611", "--evm-rpclisten=127.0.0.1:26791", "--addpeer=188.245.5.161:26611", "--addpeer=192.168.68.67:26611"] + common, +} +for label, args in hands.items(): + name = label.rsplit(".", 1)[1] + d = {"Label": label, "ProgramArguments": args, "RunAtLoad": True, "KeepAlive": True, "ThrottleInterval": 10, + "StandardOutPath": f"{logs}/{name}.out", "StandardErrorPath": f"{logs}/{name}.out", "WorkingDirectory": "/tmp/igneum-devnet", + "ProcessType": "Background", "EnvironmentVariables": {"PATH": "/usr/bin:/bin:/usr/sbin:/sbin"}} + with open(f"{la}/{label}.plist", "wb") as f: plistlib.dump(d, f) + print(f"wrote {la}/{label}.plist ({len(args)} args)") +PY + plutil -lint "$LA"/network.igneum.devnet.*.plist >/dev/null +} + +# (re)start one hand: bootstrap if launchd does not know it, else kickstart -k (stop and start); wait for its rpc port +start_one() { + local label="$1" port="$2" i + if launchctl print "gui/$UID_/$label" >/dev/null 2>&1; then launchctl kickstart -k "gui/$UID_/$label"; else launchctl bootstrap "gui/$UID_" "$LA/$label.plist"; fi + for i in $(seq 1 90); do + if curl -s -m 2 -X POST -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' "http://127.0.0.1:$port" | grep -q result; then echo "$label: evm port $port answers after $i s"; return 0; fi + sleep 1 + done + echo "$label: evm port $port silent after 90 s"; return 1 +} + +status() { + local label pid + for label in "${LABELS[@]}"; do + pid="$(launchctl print "gui/$UID_/$label" 2>/dev/null | sed -n 's/^[[:space:]]*pid = \([0-9]*\).*/\1/p' | head -1)" + echo "== $label: launchd pid ${pid:-none}, parent $( [ -n "$pid" ] && ps -o ppid= -p "$pid" | tr -d ' ' ), state $(launchctl print "gui/$UID_/$label" 2>/dev/null | sed -n 's/^[[:space:]]*state = //p' | head -1)" + local name="${label##*.}" + grep -E "igneumd/|Consensus params digest|exec sync: resumed" "$LOGS/$name.out" 2>/dev/null | tail -3 | sed -E 's/^.*\[INFO \] //' | cut -c1-150 + done + for port in 26790 26791; do printf ":%s eth_blockNumber " "$port"; curl -s -m 3 -X POST -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' "http://127.0.0.1:$port" | python3 -c "import json,sys; print(int(json.load(sys.stdin)['result'],16))" 2>/dev/null || echo "no answer"; done +} + +case "${1:-}" in + install) write_plists "$2" "${3:-}"; start_one network.igneum.devnet.observer 26790; start_one network.igneum.devnet.node1 26791; status ;; + restart) start_one network.igneum.devnet.observer 26790; start_one network.igneum.devnet.node1 26791; status ;; + status) status ;; + *) sed -n 2,16p "$0"; exit 2 ;; +esac diff --git a/infra/devnet/restart-hand-nodes.sh b/infra/devnet/restart-hand-nodes.sh index ae382ce5..8f02102a 100755 --- a/infra/devnet/restart-hand-nodes.sh +++ b/infra/devnet/restart-hand-nodes.sh @@ -27,16 +27,18 @@ printf '%s\n' "$OVJSON" > "$OV" # caller's own zsh -c "..." and the stop waited 11 minutes on it) stop() { local p; p=$( { for c in $(pgrep -f "$1" || true); do if [ "$(ps -o comm= -p "$c" 2>/dev/null | xargs basename 2>/dev/null)" = igneumd ]; then echo "$c"; fi; done; true; } | head -1 ); if [ -n "$p" ]; then kill -INT "$p"; while kill -0 "$p" 2>/dev/null; do sleep 1; done; fi; } # the filter never fails the pipeline (17:43Z: a trailing caffeinate pid made the loop exit 1 and set -e ended the script before node 1) lines() { sleep 10; grep -E "Calibrated v1 fees from the override file|Fees on igneum-devnet|Consensus params digest" "$1" | tail -3 || true; } -# the observer first +# 6 October 2026, 19:5x UK: the hands run under launchd (infra/devnet/hands-launchd.sh; a LaunchAgent per hand, KeepAlive), never +# as children of this shell: the desktop app's crash at 18:2x UK took both nohup children down for about 70 minutes. This +# script keeps the object check, the override file and the binary pin, then hands the start to launchd: each hand is booted +# out (and any stray igneumd outside launchd stopped), its exec files reset if asked, and `hands-launchd.sh install` writes +# the plists for THIS binary, starts the observer, waits for its evm port, then node 1, and reads both back. +HERE="$(cd "$(dirname "$0")" && pwd)" +for label in network.igneum.devnet.observer network.igneum.devnet.node1; do launchctl bootout "gui/$(id -u)/$label" 2>/dev/null || true; done stop "igneumd --devnet.*observer-v4" -reset_exec /tmp/igneum-devnet/observer-v4 -nohup "$BIN" --devnet --nodnsseed --disable-upnp --appdir=/tmp/igneum-devnet/observer-v4 --rpclisten=127.0.0.1:26640 --rpclisten-json=127.0.0.1:28640 --listen=127.0.0.1:26641 \ - --addpeer=127.0.0.1:26611 --addpeer=188.245.5.161:26611 --addpeer=192.168.68.67:26611 --override-params-file="$OV" $SNAPARG --nologfiles --yes >> /tmp/igneum-devnet/observer-v4.out 2>&1 & -lines /tmp/igneum-devnet/observer-v4.out -# node 1, under caffeinate as it runs today stop "igneumd --devnet.*appdir=/tmp/igneum-devnet/node1 " +reset_exec /tmp/igneum-devnet/observer-v4 reset_exec /tmp/igneum-devnet/node1 -nohup caffeinate -dims "$BIN" --devnet --nodnsseed --disable-upnp --enable-unsynced-mining --appdir=/tmp/igneum-devnet/node1 --rpclisten=0.0.0.0:26610 --listen=0.0.0.0:26611 --evm-rpclisten=127.0.0.1:26791 \ - --addpeer=188.245.5.161:26611 --addpeer=192.168.68.67:26611 --override-params-file="$OV" $SNAPARG --nologfiles --yes >> /tmp/igneum-devnet/node1.out 2>&1 & -lines /tmp/igneum-devnet/node1.out -echo "running now:"; ps -o pid=,lstart=,command= -p "$(pgrep -f 'igneumd --devnet' | tr '\n' ',' | sed 's/,$//')" | cut -c1-160 +"$HERE/hands-launchd.sh" install "$BIN" "$SNAP" +lines "$HOME/Library/Logs/Igneum/observer.out" +lines "$HOME/Library/Logs/Igneum/node1.out" +echo "running now (launchd children, parent pid 1):"; ps -o pid=,ppid=,lstart=,command= -p "$(pgrep -f 'igneumd --devnet' | tr '\n' ',' | sed 's/,$//')" | cut -c1-160