diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 5951f79a..d3195938 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -93,6 +93,9 @@ Also logged for context: the Mac's Linux cross-build with zig (`infra/cross/buil | `grep -q` plus `pipefail` turned a strings hit into a miss | the commit-string gate failed a stamped igneumd on its first use (SIGPIPE on `strings`) | `grep -c` | | A path dependency inside a vendor repo (the shipper's proving build, 6 Oct 2026) | proving/igneum-prove depends on vendor/igneum-node-exec/igneum/evm-types, a MEMBER of the fork's workspace (it inherits `thiserror` from the fork's root manifest); the first design synced that one directory, so cargo found no workspace root on the box ("failed to load manifest for workspace member"), and the shipper cross-built the Linux prove-host on the Mac with cargo-zigbuild meanwhile | lib.sh groups path dependencies by git top level: one under vendor/ is a whole repository, pushed to its mirror (a fork worktree such as igneum-node-exec goes to /srv/igneum-node.git, which already held its branch; a repository of its own gets /srv/.git, created on first use) and checked out whole at /srv/builds//vendor/; run-from-mac.sh wires every vendor repo the Cargo.toml files reach (today only igneum-node-exec). The detector's first version tested "under BS_TOP" before "own repository" and missed it, since vendor/ lies under the igneum top level on disk. Then `libprotobuf-dev` was missing (sp1-prover-types's build script imports google/protobuf/empty.proto); added to provision.sh. Proof: `cd proving/igneum-prove && tools/build-remote.sh -- build --release -p igneum-prove-host`: igneum-prove-host 71,943,192 B, sha256 e9213e3a6c979512d7859f6d8e848105bab53f4355e99fb0d30fb4a72c2d5714, ELF x86-64, 1 min 05 s warm (the cold run compiled 605 crates in 55 s before protoc stopped it). A Mac worktree has no vendor/ of its own, so a worktree that builds proving needs `git -C vendor/igneum-node worktree add /vendor/igneum-node-exec execution-layer` first, as the fork worktrees do | | One remote build lost its ssh session after 75 s (18:30:50 UTC, the first full proving build) | the remote bash died with it (slot line left behind, no JSONL line); no OOM, no reboot, the retry a minute later passed | the dashboard collector's one-pass unit finished within a second of the drop, so it was tested: a 90 s remote session through the same ControlMaster path survived two collector passes triggered by hand; the collector only reads (/proc, lock files, `flock -n`, `sccache --show-stats`, `kill(pid, 0)`). One event, no cause in the journal, the retry passed. A build that must survive a dropped connection would need the remote command under setsid with the Mac reconnecting to wait; not done, open if it happens again | +| A stamp file at a nested path failed every checkout of /srv/builds/igneum (18:31 to 18:5x UTC, the shipper's 18:52 run) | a repo-kind crate keeps its `.build-remote-sha-` and `target/` inside the crate dir; the checkout mode's clean-tree test only excused them at the tree root | the test is depth-agnostic and uses `--untracked-files=all` (a wholly untracked directory is otherwise collapsed to `?? dir/`); the self-test carries a nested stamp, a nested target dir and a stale `.git/index.lock`, which the mode now removes when no git runs there | +| Two runs on one worktree at once (the shipper, 18:48:56Z) | the second run's checkout replaced the first's sources mid-cargo; both died | lib.sh takes a per-worktree lock on the box (`/srv/builds/_locks/wt-`, mkdir-atomic, holder line) across sync, build and fetch; a second run waits up to 2 h (a line every minute), a lock older than 3 h is taken over; released on EXIT. The build slot (`build-`) is unchanged | +| The 0.3.15 prover pair for the PCs needs `--features igneum-prove-host/cuda` (the PCs run SP1_PROVER=cuda) | my first proving build named no feature | built on the box from master e1b5bc9: igneum-prove-host 73,161,528 B sha256 71bc2438856bb141f6cad3d18489f708568144fad5a06a002fbadefb9ce256f9, igneum-prove-export 3,609,360 B sha256 263bf4cef70af4a13a45b2e79b8dbab373282ea4c571f624d02ddb5791935361 (52 s warm, no CUDA needed at build time); handed to the shipper | | Let's Encrypt saw NXDOMAIN for build.igneum.network | the deSEC record was minutes old; Ubuntu's Caddy then fell back to ZeroSSL and failed with HTTP 422 for ever | issuer pinned to Let's Encrypt; the retry got the certificate | ## 6. What the box does not do yet diff --git a/docs/plans/hands-on-build-1.md b/docs/plans/hands-on-build-1.md new file mode 100644 index 00000000..5bf3ce2d --- /dev/null +++ b/docs/plans/hands-on-build-1.md @@ -0,0 +1,83 @@ +# The devnet hands move to igneum-build-1 (node 1 and the observer) + +the project lead's decision, 6 October 2026: the Mac runs nothing the network depends on. After the 0.3.15 cut tonight, node 1 and the +observer leave the Mac's launchd agents and run on igneum-build-1 (188.40.146.49, Falkenstein, the build server of +docs/plans/build-server.md) as systemd units. The shipper (owner of infra/devnet/restart-hand-nodes.sh and the exec recovery +recipe) and the build-server agent run it together on the shipper's "0.3.15 live" line. + +## 1. What runs on the Mac today (read 6 Oct 2026, 19:5x UK) + +| Hand | How it runs | Flags that matter | Data | +|---|---|---|---| +| node 1 | launchd `network.igneum.devnet.node1`, KeepAlive, under `caffeinate -dims`, binary `igneum-wt-ship0314/vendor/igneum-node-0314/target-integration/release/igneumd` (0.3.14), log `~/Library/Logs/Igneum/node1.out` | `--devnet --enable-unsynced-mining --appdir=/tmp/igneum-devnet/node1 --rpclisten=0.0.0.0:26610 --listen=0.0.0.0:26611 --evm-rpclisten=127.0.0.1:26791 --addpeer=188.245.5.161:26611 --addpeer=192.168.68.67:26611 --override-params-file=/tmp/igneum-devnet/override-v3.json --igneum-exec-snapshot=/tmp/igneum-devnet/node1-copy-snapshot.bin,ac101f13... --nodnsseed --disable-upnp --nologfiles --yes` | 856 MB (`consensus`, `evm` with exec-snapshot.bin 127,564,588 B and .prev, `meta`) | +| observer node | launchd `network.igneum.devnet.observer`, KeepAlive, same binary, log `observer.out` | `--appdir=/tmp/igneum-devnet/observer-v4 --rpclisten=127.0.0.1:26640 --rpclisten-json=127.0.0.1:28640 --listen=127.0.0.1:26641 --addpeer=127.0.0.1:26611 --addpeer=188.245.5.161:26611 --addpeer=192.168.68.67:26611` + the same override and snapshot flags; no EVM listener | 822 MB | +| observer process | `tools/observer/run.sh` loop (nohup, since 5 Oct 20:39) running `node tools/observer/observer.mjs` from the shared checkout, `IGNEUM_RPC=ws://127.0.0.1:28640`, `IGNEUM_EVM_RPC` default `http://127.0.0.1:26800` (the Miner app's node), `DATABASE_URL` from `~/.config/igneum/env`; `tools/observer/autosync.sh` (since 4 Oct) fast-forwards the shared checkout and restarts it on observer changes | writes Neon (`live_*` tables); `/api/live` and `/live` read Neon only | + +Also found: the Mac's own miner (`igneum-miner`, pid 7721) holds a gRPC connection to node 1 on 26610. The override file today: +`{"difficulty_v2_activation_daa":33000,"proving_v0_activation_daa":84100,"fees_v1_activation_daa":210000,"finality_v3_activation_daa":135200,"program_class_v3_activation_daa":154800,"proving_v1_activation_daa":154800,"proving_v1_segment_blocks":8,"proving_v1_unproven_daa":600,"proving_v1_aggregator_share_bps":1000,"proving_v1_fresh_rule_daa":198000,"exec_restart_number":27276,"exec_restart_hash":"bb45cf0d...","exec_restart_trust_daa":200000}`. +Node 1's last exec lines: `exec state loaded from a snapshot: tip 141700 ...` and `exec sync: resumed from .../node1/igneum-devnet/datadir/evm/exec-snapshot.bin (tip 141700, 127564588 bytes, sha256 0x67637c55...)`. + +## 2. What runs on the box afterwards + +| Unit | Runs | Ports | Files | +|---|---|---|---| +| `igneum-node1.service` | `/srv/hands/bin/run-node1.sh` -> `/srv/hands/bin/igneumd` (0.3.15 Linux, the box's own build), `--appdir=/srv/hands/node1`, `--externalip=188.40.146.49` | p2p `0.0.0.0:26611` (ufw open); gRPC `127.0.0.1:26610`, wRPC JSON `127.0.0.1:28610`, EVM `127.0.0.1:26791` on loopback | `/srv/hands/hands.env` (IGNEUMD, OVERRIDE, SNAPSHOT, EXTERNAL_IP, PEERS), `/srv/hands/override.json`, `/srv/hands/node1-copy-snapshot.bin` | +| `igneum-observer-node.service` | `run-observer-node.sh`, `--appdir=/srv/hands/observer-node`, peers node 1 on the box and the seed | p2p `127.0.0.1:26641`, gRPC `127.0.0.1:26640`, wRPC JSON `127.0.0.1:28640`, EVM `127.0.0.1:26840` (new: the observer's proving feed came from the Miner app's node on the Mac) | same env and override | +| `igneum-observer.service` | `/usr/local/bin/node tools/observer/observer.mjs` in `/srv/observer/igneum` (a clone of the mirror `/srv/igneum.git`, master), `EnvironmentFile=/srv/observer/env` (mode 600, owner build: DATABASE_URL copied from the Mac's `~/.config/igneum/env`, `IGNEUM_RPC=ws://127.0.0.1:28640`, `IGNEUM_EVM_RPC=http://127.0.0.1:26840`; never in the repo), Restart=always 3 s (run.sh's loop) | outbound to Neon only | `/srv/observer/env` | +| `igneum-observer-sync.timer` | every 5 min `observer-sync.sh`: fast-forward the clone from the mirror, restart the observer when `tools/observer` or `site/lib` changed (autosync.sh's job; the mirror is fed by every build-remote.sh and run-from-mac.sh push) | | | + +All units: `Restart=always`, journald (`journalctl -u igneum-node1 -f`), `MemoryMax=24G` on the nodes, enabled for reboot, run as user +`build` (one uid on a single-purpose box; the units are the separation). Installer: `infra/build-server/hands/install-hands.sh` +(idempotent, inert: enables, never starts). Mover: `infra/build-server/hands/move-hand.sh` (dry run by default, `--go` executes). + +## 3. Ports, names, firewall + +| Port | On the Mac today | On the box | ufw | +|---|---|---|---| +| 26611 TCP | node 1 p2p, open | node 1 p2p, open, `--externalip` so peers learn it | open already | +| 26610 TCP | node 1 gRPC on `0.0.0.0` (the Mac's miner connects to it) | gRPC on loopback; a Mac tool reaches it through `ssh -L 26610:127.0.0.1:26610 build@188.40.146.49` | not opened (decision for main: open it to a fixed IP list if a miner must feed node 1 from outside) | +| 26791 TCP | node 1 EVM RPC, loopback | loopback | closed | +| 26640, 28640, 26641 TCP | observer node, loopback | loopback (+ EVM 26840) | closed | +| 26811 TCP | (the TESTNET seed p2p port, not an RPC port: infra/seed-nodes/config.sh) | unused by the hands | open from provision; harmless | + +DNS (deSEC, main adds): `node1.devnet.igneum.network A 188.40.146.49`, `observer.devnet.igneum.network A 188.40.146.49`. The +observer node listens on loopback only, so its name is for the future public endpoint and for the runbooks' wording. The public +API (`/api/live`, `/live`) reads Neon and is unchanged. + +## 4. The move, one hand at a time (run on the shipper's "0.3.15 live" line) + +| Step | Command (Mac) | What happens | Proof | +|---|---|---|---| +| 0 | `ssh root@188.40.146.49 'bash -s' < infra/build-server/hands/install-hands.sh` | units, run scripts, dirs, the observer clone; nothing started (done 6 Oct, see section 6) | `systemd-analyze verify` clean, units enabled and inactive | +| 1 | `move-hand.sh binary --node /Users/joshm/Projects/igneum-wt-ship0315/vendor/igneum-node-0315` | the 0.3.15 Linux igneumd built ON the box (tools/build-remote.sh), installed as `/srv/hands/bin/igneumd-0.3.15-`, commit string checked; the Mac's override file and the exec snapshot copied; hands.env pointed at them | `igneumd --version`, commit in strings, sha256 lines | +| 2 | `move-hand.sh observer-node --go` | hot rsync of `/tmp/igneum-devnet/observer-v4` (822 MB) while the Mac node runs; `launchctl bootout` of the Mac's observer agent (KeepAlive would restart a killed pid); final rsync (the delta, seconds, node stopped so RocksDB is consistent); `systemctl start igneum-observer-node` | the unit's first `[igneum-exec] exec sync: resumed from ...` line and its first `Accepted N blocks` lines, printed by the script | +| 3 | `move-hand.sh observer --go` | `/srv/observer/env` written (scp, mode 600, owner build); the Mac's autosync.sh, run.sh and observer.mjs stopped FIRST (two writers would duplicate Neon rows), then `systemctl start igneum-observer` | the observer's first journal lines (`rpc load`, events); `/api/live` fresh within a minute | +| 4 | `move-hand.sh node1 --go` | the same as step 2 for node 1 (856 MB); node 1 is the last node the Mac serves, the observer node on the box already peers with the seed, so the network never loses both hands | node 1's first exec line and `PoW accepted` lines on the box | +| 5 | `move-hand.sh unload --go` | the two plists moved aside so a login never brings the Mac hands back; the Mac's miner loses node 1's RPC (section 5) | `pgrep igneumd` on the Mac shows only the wallet's node | +| 6 | `move-hand.sh status`, 10 minutes later | both nodes at the network tip, observer writing, `/live` current | the status output in this plan's section 6 | + +Exec recovery on the box: each node resumes from its data dir's `evm/exec-snapshot.bin` (copied with the data dir); if that file +is bad or missing, `--igneum-exec-snapshot=/srv/hands/node1-copy-snapshot.bin,` (the Mac's recovery snapshot, copied in +step 1) is taken, as the launchd agents do today; the override's `exec_restart_number`/`exec_restart_hash`/`exec_restart_trust_daa` +travel unchanged. A snapshot from the seed is the fallback the shipper's recipe names; the p2p snapshot path refuses one below the +node's tip (CLAUDE.md, Devnet 2 rules). + +Rollback at any step: the Mac's plist is still in `~/Library/LaunchAgents` until step 5; `launchctl bootstrap gui/$(id -u) ` +brings a hand back on the Mac within 10 s, and the box unit is stopped with `systemctl stop`. Data dirs are copies; nothing is deleted +on the Mac. + +## 5. Decisions and consequences for main + +| Finding | Means | Proposed | +|---|---|---| +| The Mac's own miner (`igneum-miner` pid 7721) mines through node 1's gRPC 26610 | after step 4 it loses its node; the project lead's rule says the Mac runs nothing the network depends on, and a miner is hashrate, not a dependency | either it stops with node 1, or it follows through an ssh tunnel to the box (`ssh -L 26610:...`); main decides, default: it stops | +| `192.168.68.67:26611` is a LAN peer of both hands | unreachable from the box; the box peers with the seed 188.245.5.161 and the two hands peer with each other | hands.env `PEERS=188.245.5.161:26611`; whoever runs 192.168.68.67 adds `--addpeer=188.40.146.49:26611` if it relied on node 1 | +| CLAUDE.md "Running agents on this Mac" says the box never hosts a live-devnet node (my R6, 6 Oct 18:xx) | contradicted by the project lead's decision the same evening | rewritten in this commit: the box hosts the two hands as units; it still holds no secret beyond `/srv/observer/env` (DATABASE_URL, mode 600) | +| The observer's proving feed on the Mac read the Miner app's node (26800) | on the box there is no app node | the observer node gets `--evm-rpclisten=127.0.0.1:26840` (0.3.15 runs the proving build) and the observer reads it; if its shard plans lag node 1's, point `IGNEUM_EVM_RPC` at node 1's 26791 | +| autosync.sh followed origin/master from GitHub; the box has no GitHub credential | the box's clone follows the MIRROR, which moves only when a Mac agent pushes (every build-remote.sh and run-from-mac.sh run pushes the branch it builds; run-from-mac.sh pushes every branch) | enough today; a read-only deploy key on the box (generated there, added by main to the repository) would make it follow GitHub directly, open | +| Two hands stop for one to three minutes each during the move (the final rsync and the start) | the other hand serves throughout; the observer feed pauses once for about a minute (step 3) | accepted by the order above | +| Data dirs are rsynced hot then with the node stopped | the hot pass moves 99 percent of 1.7 GB with the hands up; the stopped pass is the delta | the Mac's upload rate decides the hot pass (minutes); measured in section 6 | + +## 6. Run log (filled as it happens) + +RUNLOG diff --git a/infra/build-server/hands/install-hands.sh b/infra/build-server/hands/install-hands.sh new file mode 100755 index 00000000..d48fc367 --- /dev/null +++ b/infra/build-server/hands/install-hands.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Install the two devnet hands (node 1 and the observer node) and the observer process on igneum-build-1 as systemd units. +# the project lead's decision of 6 October 2026: the Mac runs nothing the network depends on. Run over ssh as root, idempotent, and +# INERT: it writes users, directories, run scripts and units and enables them, but starts nothing; infra/build-server/move-hand.sh +# on the Mac moves one hand at a time (data dir, binary, env, then start) so one hand always serves. Plan: docs/plans/hands-on-build-1.md. +# +# ssh root@ 'bash -s' < infra/build-server/hands/install-hands.sh +# +# Layout (all owned by user build, the same uid the builds use; separation is by unit, not by uid): +# /srv/hands/bin/igneumd-- the Linux igneumd from the box's own build (move-hand.sh copies it from /srv/builds) +# /srv/hands/hands.env IGNEUMD (binary path), OVERRIDE, SNAPSHOT (path,sha256), EXTERNAL_IP, PEERS +# /srv/hands/override.json the consensus override object (the Mac's /tmp/igneum-devnet/override-v3.json) +# /srv/hands/node1, /srv/hands/observer-node the data dirs (--appdir), rsynced from the Mac's /tmp/igneum-devnet/{node1,observer-v4} +# /srv/hands/node1-copy-snapshot.bin the exec snapshot both nodes may resume from (--igneum-exec-snapshot) +# /srv/observer/igneum a clone of /srv/igneum.git (master) for tools/observer and site/lib +# /srv/observer/env DATABASE_URL and the observer's variables, mode 600 (copied by move-hand.sh, never in the repo) +# Units: igneum-node1.service, igneum-observer-node.service (igneumd, Restart=always, journald, SyslogIdentifier igneumd-node1 / +# igneumd-observer), igneum-observer.service (node tools/observer/observer.mjs, Restart=always), igneum-observer-sync.timer +# (every 5 min: fast-forward the clone from the mirror and restart the observer when tools/observer or site/lib changed, the +# box's version of tools/observer/autosync.sh). +# Ports (docs/plans/hands-on-build-1.md section 3): node 1 p2p 0.0.0.0:26611 (ufw open), gRPC 127.0.0.1:26610, EVM 127.0.0.1:26791; +# observer node p2p 127.0.0.1:26641, gRPC 127.0.0.1:26640, wRPC JSON 127.0.0.1:28640, EVM 127.0.0.1:26840. Nothing new in ufw. +set -euo pipefail +U=build; H=/srv/hands; O=/srv/observer +log() { printf '%s install-hands: %s\n' "$(date -u +%H:%M:%S)" "$*"; } +[ "$(id -u)" = 0 ] || { echo "run as root" >&2; exit 1; } +id -u $U >/dev/null 2>&1 || { echo "user $U missing: run infra/build-server/provision.sh first" >&2; exit 1; } +[ -x /usr/local/bin/node ] || { echo "no /usr/local/bin/node: provision.sh installs Node 22" >&2; exit 1; } + +for d in $H $H/bin $H/node1 $H/observer-node $O; do [ -d $d ] || install -d -m 755 -o $U -g $U $d; done +if [ ! -d $O/igneum/.git ]; then su - $U -c "git clone -q /srv/igneum.git $O/igneum"; log "cloned /srv/igneum.git to $O/igneum"; else log "clone ok"; fi +[ -f $H/hands.env ] || { cat > $H/hands.env <<'ENV' +# igneum-build-1 hands (infra/build-server/hands/install-hands.sh). move-hand.sh rewrites IGNEUMD, SNAPSHOT and OVERRIDE. +IGNEUMD=/srv/hands/bin/igneumd +OVERRIDE=/srv/hands/override.json +SNAPSHOT= +EXTERNAL_IP=188.40.146.49 +# peers beside the seeds: the live seed and (until it moves) nothing else; the Mac's LAN peer 192.168.68.67 is unreachable from here +PEERS=188.245.5.161:26611 +ENV +chown $U:$U $H/hands.env; log "wrote $H/hands.env"; } + +cat > $H/bin/run-node1.sh <<'RUN' +#!/usr/bin/env bash +# node 1 on igneum-build-1: the flags of the Mac's launchd agent network.igneum.devnet.node1 (6 Oct 2026), RPC on loopback, +# --externalip so peers learn this address. Edit /srv/hands/hands.env, never this file. +set -euo pipefail +. /srv/hands/hands.env +args=(--devnet --nodnsseed --disable-upnp --enable-unsynced-mining --appdir=/srv/hands/node1 + --rpclisten=127.0.0.1:26610 --rpclisten-json=127.0.0.1:28610 --listen=0.0.0.0:26611 --externalip="$EXTERNAL_IP" --evm-rpclisten=127.0.0.1:26791 + --override-params-file="$OVERRIDE" --nologfiles --yes --maxinpeers=128 --outpeers=8) +IFS=',' read -r -a peers <<< "${PEERS:-}"; for p in "${peers[@]}"; do [ -n "$p" ] && args+=(--addpeer="$p"); done +[ -n "${SNAPSHOT:-}" ] && args+=(--igneum-exec-snapshot="$SNAPSHOT") +[ -n "${EXTRA_ARGS_NODE1:-}" ] && args+=($EXTRA_ARGS_NODE1) +exec "$IGNEUMD" "${args[@]}" +RUN +cat > $H/bin/run-observer-node.sh <<'RUN' +#!/usr/bin/env bash +# the observer's own non-mining node on igneum-build-1: the Mac's network.igneum.devnet.observer flags plus an EVM listener +# on 26840 for the observer's proving feed (the Mac used the Miner app's node for that). Peers: node 1 on this box, the seed. +set -euo pipefail +. /srv/hands/hands.env +args=(--devnet --nodnsseed --disable-upnp --appdir=/srv/hands/observer-node + --rpclisten=127.0.0.1:26640 --rpclisten-json=127.0.0.1:28640 --listen=127.0.0.1:26641 --evm-rpclisten=127.0.0.1:26840 + --addpeer=127.0.0.1:26611 --override-params-file="$OVERRIDE" --nologfiles --yes) +IFS=',' read -r -a peers <<< "${PEERS:-}"; for p in "${peers[@]}"; do [ -n "$p" ] && args+=(--addpeer="$p"); done +[ -n "${SNAPSHOT:-}" ] && args+=(--igneum-exec-snapshot="$SNAPSHOT") +[ -n "${EXTRA_ARGS_OBSERVER:-}" ] && args+=($EXTRA_ARGS_OBSERVER) +exec "$IGNEUMD" "${args[@]}" +RUN +cat > $H/bin/observer-sync.sh <<'RUN' +#!/usr/bin/env bash +# the box's tools/observer/autosync.sh: fast-forward /srv/observer/igneum from the mirror /srv/igneum.git (fed by every +# build-remote.sh and run-from-mac.sh push from the Mac), restart igneum-observer when tools/observer or site/lib changed. +set -uo pipefail +cd /srv/observer/igneum || exit 1 +before=$(git rev-parse HEAD:tools/observer HEAD:site/lib 2>/dev/null | tr '\n' ' ') +su - build -c "git -C /srv/observer/igneum pull -q --ff-only origin master" 2>&1 | head -2 +after=$(git rev-parse HEAD:tools/observer HEAD:site/lib 2>/dev/null | tr '\n' ' ') +if [ "$before" != "$after" ]; then echo "observer files changed ($(git rev-parse --short HEAD)); restarting igneum-observer"; systemctl restart igneum-observer; else echo "observer up to date at $(git rev-parse --short HEAD)"; fi +RUN +chmod 755 $H/bin/*.sh; chown $U:$U $H/bin/*.sh + +unit() { local f="/etc/systemd/system/$1" tmp; tmp=$(mktemp); cat > "$tmp"; if ! cmp -s "$tmp" "$f" 2>/dev/null; then install -m 644 "$tmp" "$f"; echo "$1: written"; else echo "$1: ok"; fi; rm -f "$tmp"; } +unit igneum-node1.service < Neon) +After=network-online.target igneum-observer-node.service +Wants=network-online.target +[Service] +Type=simple +User=$U +Group=$U +WorkingDirectory=$O/igneum +EnvironmentFile=$O/env +Environment=PATH=/usr/local/bin:/usr/bin:/bin +ExecStart=/usr/local/bin/node tools/observer/observer.mjs +Restart=always +RestartSec=3 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=igneum-observer +[Install] +WantedBy=multi-user.target +UNIT +unit igneum-observer-sync.service <&1 | grep -v 'Unit is bound to inactive' || true +# enabled so they come back after a reboot, started only by move-hand.sh +for u in igneum-node1 igneum-observer-node igneum-observer igneum-observer-sync.timer; do systemctl enable -q "$u" 2>/dev/null || true; done +log "units: $(for u in igneum-node1 igneum-observer-node igneum-observer igneum-observer-sync.timer; do printf '%s=%s ' "$u" "$(systemctl is-active "$u" 2>/dev/null)"; done)" +log "env: $( [ -f $O/env ] && stat -c '%U %a' $O/env || echo 'MISSING (move-hand.sh observer copies it)' ); binary: $( [ -x "$(. $H/hands.env; echo "$IGNEUMD")" ] && echo present || echo 'MISSING (move-hand.sh copies it)' )" +log "done (nothing started)" diff --git a/infra/build-server/hands/move-hand.sh b/infra/build-server/hands/move-hand.sh new file mode 100755 index 00000000..46efc82f --- /dev/null +++ b/infra/build-server/hands/move-hand.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Move one devnet hand from this Mac to igneum-build-1, one at a time so one hand always serves (the project lead, 6 October 2026: the Mac +# runs nothing the network depends on). Plan and order: docs/plans/hands-on-build-1.md. Dry run by default; --go executes. +# +# infra/build-server/hands/move-hand.sh binary --node build 0.3.15's Linux igneumd on the box, install it to +# /srv/hands/bin, write the override and snapshot (step 1) +# infra/build-server/hands/move-hand.sh observer-node [--go] hot rsync, stop the Mac's observer node (launchd), final rsync, +# start igneum-observer-node, print its first executing line (step 2) +# infra/build-server/hands/move-hand.sh observer [--go] copy ~/.config/igneum/env to /srv/observer/env (600), stop the +# Mac's run.sh + autosync + observer.mjs, start igneum-observer (step 3) +# infra/build-server/hands/move-hand.sh node1 [--go] the same as observer-node for node 1 (step 4) +# infra/build-server/hands/move-hand.sh unload [--go] bootout the Mac's two launchd agents for good (step 5, last) +# infra/build-server/hands/move-hand.sh status both sides: units, pids, tips, peers +# +# Needs ~/.config/igneum/build-server (build@) and the ops key; root ssh to the box for systemctl, scp of the env file and chown. +# Nothing here prints a secret: the env file travels by scp and is only ever stat'ed. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/../../.." && pwd)" +# shellcheck disable=SC2034 +BS_TOOL=move-hand +# shellcheck source=../lib.sh +. "$HERE/../lib.sh" +bs_host +IP="${BS_HOST#*@}" +ROOT_SSH=(ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new "root@$IP") +MAC_OV=/tmp/igneum-devnet/override-v3.json +MAC_SNAP=/tmp/igneum-devnet/node1-copy-snapshot.bin +MAC_SNAP_SHA=ac101f13576179fd7d7f5e8ee902c9a7b6cc47730e3a3c069f389f0ca46d9221 # the launchd agents' value (6 Oct 2026); recomputed below +H=/srv/hands +MODE="${1:-}"; shift || true +GO=0; NODE_WT="" +while [ $# -gt 0 ]; do case "$1" in --go) GO=1; shift ;; --node) NODE_WT="$2"; shift 2 ;; *) bs_die "unknown argument $1" ;; esac; done +say() { bs_log "$*"; } +run() { if [ "$GO" = 1 ]; then "$@"; else say "DRY RUN: $*"; fi; } +rssh() { "${ROOT_SSH[@]}" "$@"; } +first_exec_line() { # : wait up to 180 s for the node's exec line, print it (the proof main wants) and the chain tip + local u="$1" line="" + for _ in $(seq 1 36); do + line=$(rssh "journalctl -u $u --no-pager -o cat --since '5 min ago' | grep -m1 -E 'igneum-exec\] exec (sync: resumed|state loaded)'" 2>/dev/null || true) + [ -n "$line" ] && break; sleep 5 + done + if [ -n "$line" ]; then say "$u first executing line: ${line:0:220}"; else say "$u: no exec line in 180 s; last lines:"; rssh "journalctl -u $u --no-pager -o cat -n 5" | sed 's/^/ /'; fi + rssh "journalctl -u $u --no-pager -o cat --since '5 min ago' | grep -E 'Accepted [0-9]+ blocks|PoW accepted|IBD|Consensus params digest' | tail -3" | cut -c1-200 | sed 's/^/ /' || true +} +sync_dir() { # : rsync a data dir (hot or final), keeping RocksDB files whole + bs_rsync -a --delete --exclude '*.out' "$1/" "$BS_HOST:$2/" +} +mac_stop_agent() { #