From 8561fadb3363642e1d4b6e554fd24af9cea5bfb3 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:57:41 +0000 Subject: [PATCH] Build server: the devnet hands' move to igneum-build-1 (plan, installer, mover), a per-worktree lock, the nested-stamp checkout fix the project lead's decision of 6 October 2026: the Mac runs nothing the network depends on. docs/plans/hands-on-build-1.md plans node 1 and the observer (its node and tools/observer) as systemd units on the box, one hand at a time, with ports, DNS names, exec recovery, rollback and the decisions for main; infra/build-server/hands/install-hands.sh writes users, dirs, run scripts and units (inert, run on the box); move-hand.sh (dry run by default) builds 0.3.15 on the box, copies the override and snapshot, rsyncs each data dir hot then stopped, starts the unit and prints the hand's first executing line, copies the observer env by scp (mode 600), stops the Mac's observer first, unloads the launchd agents last. CLAUDE.md's running-agents rule carries the decision. lib.sh: a per-worktree lock on the box across sync, build and fetch (the shipper's collision at 18:48:56Z). remote-run.sh: the clean-tree test excuses stamps and target dirs at any depth with --untracked-files=all (a nested stamp failed every checkout of /srv/builds/igneum for twenty minutes) and removes a stale .git/index.lock; the self-test covers both. Plan rows for the three and the cuda prover pair built for the shipper. Co-Authored-By: Claude Fable 5.1 --- docs/plans/build-server.md | 3 + docs/plans/hands-on-build-1.md | 83 +++++++++++ infra/build-server/hands/install-hands.sh | 168 ++++++++++++++++++++++ infra/build-server/hands/move-hand.sh | 123 ++++++++++++++++ infra/build-server/lib.sh | 19 +++ infra/build-server/remote-run.sh | 6 +- tools/build-remote.sh | 1 + tools/cross-remote.sh | 1 + 8 files changed, 403 insertions(+), 1 deletion(-) create mode 100644 docs/plans/hands-on-build-1.md create mode 100755 infra/build-server/hands/install-hands.sh create mode 100755 infra/build-server/hands/move-hand.sh diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 5951f79a..d3195938 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -93,6 +93,9 @@ Also logged for context: the Mac's Linux cross-build with zig (`infra/cross/buil | `grep -q` plus `pipefail` turned a strings hit into a miss | the commit-string gate failed a stamped igneumd on its first use (SIGPIPE on `strings`) | `grep -c` | | A path dependency inside a vendor repo (the shipper's proving build, 6 Oct 2026) | proving/igneum-prove depends on vendor/igneum-node-exec/igneum/evm-types, a MEMBER of the fork's workspace (it inherits `thiserror` from the fork's root manifest); the first design synced that one directory, so cargo found no workspace root on the box ("failed to load manifest for workspace member"), and the shipper cross-built the Linux prove-host on the Mac with cargo-zigbuild meanwhile | lib.sh groups path dependencies by git top level: one under vendor/ is a whole repository, pushed to its mirror (a fork worktree such as igneum-node-exec goes to /srv/igneum-node.git, which already held its branch; a repository of its own gets /srv/.git, created on first use) and checked out whole at /srv/builds//vendor/; run-from-mac.sh wires every vendor repo the Cargo.toml files reach (today only igneum-node-exec). The detector's first version tested "under BS_TOP" before "own repository" and missed it, since vendor/ lies under the igneum top level on disk. Then `libprotobuf-dev` was missing (sp1-prover-types's build script imports google/protobuf/empty.proto); added to provision.sh. Proof: `cd proving/igneum-prove && tools/build-remote.sh -- build --release -p igneum-prove-host`: igneum-prove-host 71,943,192 B, sha256 e9213e3a6c979512d7859f6d8e848105bab53f4355e99fb0d30fb4a72c2d5714, ELF x86-64, 1 min 05 s warm (the cold run compiled 605 crates in 55 s before protoc stopped it). A Mac worktree has no vendor/ of its own, so a worktree that builds proving needs `git -C vendor/igneum-node worktree add /vendor/igneum-node-exec execution-layer` first, as the fork worktrees do | | One remote build lost its ssh session after 75 s (18:30:50 UTC, the first full proving build) | the remote bash died with it (slot line left behind, no JSONL line); no OOM, no reboot, the retry a minute later passed | the dashboard collector's one-pass unit finished within a second of the drop, so it was tested: a 90 s remote session through the same ControlMaster path survived two collector passes triggered by hand; the collector only reads (/proc, lock files, `flock -n`, `sccache --show-stats`, `kill(pid, 0)`). One event, no cause in the journal, the retry passed. A build that must survive a dropped connection would need the remote command under setsid with the Mac reconnecting to wait; not done, open if it happens again | +| A stamp file at a nested path failed every checkout of /srv/builds/igneum (18:31 to 18:5x UTC, the shipper's 18:52 run) | a repo-kind crate keeps its `.build-remote-sha-` and `target/` inside the crate dir; the checkout mode's clean-tree test only excused them at the tree root | the test is depth-agnostic and uses `--untracked-files=all` (a wholly untracked directory is otherwise collapsed to `?? dir/`); the self-test carries a nested stamp, a nested target dir and a stale `.git/index.lock`, which the mode now removes when no git runs there | +| Two runs on one worktree at once (the shipper, 18:48:56Z) | the second run's checkout replaced the first's sources mid-cargo; both died | lib.sh takes a per-worktree lock on the box (`/srv/builds/_locks/wt-`, mkdir-atomic, holder line) across sync, build and fetch; a second run waits up to 2 h (a line every minute), a lock older than 3 h is taken over; released on EXIT. The build slot (`build-`) is unchanged | +| The 0.3.15 prover pair for the PCs needs `--features igneum-prove-host/cuda` (the PCs run SP1_PROVER=cuda) | my first proving build named no feature | built on the box from master e1b5bc9: igneum-prove-host 73,161,528 B sha256 71bc2438856bb141f6cad3d18489f708568144fad5a06a002fbadefb9ce256f9, igneum-prove-export 3,609,360 B sha256 263bf4cef70af4a13a45b2e79b8dbab373282ea4c571f624d02ddb5791935361 (52 s warm, no CUDA needed at build time); handed to the shipper | | Let's Encrypt saw NXDOMAIN for build.igneum.network | the deSEC record was minutes old; Ubuntu's Caddy then fell back to ZeroSSL and failed with HTTP 422 for ever | issuer pinned to Let's Encrypt; the retry got the certificate | ## 6. What the box does not do yet diff --git a/docs/plans/hands-on-build-1.md b/docs/plans/hands-on-build-1.md new file mode 100644 index 00000000..5bf3ce2d --- /dev/null +++ b/docs/plans/hands-on-build-1.md @@ -0,0 +1,83 @@ +# The devnet hands move to igneum-build-1 (node 1 and the observer) + +the project lead's decision, 6 October 2026: the Mac runs nothing the network depends on. After the 0.3.15 cut tonight, node 1 and the +observer leave the Mac's launchd agents and run on igneum-build-1 (188.40.146.49, Falkenstein, the build server of +docs/plans/build-server.md) as systemd units. The shipper (owner of infra/devnet/restart-hand-nodes.sh and the exec recovery +recipe) and the build-server agent run it together on the shipper's "0.3.15 live" line. + +## 1. What runs on the Mac today (read 6 Oct 2026, 19:5x UK) + +| Hand | How it runs | Flags that matter | Data | +|---|---|---|---| +| node 1 | launchd `network.igneum.devnet.node1`, KeepAlive, under `caffeinate -dims`, binary `igneum-wt-ship0314/vendor/igneum-node-0314/target-integration/release/igneumd` (0.3.14), log `~/Library/Logs/Igneum/node1.out` | `--devnet --enable-unsynced-mining --appdir=/tmp/igneum-devnet/node1 --rpclisten=0.0.0.0:26610 --listen=0.0.0.0:26611 --evm-rpclisten=127.0.0.1:26791 --addpeer=188.245.5.161:26611 --addpeer=192.168.68.67:26611 --override-params-file=/tmp/igneum-devnet/override-v3.json --igneum-exec-snapshot=/tmp/igneum-devnet/node1-copy-snapshot.bin,ac101f13... --nodnsseed --disable-upnp --nologfiles --yes` | 856 MB (`consensus`, `evm` with exec-snapshot.bin 127,564,588 B and .prev, `meta`) | +| observer node | launchd `network.igneum.devnet.observer`, KeepAlive, same binary, log `observer.out` | `--appdir=/tmp/igneum-devnet/observer-v4 --rpclisten=127.0.0.1:26640 --rpclisten-json=127.0.0.1:28640 --listen=127.0.0.1:26641 --addpeer=127.0.0.1:26611 --addpeer=188.245.5.161:26611 --addpeer=192.168.68.67:26611` + the same override and snapshot flags; no EVM listener | 822 MB | +| observer process | `tools/observer/run.sh` loop (nohup, since 5 Oct 20:39) running `node tools/observer/observer.mjs` from the shared checkout, `IGNEUM_RPC=ws://127.0.0.1:28640`, `IGNEUM_EVM_RPC` default `http://127.0.0.1:26800` (the Miner app's node), `DATABASE_URL` from `~/.config/igneum/env`; `tools/observer/autosync.sh` (since 4 Oct) fast-forwards the shared checkout and restarts it on observer changes | writes Neon (`live_*` tables); `/api/live` and `/live` read Neon only | + +Also found: the Mac's own miner (`igneum-miner`, pid 7721) holds a gRPC connection to node 1 on 26610. The override file today: +`{"difficulty_v2_activation_daa":33000,"proving_v0_activation_daa":84100,"fees_v1_activation_daa":210000,"finality_v3_activation_daa":135200,"program_class_v3_activation_daa":154800,"proving_v1_activation_daa":154800,"proving_v1_segment_blocks":8,"proving_v1_unproven_daa":600,"proving_v1_aggregator_share_bps":1000,"proving_v1_fresh_rule_daa":198000,"exec_restart_number":27276,"exec_restart_hash":"bb45cf0d...","exec_restart_trust_daa":200000}`. +Node 1's last exec lines: `exec state loaded from a snapshot: tip 141700 ...` and `exec sync: resumed from .../node1/igneum-devnet/datadir/evm/exec-snapshot.bin (tip 141700, 127564588 bytes, sha256 0x67637c55...)`. + +## 2. What runs on the box afterwards + +| Unit | Runs | Ports | Files | +|---|---|---|---| +| `igneum-node1.service` | `/srv/hands/bin/run-node1.sh` -> `/srv/hands/bin/igneumd` (0.3.15 Linux, the box's own build), `--appdir=/srv/hands/node1`, `--externalip=188.40.146.49` | p2p `0.0.0.0:26611` (ufw open); gRPC `127.0.0.1:26610`, wRPC JSON `127.0.0.1:28610`, EVM `127.0.0.1:26791` on loopback | `/srv/hands/hands.env` (IGNEUMD, OVERRIDE, SNAPSHOT, EXTERNAL_IP, PEERS), `/srv/hands/override.json`, `/srv/hands/node1-copy-snapshot.bin` | +| `igneum-observer-node.service` | `run-observer-node.sh`, `--appdir=/srv/hands/observer-node`, peers node 1 on the box and the seed | p2p `127.0.0.1:26641`, gRPC `127.0.0.1:26640`, wRPC JSON `127.0.0.1:28640`, EVM `127.0.0.1:26840` (new: the observer's proving feed came from the Miner app's node on the Mac) | same env and override | +| `igneum-observer.service` | `/usr/local/bin/node tools/observer/observer.mjs` in `/srv/observer/igneum` (a clone of the mirror `/srv/igneum.git`, master), `EnvironmentFile=/srv/observer/env` (mode 600, owner build: DATABASE_URL copied from the Mac's `~/.config/igneum/env`, `IGNEUM_RPC=ws://127.0.0.1:28640`, `IGNEUM_EVM_RPC=http://127.0.0.1:26840`; never in the repo), Restart=always 3 s (run.sh's loop) | outbound to Neon only | `/srv/observer/env` | +| `igneum-observer-sync.timer` | every 5 min `observer-sync.sh`: fast-forward the clone from the mirror, restart the observer when `tools/observer` or `site/lib` changed (autosync.sh's job; the mirror is fed by every build-remote.sh and run-from-mac.sh push) | | | + +All units: `Restart=always`, journald (`journalctl -u igneum-node1 -f`), `MemoryMax=24G` on the nodes, enabled for reboot, run as user +`build` (one uid on a single-purpose box; the units are the separation). Installer: `infra/build-server/hands/install-hands.sh` +(idempotent, inert: enables, never starts). Mover: `infra/build-server/hands/move-hand.sh` (dry run by default, `--go` executes). + +## 3. Ports, names, firewall + +| Port | On the Mac today | On the box | ufw | +|---|---|---|---| +| 26611 TCP | node 1 p2p, open | node 1 p2p, open, `--externalip` so peers learn it | open already | +| 26610 TCP | node 1 gRPC on `0.0.0.0` (the Mac's miner connects to it) | gRPC on loopback; a Mac tool reaches it through `ssh -L 26610:127.0.0.1:26610 build@188.40.146.49` | not opened (decision for main: open it to a fixed IP list if a miner must feed node 1 from outside) | +| 26791 TCP | node 1 EVM RPC, loopback | loopback | closed | +| 26640, 28640, 26641 TCP | observer node, loopback | loopback (+ EVM 26840) | closed | +| 26811 TCP | (the TESTNET seed p2p port, not an RPC port: infra/seed-nodes/config.sh) | unused by the hands | open from provision; harmless | + +DNS (deSEC, main adds): `node1.devnet.igneum.network A 188.40.146.49`, `observer.devnet.igneum.network A 188.40.146.49`. The +observer node listens on loopback only, so its name is for the future public endpoint and for the runbooks' wording. The public +API (`/api/live`, `/live`) reads Neon and is unchanged. + +## 4. The move, one hand at a time (run on the shipper's "0.3.15 live" line) + +| Step | Command (Mac) | What happens | Proof | +|---|---|---|---| +| 0 | `ssh root@188.40.146.49 'bash -s' < infra/build-server/hands/install-hands.sh` | units, run scripts, dirs, the observer clone; nothing started (done 6 Oct, see section 6) | `systemd-analyze verify` clean, units enabled and inactive | +| 1 | `move-hand.sh binary --node /Users/joshm/Projects/igneum-wt-ship0315/vendor/igneum-node-0315` | the 0.3.15 Linux igneumd built ON the box (tools/build-remote.sh), installed as `/srv/hands/bin/igneumd-0.3.15-`, commit string checked; the Mac's override file and the exec snapshot copied; hands.env pointed at them | `igneumd --version`, commit in strings, sha256 lines | +| 2 | `move-hand.sh observer-node --go` | hot rsync of `/tmp/igneum-devnet/observer-v4` (822 MB) while the Mac node runs; `launchctl bootout` of the Mac's observer agent (KeepAlive would restart a killed pid); final rsync (the delta, seconds, node stopped so RocksDB is consistent); `systemctl start igneum-observer-node` | the unit's first `[igneum-exec] exec sync: resumed from ...` line and its first `Accepted N blocks` lines, printed by the script | +| 3 | `move-hand.sh observer --go` | `/srv/observer/env` written (scp, mode 600, owner build); the Mac's autosync.sh, run.sh and observer.mjs stopped FIRST (two writers would duplicate Neon rows), then `systemctl start igneum-observer` | the observer's first journal lines (`rpc load`, events); `/api/live` fresh within a minute | +| 4 | `move-hand.sh node1 --go` | the same as step 2 for node 1 (856 MB); node 1 is the last node the Mac serves, the observer node on the box already peers with the seed, so the network never loses both hands | node 1's first exec line and `PoW accepted` lines on the box | +| 5 | `move-hand.sh unload --go` | the two plists moved aside so a login never brings the Mac hands back; the Mac's miner loses node 1's RPC (section 5) | `pgrep igneumd` on the Mac shows only the wallet's node | +| 6 | `move-hand.sh status`, 10 minutes later | both nodes at the network tip, observer writing, `/live` current | the status output in this plan's section 6 | + +Exec recovery on the box: each node resumes from its data dir's `evm/exec-snapshot.bin` (copied with the data dir); if that file +is bad or missing, `--igneum-exec-snapshot=/srv/hands/node1-copy-snapshot.bin,` (the Mac's recovery snapshot, copied in +step 1) is taken, as the launchd agents do today; the override's `exec_restart_number`/`exec_restart_hash`/`exec_restart_trust_daa` +travel unchanged. A snapshot from the seed is the fallback the shipper's recipe names; the p2p snapshot path refuses one below the +node's tip (CLAUDE.md, Devnet 2 rules). + +Rollback at any step: the Mac's plist is still in `~/Library/LaunchAgents` until step 5; `launchctl bootstrap gui/$(id -u) ` +brings a hand back on the Mac within 10 s, and the box unit is stopped with `systemctl stop`. Data dirs are copies; nothing is deleted +on the Mac. + +## 5. Decisions and consequences for main + +| Finding | Means | Proposed | +|---|---|---| +| The Mac's own miner (`igneum-miner` pid 7721) mines through node 1's gRPC 26610 | after step 4 it loses its node; the project lead's rule says the Mac runs nothing the network depends on, and a miner is hashrate, not a dependency | either it stops with node 1, or it follows through an ssh tunnel to the box (`ssh -L 26610:...`); main decides, default: it stops | +| `192.168.68.67:26611` is a LAN peer of both hands | unreachable from the box; the box peers with the seed 188.245.5.161 and the two hands peer with each other | hands.env `PEERS=188.245.5.161:26611`; whoever runs 192.168.68.67 adds `--addpeer=188.40.146.49:26611` if it relied on node 1 | +| CLAUDE.md "Running agents on this Mac" says the box never hosts a live-devnet node (my R6, 6 Oct 18:xx) | contradicted by the project lead's decision the same evening | rewritten in this commit: the box hosts the two hands as units; it still holds no secret beyond `/srv/observer/env` (DATABASE_URL, mode 600) | +| The observer's proving feed on the Mac read the Miner app's node (26800) | on the box there is no app node | the observer node gets `--evm-rpclisten=127.0.0.1:26840` (0.3.15 runs the proving build) and the observer reads it; if its shard plans lag node 1's, point `IGNEUM_EVM_RPC` at node 1's 26791 | +| autosync.sh followed origin/master from GitHub; the box has no GitHub credential | the box's clone follows the MIRROR, which moves only when a Mac agent pushes (every build-remote.sh and run-from-mac.sh run pushes the branch it builds; run-from-mac.sh pushes every branch) | enough today; a read-only deploy key on the box (generated there, added by main to the repository) would make it follow GitHub directly, open | +| Two hands stop for one to three minutes each during the move (the final rsync and the start) | the other hand serves throughout; the observer feed pauses once for about a minute (step 3) | accepted by the order above | +| Data dirs are rsynced hot then with the node stopped | the hot pass moves 99 percent of 1.7 GB with the hands up; the stopped pass is the delta | the Mac's upload rate decides the hot pass (minutes); measured in section 6 | + +## 6. Run log (filled as it happens) + +RUNLOG diff --git a/infra/build-server/hands/install-hands.sh b/infra/build-server/hands/install-hands.sh new file mode 100755 index 00000000..d48fc367 --- /dev/null +++ b/infra/build-server/hands/install-hands.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Install the two devnet hands (node 1 and the observer node) and the observer process on igneum-build-1 as systemd units. +# the project lead's decision of 6 October 2026: the Mac runs nothing the network depends on. Run over ssh as root, idempotent, and +# INERT: it writes users, directories, run scripts and units and enables them, but starts nothing; infra/build-server/move-hand.sh +# on the Mac moves one hand at a time (data dir, binary, env, then start) so one hand always serves. Plan: docs/plans/hands-on-build-1.md. +# +# ssh root@ 'bash -s' < infra/build-server/hands/install-hands.sh +# +# Layout (all owned by user build, the same uid the builds use; separation is by unit, not by uid): +# /srv/hands/bin/igneumd-- the Linux igneumd from the box's own build (move-hand.sh copies it from /srv/builds) +# /srv/hands/hands.env IGNEUMD (binary path), OVERRIDE, SNAPSHOT (path,sha256), EXTERNAL_IP, PEERS +# /srv/hands/override.json the consensus override object (the Mac's /tmp/igneum-devnet/override-v3.json) +# /srv/hands/node1, /srv/hands/observer-node the data dirs (--appdir), rsynced from the Mac's /tmp/igneum-devnet/{node1,observer-v4} +# /srv/hands/node1-copy-snapshot.bin the exec snapshot both nodes may resume from (--igneum-exec-snapshot) +# /srv/observer/igneum a clone of /srv/igneum.git (master) for tools/observer and site/lib +# /srv/observer/env DATABASE_URL and the observer's variables, mode 600 (copied by move-hand.sh, never in the repo) +# Units: igneum-node1.service, igneum-observer-node.service (igneumd, Restart=always, journald, SyslogIdentifier igneumd-node1 / +# igneumd-observer), igneum-observer.service (node tools/observer/observer.mjs, Restart=always), igneum-observer-sync.timer +# (every 5 min: fast-forward the clone from the mirror and restart the observer when tools/observer or site/lib changed, the +# box's version of tools/observer/autosync.sh). +# Ports (docs/plans/hands-on-build-1.md section 3): node 1 p2p 0.0.0.0:26611 (ufw open), gRPC 127.0.0.1:26610, EVM 127.0.0.1:26791; +# observer node p2p 127.0.0.1:26641, gRPC 127.0.0.1:26640, wRPC JSON 127.0.0.1:28640, EVM 127.0.0.1:26840. Nothing new in ufw. +set -euo pipefail +U=build; H=/srv/hands; O=/srv/observer +log() { printf '%s install-hands: %s\n' "$(date -u +%H:%M:%S)" "$*"; } +[ "$(id -u)" = 0 ] || { echo "run as root" >&2; exit 1; } +id -u $U >/dev/null 2>&1 || { echo "user $U missing: run infra/build-server/provision.sh first" >&2; exit 1; } +[ -x /usr/local/bin/node ] || { echo "no /usr/local/bin/node: provision.sh installs Node 22" >&2; exit 1; } + +for d in $H $H/bin $H/node1 $H/observer-node $O; do [ -d $d ] || install -d -m 755 -o $U -g $U $d; done +if [ ! -d $O/igneum/.git ]; then su - $U -c "git clone -q /srv/igneum.git $O/igneum"; log "cloned /srv/igneum.git to $O/igneum"; else log "clone ok"; fi +[ -f $H/hands.env ] || { cat > $H/hands.env <<'ENV' +# igneum-build-1 hands (infra/build-server/hands/install-hands.sh). move-hand.sh rewrites IGNEUMD, SNAPSHOT and OVERRIDE. +IGNEUMD=/srv/hands/bin/igneumd +OVERRIDE=/srv/hands/override.json +SNAPSHOT= +EXTERNAL_IP=188.40.146.49 +# peers beside the seeds: the live seed and (until it moves) nothing else; the Mac's LAN peer 192.168.68.67 is unreachable from here +PEERS=188.245.5.161:26611 +ENV +chown $U:$U $H/hands.env; log "wrote $H/hands.env"; } + +cat > $H/bin/run-node1.sh <<'RUN' +#!/usr/bin/env bash +# node 1 on igneum-build-1: the flags of the Mac's launchd agent network.igneum.devnet.node1 (6 Oct 2026), RPC on loopback, +# --externalip so peers learn this address. Edit /srv/hands/hands.env, never this file. +set -euo pipefail +. /srv/hands/hands.env +args=(--devnet --nodnsseed --disable-upnp --enable-unsynced-mining --appdir=/srv/hands/node1 + --rpclisten=127.0.0.1:26610 --rpclisten-json=127.0.0.1:28610 --listen=0.0.0.0:26611 --externalip="$EXTERNAL_IP" --evm-rpclisten=127.0.0.1:26791 + --override-params-file="$OVERRIDE" --nologfiles --yes --maxinpeers=128 --outpeers=8) +IFS=',' read -r -a peers <<< "${PEERS:-}"; for p in "${peers[@]}"; do [ -n "$p" ] && args+=(--addpeer="$p"); done +[ -n "${SNAPSHOT:-}" ] && args+=(--igneum-exec-snapshot="$SNAPSHOT") +[ -n "${EXTRA_ARGS_NODE1:-}" ] && args+=($EXTRA_ARGS_NODE1) +exec "$IGNEUMD" "${args[@]}" +RUN +cat > $H/bin/run-observer-node.sh <<'RUN' +#!/usr/bin/env bash +# the observer's own non-mining node on igneum-build-1: the Mac's network.igneum.devnet.observer flags plus an EVM listener +# on 26840 for the observer's proving feed (the Mac used the Miner app's node for that). Peers: node 1 on this box, the seed. +set -euo pipefail +. /srv/hands/hands.env +args=(--devnet --nodnsseed --disable-upnp --appdir=/srv/hands/observer-node + --rpclisten=127.0.0.1:26640 --rpclisten-json=127.0.0.1:28640 --listen=127.0.0.1:26641 --evm-rpclisten=127.0.0.1:26840 + --addpeer=127.0.0.1:26611 --override-params-file="$OVERRIDE" --nologfiles --yes) +IFS=',' read -r -a peers <<< "${PEERS:-}"; for p in "${peers[@]}"; do [ -n "$p" ] && args+=(--addpeer="$p"); done +[ -n "${SNAPSHOT:-}" ] && args+=(--igneum-exec-snapshot="$SNAPSHOT") +[ -n "${EXTRA_ARGS_OBSERVER:-}" ] && args+=($EXTRA_ARGS_OBSERVER) +exec "$IGNEUMD" "${args[@]}" +RUN +cat > $H/bin/observer-sync.sh <<'RUN' +#!/usr/bin/env bash +# the box's tools/observer/autosync.sh: fast-forward /srv/observer/igneum from the mirror /srv/igneum.git (fed by every +# build-remote.sh and run-from-mac.sh push from the Mac), restart igneum-observer when tools/observer or site/lib changed. +set -uo pipefail +cd /srv/observer/igneum || exit 1 +before=$(git rev-parse HEAD:tools/observer HEAD:site/lib 2>/dev/null | tr '\n' ' ') +su - build -c "git -C /srv/observer/igneum pull -q --ff-only origin master" 2>&1 | head -2 +after=$(git rev-parse HEAD:tools/observer HEAD:site/lib 2>/dev/null | tr '\n' ' ') +if [ "$before" != "$after" ]; then echo "observer files changed ($(git rev-parse --short HEAD)); restarting igneum-observer"; systemctl restart igneum-observer; else echo "observer up to date at $(git rev-parse --short HEAD)"; fi +RUN +chmod 755 $H/bin/*.sh; chown $U:$U $H/bin/*.sh + +unit() { local f="/etc/systemd/system/$1" tmp; tmp=$(mktemp); cat > "$tmp"; if ! cmp -s "$tmp" "$f" 2>/dev/null; then install -m 644 "$tmp" "$f"; echo "$1: written"; else echo "$1: ok"; fi; rm -f "$tmp"; } +unit igneum-node1.service < Neon) +After=network-online.target igneum-observer-node.service +Wants=network-online.target +[Service] +Type=simple +User=$U +Group=$U +WorkingDirectory=$O/igneum +EnvironmentFile=$O/env +Environment=PATH=/usr/local/bin:/usr/bin:/bin +ExecStart=/usr/local/bin/node tools/observer/observer.mjs +Restart=always +RestartSec=3 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=igneum-observer +[Install] +WantedBy=multi-user.target +UNIT +unit igneum-observer-sync.service <&1 | grep -v 'Unit is bound to inactive' || true +# enabled so they come back after a reboot, started only by move-hand.sh +for u in igneum-node1 igneum-observer-node igneum-observer igneum-observer-sync.timer; do systemctl enable -q "$u" 2>/dev/null || true; done +log "units: $(for u in igneum-node1 igneum-observer-node igneum-observer igneum-observer-sync.timer; do printf '%s=%s ' "$u" "$(systemctl is-active "$u" 2>/dev/null)"; done)" +log "env: $( [ -f $O/env ] && stat -c '%U %a' $O/env || echo 'MISSING (move-hand.sh observer copies it)' ); binary: $( [ -x "$(. $H/hands.env; echo "$IGNEUMD")" ] && echo present || echo 'MISSING (move-hand.sh copies it)' )" +log "done (nothing started)" diff --git a/infra/build-server/hands/move-hand.sh b/infra/build-server/hands/move-hand.sh new file mode 100755 index 00000000..46efc82f --- /dev/null +++ b/infra/build-server/hands/move-hand.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Move one devnet hand from this Mac to igneum-build-1, one at a time so one hand always serves (the project lead, 6 October 2026: the Mac +# runs nothing the network depends on). Plan and order: docs/plans/hands-on-build-1.md. Dry run by default; --go executes. +# +# infra/build-server/hands/move-hand.sh binary --node build 0.3.15's Linux igneumd on the box, install it to +# /srv/hands/bin, write the override and snapshot (step 1) +# infra/build-server/hands/move-hand.sh observer-node [--go] hot rsync, stop the Mac's observer node (launchd), final rsync, +# start igneum-observer-node, print its first executing line (step 2) +# infra/build-server/hands/move-hand.sh observer [--go] copy ~/.config/igneum/env to /srv/observer/env (600), stop the +# Mac's run.sh + autosync + observer.mjs, start igneum-observer (step 3) +# infra/build-server/hands/move-hand.sh node1 [--go] the same as observer-node for node 1 (step 4) +# infra/build-server/hands/move-hand.sh unload [--go] bootout the Mac's two launchd agents for good (step 5, last) +# infra/build-server/hands/move-hand.sh status both sides: units, pids, tips, peers +# +# Needs ~/.config/igneum/build-server (build@) and the ops key; root ssh to the box for systemctl, scp of the env file and chown. +# Nothing here prints a secret: the env file travels by scp and is only ever stat'ed. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO="$(cd "$HERE/../../.." && pwd)" +# shellcheck disable=SC2034 +BS_TOOL=move-hand +# shellcheck source=../lib.sh +. "$HERE/../lib.sh" +bs_host +IP="${BS_HOST#*@}" +ROOT_SSH=(ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new "root@$IP") +MAC_OV=/tmp/igneum-devnet/override-v3.json +MAC_SNAP=/tmp/igneum-devnet/node1-copy-snapshot.bin +MAC_SNAP_SHA=ac101f13576179fd7d7f5e8ee902c9a7b6cc47730e3a3c069f389f0ca46d9221 # the launchd agents' value (6 Oct 2026); recomputed below +H=/srv/hands +MODE="${1:-}"; shift || true +GO=0; NODE_WT="" +while [ $# -gt 0 ]; do case "$1" in --go) GO=1; shift ;; --node) NODE_WT="$2"; shift 2 ;; *) bs_die "unknown argument $1" ;; esac; done +say() { bs_log "$*"; } +run() { if [ "$GO" = 1 ]; then "$@"; else say "DRY RUN: $*"; fi; } +rssh() { "${ROOT_SSH[@]}" "$@"; } +first_exec_line() { # : wait up to 180 s for the node's exec line, print it (the proof main wants) and the chain tip + local u="$1" line="" + for _ in $(seq 1 36); do + line=$(rssh "journalctl -u $u --no-pager -o cat --since '5 min ago' | grep -m1 -E 'igneum-exec\] exec (sync: resumed|state loaded)'" 2>/dev/null || true) + [ -n "$line" ] && break; sleep 5 + done + if [ -n "$line" ]; then say "$u first executing line: ${line:0:220}"; else say "$u: no exec line in 180 s; last lines:"; rssh "journalctl -u $u --no-pager -o cat -n 5" | sed 's/^/ /'; fi + rssh "journalctl -u $u --no-pager -o cat --since '5 min ago' | grep -E 'Accepted [0-9]+ blocks|PoW accepted|IBD|Consensus params digest' | tail -3" | cut -c1-200 | sed 's/^/ /' || true +} +sync_dir() { # : rsync a data dir (hot or final), keeping RocksDB files whole + bs_rsync -a --delete --exclude '*.out' "$1/" "$BS_HOST:$2/" +} +mac_stop_agent() { #