From f4edd1bbcd670fd7262581138cae3d45e9d08e06 Mon Sep 17 00:00:00 2001 From: igneum-josh <337424239+igneum-josh@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:46:32 +0100 Subject: [PATCH] Fleet: the class v4 rehearsal tooling (CUDA worker wrapper, generator-4 worker, pack re-export), the standing-fleet rule (supervisor, library, page block, gpu-fleet.md), the 0.3.15 publish script, the Devnet 2 script's ports and bps knobs, the collector's seed knobs Co-Authored-By: Claude Fable 5.1 --- docs/plans/gpu-fleet.md | 44 ++++++++++++ tools/ci/commit-string-check.sh | 33 +++++++++ tools/fleet/box-dn2.sh | 15 ++-- tools/fleet/box-rehearsal.sh | 48 +++++++++++++ tools/fleet/box-standing.sh | 71 +++++++++++++++++++ tools/fleet/bps-collect.py | 11 +-- tools/fleet/canary-next.sh | 12 +++- tools/fleet/devnet2-gate-next.sh | 104 ---------------------------- tools/fleet/devnet2-gate.sh | 7 +- tools/fleet/dn400-worker.sh | 9 +++ tools/fleet/lib/box.py | 10 +-- tools/fleet/lib/standing.py | 114 +++++++++++++++++++++++++++++++ tools/fleet/page.py | 6 +- tools/fleet/publish-0315.py | 39 +++++++++++ tools/fleet/rehearsal-1bps.json | 1 + tools/fleet/rehearsal-kill.sh | 5 ++ tools/fleet/runpod.py | 4 +- 17 files changed, 404 insertions(+), 129 deletions(-) create mode 100644 docs/plans/gpu-fleet.md create mode 100755 tools/ci/commit-string-check.sh create mode 100755 tools/fleet/box-rehearsal.sh create mode 100755 tools/fleet/box-standing.sh delete mode 100644 tools/fleet/devnet2-gate-next.sh mode change 100755 => 100644 tools/fleet/devnet2-gate.sh create mode 100755 tools/fleet/dn400-worker.sh create mode 100644 tools/fleet/lib/standing.py create mode 100644 tools/fleet/publish-0315.py create mode 100644 tools/fleet/rehearsal-1bps.json create mode 100644 tools/fleet/rehearsal-kill.sh diff --git a/docs/plans/gpu-fleet.md b/docs/plans/gpu-fleet.md new file mode 100644 index 000000000..4dafc3476 --- /dev/null +++ b/docs/plans/gpu-fleet.md @@ -0,0 +1,44 @@ +# The rented GPU fleet: the standing rule and the one-shot rule + +Josh's ruling, 6 October 2026, 19:50 UK ("keep rented cards up"), written as the fleet agent runs it. Tooling in +`tools/fleet/` on branch `gpu-fleet`; every box operation goes through `tools/fleet/lib/` (the box library) and the +box-side scripts it ships. + +## Two kinds of box + +| Kind | Rule | Examples | +|---|---|---| +| Standing | stays up, never destroyed on a job's end; replaced in the same shape when its host dies; node under a supervisor; version follows the live manifest; never joins an experiment | the 13 live-devnet boxes of 6 October (USD 3.50/h), the Devnet 2 set (seed, 3 miners, 2 provers) | +| One-shot | rented for one measurement, destroyed the minute its measurement is in (a running meter is a bug) | the memory-matrix cards, the 8x rigs, the wave pods, the RISC Zero box | + +The standing set's size and cost: 16 on the live devnet (a mix: 5090, 4090s, 3090s, 4070, 3080, A5000, an L4, at least +two AMD when Vast reopens) plus 6 on Devnet 2, about USD 8 an hour, USD 200 a day, inside the daily budget (USD 250 to +1,000). Tonight's 13 cost USD 3.50 an hour (USD 84 a day); the L4, the AMD pair and the Devnet 2 six are owed to the +roster as providers free cards (RunPod gave no pod of 8 card types from 18:59Z, Vast refuses every new rent). + +## What a standing box carries + +| Piece | Where | What it does | +|---|---|---| +| `box-standing.sh` (the supervisor) | `/root/fleet/in/`, started once by `lib/standing.py install` | every 60 s restarts the node, the miner loop (CUDA worker, pack re-exported on seed change) and the prover when gone; every 10 min writes a `RESULT standing` line (uptime, node pid and binary, blocks, DAA, peers, synced, exec tip, miner rate, prover) to `/root/fleet/out/standing.log`; runs the shipper's recovery recipe (`box-exec-snapshot.sh`) when consensus is synced above 1,000 blocks and the exec tip reads 0 | +| `/root/fleet/standing.node` | the box | the node binary the supervisor runs; the supervisor adopts whatever node already runs when the file is absent (a canary's newer binary is never downgraded) and restarts the node only when a publish rewrites the pointer | +| the registry row | `~/Desktop/fleet/boxes.json` | `standing: true`, `role: live|dn2`, `standing_since`, `node_sha16_wanted` after a publish | +| `lib/standing.py` | the Mac | `roster` (role, card, price, uptime), `install`, `check` (ssh alive, supervisor up, synced, exec moving, prover, node binary against the wanted sha), `update` (the manifest's hive package onto the box, pointer rewritten), `rerent` (same card and provider, label suffixed `-r`, old row marked destroyed), `loop` (check every 10 min, re-rent after two dead checks, report what is behind) | +| the fleet page | `dl.igneum.network/fleet-22adafa34bc2/` | a `standing` block (count, USD/h, USD/day, roles, the rule) and `standing`, `role`, `uptime_h` on every box row | + +A publish on the standing set is a script, not the loop: `tools/fleet/publish-0315.py` is the first (the Linux igneumd, +igneum-miner and the generator-4 workers over the package's, the pointer rewritten, the read-back table of version line, +digest and worker sha per box). The loop reports a box behind its wanted sha; it does not move binaries by itself. + +## Ports + +A standing box should carry a mapped p2p port whenever the provider allows it (RunPod: request `26611/tcp` at rent; +Vast: none of tonight's offers mapped one), so that the live devnet stops being a star in which every rented node dials +only the two hand nodes and the hub: the 18:39Z to 19:4xZ finality pause coincided with the hands being down, and run A +(10 blocks/s on a star, 77 to 84 percent red blocks) showed the same weakness on Devnet 2. Tonight's inbound-capable +nodes were igneum-build-1 (26611) and dn2-seed's one mapped port; a true mesh needs pods rented with a p2p port. + +## Clock + +- 19:41Z: the 13 live boxes converted (supervisor started, every node synced, every prover up). +- Owed: the Devnet 2 six after the block-rate runs; the L4 and the AMD pair when a provider reopens; the ports on re-rent. diff --git a/tools/ci/commit-string-check.sh b/tools/ci/commit-string-check.sh new file mode 100755 index 000000000..0ef612d31 --- /dev/null +++ b/tools/ci/commit-string-check.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# The empty-commit class (6 October 2026, found on igneum-build-1): the fork's kaspa-build-info embeds the git commit in +# igneumd and igneum-miner only when the source tree's .git is a DIRECTORY whose HEAD is a symbolic ref to a branch file, +# and once its build script has found nothing it emits no rerun-if-changed, so a persistent target dir keeps the empty hash +# for ever. Every Mac worktree build (.git is a file there) and every PC job build (the zip has no .git) shipped with no +# commit string while the release plans' "commit in its strings" line was being ticked from main-checkout builds. +# Rule: an igneumd binary whose strings do not contain its short commit fails (igneum-miner has no kaspa-build-info +# dependency and never carried one; checked 6 October 2026). tools/build-remote.sh, tools/cross-remote.sh and +# proto-cuda/windows-node/cross-build.sh run this on every igneumd they produce; the shipper's preflight should too. +# +# tools/ci/commit-string-check.sh exit 0 when the binary carries the commit +# tools/ci/commit-string-check.sh --self-test fires on a known-bad and passes a known-good case +set -euo pipefail +if [ "${1:-}" = --self-test ]; then + tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT + printf 'igneumd v2.1.0-3bfe346f\0junk\0' > "$tmp/good"; printf 'igneumd v2.1.0\0junk\0' > "$tmp/bad" + "$0" "$tmp/good" 3bfe346f4de0d9a982627270c791b7855abc5af6 >/dev/null || { echo "commit-string self-test: the good case FAILED"; exit 1; } + if "$0" "$tmp/bad" 3bfe346f4de0d9a982627270c791b7855abc5af6 >/dev/null 2>&1; then echo "commit-string self-test: the bad case PASSED (the gate is blind)"; exit 1; fi + echo "commit-string self-test: fires on the empty binary, passes the stamped one"; exit 0 +fi +bin="${1:-}"; sha="${2:-}" +[ -f "$bin" ] && [ -n "$sha" ] || { echo "usage: commit-string-check.sh | --self-test" >&2; exit 2; } +printf '%s' "$sha" | grep -qE '^[0-9a-f]{7,40}$' || { echo "commit-string: '$sha' is not a commit sha" >&2; exit 2; } +short="${sha:0:7}" # build-info's fallback embeds 7 characters; git rev-parse --short gives 7 or more, all starting the same +# grep -c, not grep -q: -q closes the pipe at the first match, strings dies with SIGPIPE and pipefail turns a hit into a miss +# (the first version of this gate failed a stamped binary that way, 6 October 2026) +hits=$(strings "$bin" | grep -c "$short" || true) +if [ "${hits:-0}" -gt 0 ]; then + echo "commit-string: $(basename "$bin") carries $short" +else + echo "commit-string: $(basename "$bin") does NOT carry its commit $short (kaspa-build-info found no .git directory on a branch; see the header)" >&2 + exit 1 +fi diff --git a/tools/fleet/box-dn2.sh b/tools/fleet/box-dn2.sh index ed937a769..ed617b7d0 100755 --- a/tools/fleet/box-dn2.sh +++ b/tools/fleet/box-dn2.sh @@ -8,7 +8,7 @@ set -uo pipefail F=/root/fleet; OUT=$F/out; mkdir -p $F/in $OUT $F/dn2 $F/mine/packs; exec >> $OUT/dn2.log 2>&1 stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } -LABEL="${LABEL:-dn2}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}"; SEED="${SEED:-}"; NODE_BIN="${NODE_BIN:-$F/in/igneumd-0313}"; PROVER="${PROVER:-0}" +LABEL="${LABEL:-dn2}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}"; SEED="${SEED:-}"; NODE_BIN="${NODE_BIN:-$F/in/igneumd-0313}"; PROVER="${PROVER:-0}"; BPS="${BPS:-}"; RPC_PORT="${RPC_PORT:-26610}"; P2P_PORT="${P2P_PORT:-26611}"; EVM_PORT="${EVM_PORT:-26790}" # other ports on a box whose live node holds 26610/26611 (the wave pods in run A) # BPS=10|5 -> the devnet2-bps fork's IGNEUMD_DEVNET_BPS profile (block-rate run A); unset = 1 block/s echo "RESULT dn2_start $(stamp) label=$LABEL node=$(sha256sum $NODE_BIN | cut -c1-16) seed=${SEED:-none} prover=$PROVER" command -v curl >/dev/null || { apt-get update -qq >/dev/null 2>&1; apt-get install -y -qq curl ca-certificates python3 >/dev/null 2>&1; } if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then @@ -16,16 +16,17 @@ if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then curl -fsSL -o $F/pkg.tgz "https://dl.igneum.network$PKG_PATH" && echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null && mkdir -p /opt/igneum/pkg && tar -C /opt/igneum/pkg --strip-components=1 -xzf $F/pkg.tgz || { echo "RESULT dn2_failed package"; exit 2; } fi B=/opt/igneum/pkg/bin; cp $F/in/dn2-override.json $F/dn2-override.json -pkill -9 -f '^/root/fleet/in/igneumd' 2>/dev/null; pkill -9 -x igneum-miner 2>/dev/null; sleep 2 # never the script's own pattern (17:18Z: dn2-kill.sh killed its caller) +pkill -9 -f '^/root/fleet/in/igneumd-(0313|[0-9a-f]{16}) ' 2>/dev/null; pkill -9 -f "^/opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:$RPC_PORT " 2>/dev/null; sleep 2 # the Devnet 2 node only, never igneumd-v4 (the rehearsal node beside it, 19:00Z) +[ "${FRESH:-0}" = 1 ] && { rm -rf $F/dn2; mkdir -p $F/dn2; [ -s $F/dn2-node.log ] && mv $F/dn2-node.log $F/dn2-node.prev.log; echo "RESULT dn2_fresh $(stamp) appdir wiped for a new genesis"; } # never the script's own pattern (17:18Z: dn2-kill.sh killed its caller) PEER=""; [ -n "$SEED" ] && PEER="--addpeer=$SEED" -IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host nohup $NODE_BIN --devnet --devnet-suffix=2 --appdir=$F/dn2 --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 $PEER --override-params-file=$F/dn2-override.json --nodnsseed --disable-upnp --nologfiles --yes --enable-unsynced-mining ${NODE_EXTRA:-} >> $F/dn2-node.log 2>&1 & +env ${BPS:+IGNEUMD_DEVNET_BPS=$BPS} IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host nohup $NODE_BIN --devnet --devnet-suffix=2 --appdir=$F/dn2 --rpclisten=0.0.0.0:$RPC_PORT --evm-rpclisten=127.0.0.1:$EVM_PORT --listen=0.0.0.0:$P2P_PORT $PEER --override-params-file=$F/dn2-override.json --nodnsseed --disable-upnp --nologfiles --yes --enable-unsynced-mining ${NODE_EXTRA:-} >> $F/dn2-node.log 2>&1 & # (--enable-unsynced-mining: a fresh chain's nodes start unsynced and must mine anyway, Reject(IsInIBD) on the seed at 16:52Z; a comment put inside this line at 17:00Z swallowed the redirect and the ampersand, so the node ran in the foreground and the script never reached the miner) sleep 10 -echo "RESULT dn2_node $(stamp) pid=$(pgrep -f '^/root/fleet/in/igneumd' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)" -for i in $(seq 1 30); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [ -n "$w" ] && break; sleep 5; done +echo "RESULT dn2_node $(stamp) pid=$(pgrep -f '^/root/fleet/in/igneumd-(0313|[0-9a-f]{16}) ' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)" +for i in $(seq 1 30); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:$RPC_PORT 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [ -n "$w" ] && break; sleep 5; done echo "RESULT dn2_watch $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //')" -cd $F/mine && rm -rf packs/dn2 && $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/dn2 > $OUT/dn2-export-pack.log 2>&1 -( while :; do $B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/dn2" --prepare-packs packs/dn2-prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 30 >> $OUT/dn2-miner.log 2>&1; rc=$?; echo "RESULT dn2_miner_exit $(stamp) rc=$rc" >> $OUT/dn2.log; [ $rc = 42 ] && { rm -rf packs/dn2; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/dn2 >> $OUT/dn2-export-pack.log 2>&1; } || sleep 10; done ) & +cd $F/mine && rm -rf packs/dn2 && $B/igneum-miner export-pack grpc://127.0.0.1:$RPC_PORT packs/dn2 > $OUT/dn2-export-pack.log 2>&1 +( while :; do $B/igneum-miner mine grpc://127.0.0.1:$RPC_PORT 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/dn2" --prepare-packs packs/dn2-prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 30 >> $OUT/dn2-miner.log 2>&1; rc=$?; echo "RESULT dn2_miner_exit $(stamp) rc=$rc" >> $OUT/dn2.log; [ $rc = 42 ] && { rm -rf packs/dn2; $B/igneum-miner export-pack grpc://127.0.0.1:$RPC_PORT packs/dn2 >> $OUT/dn2-export-pack.log 2>&1; } || sleep 10; done ) & echo "RESULT dn2_miner_started $(stamp)" if [ "$PROVER" = 1 ] && [ -x /opt/igneum-floor/bin/igneum-prove-host ]; then cd $F && LABEL="$LABEL" WALLET="$WALLET" EXPORT_FROM=0 CHAIN_NAME=igneum-devnet-2 MINER=none RUN_HOURS=48 setsid nohup python3 -u $F/in/box-prover.py > $OUT/dn2-prover-launch.log 2>&1 & diff --git a/tools/fleet/box-rehearsal.sh b/tools/fleet/box-rehearsal.sh new file mode 100755 index 000000000..84c3c76eb --- /dev/null +++ b/tools/fleet/box-rehearsal.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# The class v4 rehearsal node on a fleet box (docs/plans/counter-asic-3-rehearsal.md, 6 October 2026, the re-cut 1-block/s +# object): igneum-devnet-400 from its own genesis in a FRESH appdir, beside the box's live-devnet node (other ports). +# MODE=seed the fork's igneumd, --listen on P2P_LISTEN (the seed box's public port), --utxoindex, no miner +# MODE=miner the fork's igneumd --connect=$SEED, then the miner (igneum-miner-v4 --no-vote): ENGINE=cpu is --engine igneum-pow +# (5 kH/s a box, useless at genesis difficulty 2^27); ENGINE=cuda (default) is the app's shape, the package's +# igneum-worker-cuda through in/dn400-worker.sh (GPU IGNEUM_CUDA_DEVICE, packs under the dn400 appdir). MINER_ONLY=1 leaves the node up and restarts +# the miner and the sampler (run rehearsal-kill.sh first: it stops the loops, the miner and the worker, not the node). +# MODE=stale 0.3.13's igneumd (NODE_BIN) with the same file, --connect=$SEED, no miner: refused at the handshake +# (0.3.13 refuses the v4 fields at parse; OVERRIDE_SRC= gives the handshake refusal) +# A 5-minute sampler writes RESULT lines to /root/fleet/out/dn400.log: the watch line, the node's digest, switch and +# signal lines, PoW rejections, the miner's seed and program-id lines and its rejected= count. +set -uo pipefail +F=/root/fleet; OUT=$F/out; R=$F/in; mkdir -p $OUT; exec >> $OUT/dn400.log 2>&1 +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +MODE="${MODE:?seed|miner|stale}"; LABEL="${LABEL:-box}"; SEED="${SEED:-}"; NODE_BIN="${NODE_BIN:-$R/igneumd-v4}"; MINER_BIN="$R/igneum-miner-v4" +P2P_LISTEN="${P2P_LISTEN:-0.0.0.0:16411}"; RPC=127.0.0.1:16410; JSON=127.0.0.1:16412; EVM=127.0.0.1:16413; ENGINE="${ENGINE:-cuda}"; MINER_ONLY="${MINER_ONLY:-0}"; OVERRIDE_SRC="${OVERRIDE_SRC:-$R/rehearsal-1bps.json}"; APP=$F/dn400; LOG=$F/dn400-node.log +if [ "$MINER_ONLY" != 1 ]; then + pkill -9 -f "^$R/igneumd-v4 " 2>/dev/null; pkill -9 -f "^$R/igneumd-0313 .*devnet-suffix=400" 2>/dev/null; pkill -9 -f "^$MINER_BIN " 2>/dev/null; sleep 2 + rm -rf $APP; mkdir -p $APP + cp "$OVERRIDE_SRC" $F/dn400-override.json + echo "RESULT dn400_start $(stamp) mode=$MODE label=$LABEL node=$(sha256sum $NODE_BIN | cut -c1-16) object=$(sha256sum $F/dn400-override.json | cut -c1-16) seed=${SEED:-none}" + PEER=""; [ -n "$SEED" ] && PEER="--connect=$SEED" + EXTRA=""; [ "$MODE" = seed ] && EXTRA="--utxoindex" + nohup $NODE_BIN --devnet --devnet-suffix=400 --appdir=$APP --rpclisten=$RPC --rpclisten-json=$JSON --evm-rpclisten=$EVM --listen=$P2P_LISTEN $PEER --override-params-file=$F/dn400-override.json $EXTRA --enable-unsynced-mining --nodnsseed --disable-upnp --nologfiles --yes >> $LOG 2>&1 & + sleep 10 + echo "RESULT dn400_node $(stamp) pid=$(pgrep -f "^$NODE_BIN " | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $LOG | tail -1 | awk '{print $2}') switch=\"$(grep -iE 'class v4' $LOG | head -2 | sed -E 's/^[0-9: .+-]*//' | tr '\n' ';' | cut -c1-300)\"" +else + echo "RESULT dn400_miner_restart $(stamp) mode=$MODE label=$LABEL engine=$ENGINE node_pid=$(pgrep -f "^$NODE_BIN " | head -1)" +fi +if [ "$MODE" = miner ]; then + if [ "$ENGINE" = cpu ]; then MARGS="--engine igneum-pow"; else + mkdir -p $APP/packs/prepare; [ -s $APP/packs/live/program.h ] || $MINER_BIN export-pack grpc://$RPC $APP/packs/live >> $OUT/dn400-miner.log 2>&1 + echo "RESULT dn400_pack $(stamp) $(grep -hoE 'IGNEUM_(PROGRAM_CLASS|PROGRAM_ID)[^\n]{0,40}' $APP/packs/live/program.h 2>/dev/null | tr '\n' ' ' | cut -c1-120)" + MARGS="--worker $R/dn400-worker.sh --prepare-packs $APP/packs/prepare --exit-on-seed-change"; fi + ( while :; do $MINER_BIN mine grpc://$RPC 1 100000000 "$LABEL" $MARGS --no-vote --payout-label "$LABEL" --status-secs 30 >> $OUT/dn400-miner.log 2>&1; rc=$?; echo "RESULT dn400_miner_exit $(stamp) rc=$rc" >> $OUT/dn400.log + # the worker restarts on --pack: re-export it from the node so it holds the CURRENT epoch's program (19:00Z: a worker + # restarted on the epoch-0 pack answered every epoch-2 job "epoch seed mismatch" and the chain stood at DAA 1,202) + [ "$ENGINE" = cpu ] || { rm -rf $APP/packs/live; $MINER_BIN export-pack grpc://$RPC $APP/packs/live >> $OUT/dn400-miner.log 2>&1; echo "RESULT dn400_pack $(stamp) $(grep -hoE 'IGNEUM_(PROGRAM_CLASS|PROGRAM_ID)[^\n]{0,40}' $APP/packs/live/program.h 2>/dev/null | tr '\n' ' ' | cut -c1-120)" >> $OUT/dn400.log; } + sleep 3; done ) & + echo "RESULT dn400_miner_started $(stamp)" +fi +( while :; do + sleep 300 + w="$($MINER_BIN watch 1 grpc://$RPC 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | sed -E 's/difficulty=[0-9.]* sink=//' | tail -1)" + echo "RESULT sample $(stamp) $w rejected_node=$(grep -c 'PoW rejected' $LOG) rejected_miner=$(grep STATUS $OUT/dn400-miner.log 2>/dev/null | tail -1 | grep -oE 'rejected=[0-9]+' | head -1) signal=\"$(grep -E 'Program class v4 by miner signal' $LOG | tail -1 | sed -E 's/^[0-9: .+-]*\[INFO \] //' | cut -c1-220)\" ids=\"$(grep -oE 'epoch seed [0-9a-f]{16}.*(class v[34]|program id [0-9a-fx]{18})' $OUT/dn400-miner.log 2>/dev/null | tail -2 | tr '\n' ';' | cut -c1-240)\"" + done ) & +echo "RESULT dn400_sampler_started $(stamp)" diff --git a/tools/fleet/box-standing.sh b/tools/fleet/box-standing.sh new file mode 100755 index 000000000..e1e10cd28 --- /dev/null +++ b/tools/fleet/box-standing.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# The standing-fleet supervisor on a box (Josh's ruling, 6 October 2026 19:50 UK: rented cards stay up; a standing box is +# never destroyed on a job's end). One loop, started once by lib/standing.py and left alone: +# every 60 s the live-devnet node (ROLE=live: /opt/igneum/pkg/bin/igneumd- on the live object) or the Devnet 2 +# node (ROLE=dn2: --devnet-suffix=2 on dn2-override.json) is restarted if its process is gone; the miner +# loop likewise (the package's igneum-miner on the CUDA worker, --exit-on-seed-change, pack re-exported on +# rc 42); PROVER=1 keeps box-prover.py up (the floor host on /opt/igneum-floor). +# every 10 min the recovery recipe (box-exec-snapshot.sh, the shipper's) runs when the node's exec tip reads 0 "from +# snapshot" while consensus is synced above 1,000 blocks (the dead-exec class of the fleet night, row 1/5). +# every 10 min RESULT standing lines to /root/fleet/out/standing.log: uptime, node pid/version/digest, blocks, daa, +# peers, synced, exec tip, miner MH/s and accepted/rejected, prover segments; lib/standing.py reads them. +# The version follows the live manifest from the Mac (lib/standing.py update step): it stages the new binary as +# /root/fleet/in/igneumd- and writes its path into /root/fleet/standing.node; this loop picks it up at the next +# check and restarts the node on it (data dir kept, so seconds). Patterns are anchored on paths (the pgrep rule). +set -uo pipefail +F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; mkdir -p $OUT $F/mine/packs; exec >> $OUT/standing-supervisor.log 2>&1 +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +ROLE="${ROLE:-live}"; LABEL="${LABEL:-standing}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}"; PROVER="${PROVER:-0}"; MINE="${MINE:-1}" +HUB_PEER="${HUB_PEER:-}"; SEED="${SEED:-}"; STARTED=$(date -u +%s) +if [ "$ROLE" = dn2 ]; then APP=$F/dn2; LOG=$F/dn2-node.log; OVR=$F/dn2-override.json; SUFFIX="--devnet-suffix=2"; PACK=dn2; NET=igneum-devnet-2 +else APP=$F/node; LOG=$F/node.log; OVR=$F/override.json; SUFFIX=""; PACK=devnet; NET=igneum-devnet; fi +echo "RESULT standing_start $(stamp) role=$ROLE label=$LABEL prover=$PROVER mine=$MINE" +NODE_RE='^/(opt/igneum/pkg/bin|opt/igneum/pkg\.prev/bin|root/fleet/in)/igneumd[^ ]* ' # any node binary the fleet has ever started on a box +node_running_bin() { pgrep -af "$NODE_RE" | head -1 | awk '{print $2}'; } +# adopt the node that already runs (a canary or swap may have put a newer binary than the package's in place): never downgrade it +[ -s $F/standing.node ] || { rb=$(node_running_bin); [ -n "$rb" ] && echo "$rb" > $F/standing.node; } +node_bin() { [ -s $F/standing.node ] && cat $F/standing.node || { [ -x $B/igneumd-0313 ] && echo $B/igneumd-0313 || echo $B/igneumd; }; } +node_pid() { pgrep -f "$NODE_RE" | head -1; } +start_node() { + local nb; nb=$(node_bin); local peers=""; [ -n "$HUB_PEER" ] && peers="--addpeer=$HUB_PEER"; [ -n "$SEED" ] && peers="$peers --addpeer=$SEED" + [ "$ROLE" = live ] && peers="$peers --addpeer=188.245.5.161:26611" + local ver; ver="$HOST_VERIFIER"; [ -x /opt/igneum-floor/bin/igneum-prove-host ] && ver=/opt/igneum-floor/bin/igneum-prove-host + pkill -9 -f "$NODE_RE" 2>/dev/null; sleep 2 + IGNEUM_PROOF_VERIFIER=${ver:-} setsid nohup $nb --devnet $SUFFIX --appdir=$APP --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 $peers --override-params-file=$OVR --nodnsseed --disable-upnp --nologfiles --yes > $LOG 2>&1 & + sleep 8; echo "RESULT standing_node_started $(stamp) bin=$nb pid=$(node_pid) digest=$(grep -o 'digest: [0-9a-f]*' $LOG | tail -1 | awk '{print substr($2,1,16)}')" +} +HOST_VERIFIER="" +miner_loop() { + cd $F/mine; [ -d packs/$PACK ] || $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/$PACK > $OUT/export-pack.log 2>&1 + while :; do + $B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/$PACK" --prepare-packs packs/$PACK-prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 60 >> $OUT/miner-0.log 2>&1; rc=$? + echo "RESULT standing_miner_exit $(stamp) rc=$rc"; [ $rc = 42 ] && { rm -rf packs/$PACK; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/$PACK >> $OUT/export-pack.log 2>&1; } || sleep 10 + done +} +n=0 +while :; do + cur=$(node_bin); rb=$(node_running_bin) + if [ -z "$rb" ]; then start_node + elif [ -n "$cur" ] && [ "$rb" != "$cur" ] && [ -s $F/standing.node ]; then echo "RESULT standing_node_update $(stamp) from=$rb to=$cur"; start_node; fi + if [ "$MINE" = 1 ] && ! pgrep -f '^/opt/igneum/pkg/bin/igneum-miner mine' >/dev/null && [ -z "${MINER_LOOP_PID:-}" -o ! -e "/proc/${MINER_LOOP_PID:-0}" ]; then + w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'synced=[a-z]*' | tail -1)" + if [ "$w" = synced=true ]; then miner_loop & MINER_LOOP_PID=$!; echo "RESULT standing_miner_started $(stamp) loop=$MINER_LOOP_PID"; fi + fi + if [ "$PROVER" = 1 ] && [ -x /opt/igneum-floor/bin/igneum-prove-host ] && ! pgrep -f '^python3 -u /root/fleet/in/box-prover.py' >/dev/null; then + cd $F && LABEL="$LABEL" WALLET="$WALLET" EXPORT_FROM="${EXPORT_FROM:-0}" CHAIN_NAME=$NET MINER=none RUN_HOURS=240 setsid nohup python3 -u $F/in/box-prover.py > $OUT/prover-launch.log 2>&1 & + echo "RESULT standing_prover_started $(stamp)" + fi + n=$((n+1)) + if [ $((n % 10)) = 1 ]; then + w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1 | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //')" + ex=$(curl -s -m 5 127.0.0.1:26790 -H 'content-type: application/json' -d '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' | grep -oE '"result":"0x[0-9a-f]+"' | cut -d'"' -f4) + exn=$((${ex:-0x0})); blocks=$(printf '%s' "$w" | grep -oE 'blocks=[0-9]+' | cut -d= -f2) + m="$(grep STATUS $OUT/miner-0.log 2>/dev/null | tail -1 | grep -oE 'accepted=[0-9]+ rejected=[0-9]+|\([0-9.]+ MH/s inside jobs' | tr '\n' ' ' | tr -d '(')" + echo "RESULT standing $(stamp) up=$(( $(date -u +%s) - STARTED ))s role=$ROLE node=$(node_pid) bin=$cur version=$($cur --version 2>&1 | head -1 | awk '{print $2}') $w exec=$exn miner=\"$m\" prover=$(pgrep -c -f '^python3 -u /root/fleet/in/box-prover.py') gpu=$(nvidia-smi --query-gpu=utilization.gpu,power.draw --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')" >> $OUT/standing.log + # the dead-exec class: synced consensus, exec at 0 from snapshot -> the shipper's recovery recipe (live devnet only) + if [ "$ROLE" = live ] && [ "${blocks:-0}" -gt 1000 ] && [ "$exn" = 0 ] && [ -x $F/in/box-exec-snapshot.sh ] && [ $((n / 10)) -gt 2 ]; then + echo "RESULT standing_recovery $(stamp) exec=0 blocks=$blocks: running box-exec-snapshot.sh"; HUB_PEER="$HUB_PEER" bash $F/in/box-exec-snapshot.sh; sleep 30 + fi + fi + sleep 60 +done diff --git a/tools/fleet/bps-collect.py b/tools/fleet/bps-collect.py index 9d972a3e8..b80561afe 100644 --- a/tools/fleet/bps-collect.py +++ b/tools/fleet/bps-collect.py @@ -9,18 +9,19 @@ import sys, os, json, time, datetime, re sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); from lib import Box, Registry ROOT = os.path.expanduser("~/Desktop/fleet/bps"); os.makedirs(ROOT, exist_ok=True) def now(): return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -SEED_CMD = r"""L=/root/fleet/dn2-node.log; P=$(pgrep -f '^/root/fleet/in/igneumd' | head -1) +SEED_CMD = r"""L=${SEED_LOG:-/root/fleet/dn2-node.log}; P=$(pgrep -f "${SEED_PAT:-^/root/fleet/in/igneumd}" | head -1) echo accepted=$(grep -c 'PoW accepted' $L) red=$(grep -ciE 'red block|colored red|is red' $L) reorg_max=$(grep -oE 'selected-chain reorg: [0-9]+ chain blocks' $L | grep -oE '[0-9]+ chain' | awk '{if ($1>m) m=$1} END {print m+0}') locks=$(grep -c 'LOCKED' $L) grep -E 'Finality: (checkpoint [0-9]+ determined|checkpoint [0-9]+ LOCKED|certificate at index [0-9]+ received)' $L | tail -40 | sed -E 's/^([0-9-]+ [0-9:.]+)[^]]*\] //' | cut -c1-170 echo cpu_rss=$(ps -o %cpu=,rss= -p $P 2>/dev/null | tr -s ' ') rx_tx=$(cat /proc/$P/net/dev 2>/dev/null | awk 'NR>2 && $1!="lo:" {rx+=$2; tx+=$10} END {print rx, tx}') -/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //' | tail -1 -curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"igneum_getExecStatus","params":[]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; r=sys.stdin.read(); d=json.loads(r).get("result",{}) if r.strip() else {}; print("exec", int(d.get("executedTip","0x0"),16))' 2>/dev/null""" +${SEED_MINER:-/opt/igneum/pkg/bin/igneum-miner} watch 1 grpc://127.0.0.1:${SEED_RPC:-26610} 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //' | tail -1 +curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"igneum_getExecStatus","params":[]}' http://127.0.0.1:${SEED_EVM:-26790}/ | python3 -c 'import sys,json; r=sys.stdin.read(); d=json.loads(r).get("result",{}) if r.strip() else {}; print("exec", int(d.get("executedTip","0x0"),16))' 2>/dev/null""" def sample(run): reg = Registry.load() - seed = next((Box(b["ssh_host"], b["ssh_port"], b["label"], i, b.get("wallet"), b.get("provider")) for i, b in reg.items() if b.get("dn2_seed") and b.get("state") != "destroyed"), None) + sb = next((b for i, b in reg.items() if b.get("dn2_seed") and b.get("state") != "destroyed"), None) + seed = Box(sb["ssh_host"], sb["ssh_port"], sb["label"], None, sb.get("wallet"), sb.get("provider"), sb.get("ssh_user", "root")) if sb else None; seed_env = (sb or {}).get("seed_env") row = {"t": now(), "run": run} if seed: - rc, out, err = seed.run(SEED_CMD, 60); row["seed_raw"] = out[-4000:] + env = " ".join(f"{k}={v}" for k, v in (seed_env or {}).items()); rc, out, err = seed.run((env + " " if env else "") + SEED_CMD, 60); row["seed_raw"] = out[-4000:] for l in out.split("\n"): if l.startswith("accepted="): row.update(dict(kv.split("=") for kv in l.split())) elif l.startswith("blocks="): row.update({"w_" + k: v for k, v in (kv.split("=") for kv in l.split() if "=" in kv)}) diff --git a/tools/fleet/canary-next.sh b/tools/fleet/canary-next.sh index f070a2bf5..ffd5065cc 100755 --- a/tools/fleet/canary-next.sh +++ b/tools/fleet/canary-next.sh @@ -35,8 +35,13 @@ root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/js WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; }; rej0() { cat "$WORK/rej0.$1" 2>/dev/null || echo 0; } echo "CANARY baseline $(stamp)" for r in "${ROWS[@]}"; do set -- $r; x="$(read_box "$2" "$3")"; echo " $1: daa=$(awk '{print $1}' <<< "$x") tip=$(awk '{print $2}' <<< "$x") paidSeg=$(awk '{print $3}' <<< "$x") exec=$(awk '{print $4}' <<< "$x") ver=$(awk '{print $5}' <<< "$x") rej=$(awk '{print $6}' <<< "$x")"; awk '{print $3}' <<< "$x" > "$WORK/paid0.$1"; awk '{print $6}' <<< "$x" > "$WORK/rej0.$1"; done -# install: the binary swap with the override file kept (box-node-swap.sh STEP=binary, which refuses on a digest change) +# install: the binary swap with the override file kept (box-node-swap.sh STEP=binary, which refuses on a digest change); +# SKIP_INSTALL=1 when every box already runs the binary (the window alone) T0=$(date +%s) +if [[ "${SKIP_INSTALL:-0}" == 1 ]]; then + for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue; x="$(read_box "$2" "$3")"; v="$(awk '{print $5}' <<< "$x")"; [[ "$v" == "$WANT" ]] || fail "$1: SKIP_INSTALL but the box runs $v, not $WANT"; done + echo "CANARY install skipped $(stamp): every box already runs $WANT" +else for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue okc=0; for try in 1 2 3; do scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -P "$3" "$LOCAL" "root@$2:/root/fleet/in/igneumd-0313" >/dev/null 2>&1 && { okc=1; break; }; sleep 5; done # Vast's ssh proxy drops a connection now and then (17:14Z); three tries before a FAIL [[ $okc == 1 ]] || fail "$1: scp failed three times" @@ -45,12 +50,15 @@ for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue done echo "CANARY installed $(stamp) on $(( ${#ROWS[@]} - 1 )) boxes in $(( $(date +%s) - T0 )) s" sleep 75 +fi +if [[ "${SKIP_INSTALL:-0}" != 1 ]]; then for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue - l="$(SSH "$2" "$3" "grep -E '^RESULT (node_started|swap_failed)' /root/fleet/out/node-swap.log | tail -1 | cut -c1-140")"; echo " $1: $l"; [[ "$l" == *digest_ok* ]] || fail "$1: the swap did not reach digest_ok ($l)" + l="$(SSH "$2" "$3" "grep -E '^RESULT (node_started|swap_failed)' /root/fleet/out/node-swap.log | tail -1")"; echo " $1: ${l:0:160}"; [[ "$l" == *digest_ok* ]] || fail "$1: the swap did not reach digest_ok (${l:0:200})" done # the provers back on (box-prover.sh keeps a running igneumd-0313 with the verifier) for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh /dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done for p in $MINERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet/mine && pkill -x igneum-miner; sleep 2; rm -rf packs/devnet; /opt/igneum/pkg/bin/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >/dev/null 2>&1; setsid nohup /opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 $1 --worker /opt/igneum/pkg/bin/igneum-worker-cuda --worker-args '--device 0 --pack packs/devnet' --prepare-packs packs/prepare --exit-on-seed-change --evm-address $4 --payout-label $1 --status-secs 30 > /root/fleet/out/mine-only-0.log 2>&1 & echo miner" >/dev/null || fail "$1: miner relaunch"; done +fi echo "CANARY running $(stamp) $MINUTES min"; sleep $((MINUTES * 60)) echo "CANARY checks $(stamp)"; ok=1; paid_any=0; H="" for r in "${ROWS[@]}"; do set -- $r; x="$(read_box "$2" "$3")"; paid="$(awk '{print $3}' <<< "$x")"; ex="$(awk '{print $4}' <<< "$x")"; ver="$(awk '{print $5}' <<< "$x")"; rej="$(awk '{print $6}' <<< "$x")" diff --git a/tools/fleet/devnet2-gate-next.sh b/tools/fleet/devnet2-gate-next.sh deleted file mode 100644 index 90a745e13..000000000 --- a/tools/fleet/devnet2-gate-next.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -# The Devnet 2 release gate (6 October 2026). Given a release's Linux igneumd (a download URL or a local file) and its -# manifest (for the activation field and value, or --activation name=value), it: (1) installs the binary on every -# Devnet 2 box and restarts each node on the CURRENT override, checking the version; (2) sets the activation at tip + MARGIN -# in a new override, restarts every box on it (the seed first) and waits for the crossing; (3) runs N minutes past it; -# (4) prints PASS only when: zero rejected blocks on every box, no selected-chain reorg over depth 3, exec state roots -# equal on every box at a common height, at least one segment record paid since the start, every node's version equal -# to the release's. FAIL names the box and the line. The one-line status goes to ~/Desktop/fleet/devnet2-status.json. -# -# devnet2-gate.sh --binary --sha256 [--activation proving_v1_fresh_rule_daa=NNN | --activation-field F] -# [--margin 600] [--minutes 20] [--dry-run] (dry-run: steps 1 and 4 against the current object, no activation) -set -uo pipefail -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$HOME/Desktop/fleet" -BIN=""; SHA=""; ACT=""; ACT_FIELD=""; MARGIN=600; MINUTES=20; DRY=0 -while [[ $# -gt 0 ]]; do case "$1" in - --binary) BIN="$2"; shift 2 ;; --sha256) SHA="$2"; shift 2 ;; --activation) ACT="$2"; shift 2 ;; --activation-field) ACT_FIELD="$2"; shift 2 ;; - --margin) MARGIN="$2"; shift 2 ;; --minutes) MINUTES="$2"; shift 2 ;; --dry-run) DRY=1; shift ;; *) echo "unknown $1"; exit 2 ;; esac; done -[[ -n "$BIN" && -n "$SHA" ]] || { echo "usage: $0 --binary --sha256 [--activation name=value] [--minutes N] [--dry-run]"; exit 2; } -stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } -LOG="$ROOT/devnet2-gate-$(date -u +%Y%m%dT%H%M%SZ).log"; exec > >(tee -a "$LOG") 2>&1 -echo "GATE start $(stamp) binary=$BIN sha256=${SHA:0:16} activation=${ACT:-none} minutes=$MINUTES dry_run=$DRY" -fail() { echo "FAIL $(stamp) $*"; python3 - "$ROOT" "FAIL: $*" <<'PY' -import json, sys, os, datetime; p=os.path.join(sys.argv[1], "devnet2-status.json"); d=json.load(open(p)) if os.path.exists(p) else {} -d.update({"last_gate": sys.argv[2][:200], "last_gate_at": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")}); json.dump(d, open(p, "w")) -PY -exit 1; } -# the binary, local -LOCAL="$ROOT/dn2-gate-igneumd" -if [[ "$BIN" == http* ]]; then curl -fsSL -o "$LOCAL" "$BIN" || fail "download $BIN"; else cp "$BIN" "$LOCAL"; fi -[[ "$(shasum -a 256 "$LOCAL" | cut -c1-64)" == "$SHA" ]] || fail "sha256 of the binary is $(shasum -a 256 "$LOCAL" | cut -c1-16), not ${SHA:0:16}" -VERSION_WANT="${SHA:0:16}_igneumd_2.1.0" # the running binary's sha256 (first 16) and its --version word -# the boxes -BOXES=(); while IFS= read -r line; do BOXES+=("$line"); done < <(python3 - <<'PY' -import json, os; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json"))) -for iid,b in reg.items(): - if b.get("devnet2") and b.get("state")!="destroyed" and b.get("ssh_host"): print(f"{b['label']} {b['ssh_host']} {b['ssh_port']} {int(bool(b.get('dn2_seed')))} {b.get('wallet')} {int(bool(b.get('dn2_prover')))}") -PY -) -[[ ${#BOXES[@]} -gt 0 ]] || fail "no Devnet 2 boxes in the registry" -SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } -SCP() { scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -P "$2" "${@:3}" "root@$1:/root/fleet/in/" >/dev/null; } -seed_line="$(printf '%s\n' "${BOXES[@]}" | awk '$4==1 {print; exit}')"; SEED_HOST="$(awk '{print $2}' <<< "$seed_line")"; SEED_PORT="$(awk '{print $3}' <<< "$seed_line")" -SEED_PEER="$(python3 - <<'PY' -import json, os; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json"))) -print(next((b.get("dn2_peer","") for b in reg.values() if b.get("dn2_seed") and b.get("state")!="destroyed"), "")) -PY -)" -echo "GATE boxes=${#BOXES[@]} seed=$SEED_HOST:$SEED_PORT peer=$SEED_PEER" -# a reading of every box: height, exec tip and root at a height, rejects, max reorg, version, paid segments -SINCE="${SINCE:-1970-01-01 00:00:00}" # log lines at or after this UTC stamp count (set at step 1) -read_box() { # host port -> "height daa exec_tip paid_seg version rejects max_reorg", rejects and reorgs since $SINCE - SSH "$1" "$2" 'SINCE="'"$SINCE"'"; B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "^/root/fleet/in/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(awk -v s="$SINCE" "substr(\$0,1,19) >= s" /root/fleet/dn2-node.log 2>/dev/null | grep -cE "reject message|PoW rejected|block rejected|invalid block"); mr=$(awk -v s="$SINCE" "substr(\$0,1,19) >= s" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "selected-chain reorg: [0-9]+ chain blocks" | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null -} -root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } -WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; } -echo "GATE baseline $(stamp)" -for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; awk '{print $4}' <<< "$r" > "$WORK/paid0.$1"; done -# 1. install the binary on every box, restart on the current override -for l in "${BOXES[@]}"; do set -- $l - SCP "$2" "$3" "$LOCAL" || fail "$1: scp of the binary" - SSH "$2" "$3" "mv /root/fleet/in/dn2-gate-igneumd /root/fleet/in/igneumd-gate && chmod +x /root/fleet/in/igneumd-gate && [ \"\$(sha256sum /root/fleet/in/igneumd-gate | cut -c1-64)\" = $SHA ]" || fail "$1: the binary's sha256 on the box" -done -restart_all() { # with the override file already on each box as /root/fleet/in/dn2-override.json - for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] || continue - SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f '^/root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" - sleep 15; done - for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] && continue - SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f '^/root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 SEED=$SEED_PEER NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" - done -} -SINCE="$(date -u +'%Y-%m-%d %H:%M:%S')"; echo "GATE step1 $(stamp) restart on the release binary, the current override (rejects and reorgs counted from $SINCE)"; restart_all; sleep 60 -for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; v="$(awk '{print $5}' <<< "$r")"; [[ -z "$VERSION_WANT" || "$v" == "$VERSION_WANT" ]] || fail "$1: version $v, want $VERSION_WANT"; done -# 2. the activation -if [[ $DRY == 0 && -n "$ACT" ]]; then - tip="$(read_box "$SEED_HOST" "$SEED_PORT" | awk '{print $2}')"; name="${ACT%%=*}"; val="${ACT#*=}"; [[ "$val" == auto ]] && val=$((tip + MARGIN)) - python3 - "$HERE/devnet2-override.json" "$name" "$val" "$ROOT/dn2-override-next.json" <<'PY' -import json, sys; d=json.load(open(sys.argv[1])); d[sys.argv[2]]=int(sys.argv[3]); json.dump(d, open(sys.argv[4], "w")) -PY - for l in "${BOXES[@]}"; do set -- $l; SCP "$2" "$3" "$ROOT/dn2-override-next.json" && SSH "$2" "$3" "mv /root/fleet/in/dn2-override-next.json /root/fleet/in/dn2-override.json" || fail "$1: override push"; done - echo "GATE step2 $(stamp) activation $name=$val at tip $tip (margin $MARGIN); restart on the new object"; restart_all - for i in $(seq 1 120); do sleep 15; d="$(read_box "$SEED_HOST" "$SEED_PORT" | awk '{print $2}')"; [[ "${d:-0}" -ge "$val" ]] && { echo "GATE crossed $name=$val at $(stamp) (daa $d)"; break; }; done -fi -# 3. run -echo "GATE step3 $(stamp) running $MINUTES min"; sleep $((MINUTES * 60)) -# 4. the checks -echo "GATE step4 $(stamp) checks"; H=""; ok=1; paid_any=0 -for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r" - rej="$(awk '{print $6}' <<< "$r")"; mr="$(awk '{print $7}' <<< "$r")"; v="$(awk '{print $5}' <<< "$r")"; paid="$(awk '{print $4}' <<< "$r")"; et="$(awk '{print $3}' <<< "$r")" - [[ "$rej" == 0 ]] || { echo "FAIL $1: $rej rejected blocks (grep 'reject' /root/fleet/dn2-node.log)"; ok=0; } - [[ "${mr:-0}" -le 3 ]] || { echo "FAIL $1: a selected-chain reorg of depth $mr"; ok=0; } - [[ -z "$VERSION_WANT" || "$v" == "$VERSION_WANT" ]] || { echo "FAIL $1: version $v"; ok=0; } - [[ "${paid:-0}" -gt "$(paid0 $1)" ]] && paid_any=1 - [[ -z "$H" || "$et" -lt "$H" ]] && H="$et" -done -H=$((H > 20 ? H - 20 : 1)); HX="$(printf '0x%x' "$H")" -first=""; for l in "${BOXES[@]}"; do set -- $l; rt="$(root_at "$2" "$3" "$HX")"; echo " $1 root@$H ${rt:0:18}"; [[ -z "$first" ]] && first="$rt"; [[ "$rt" == "$first" ]] || { echo "FAIL $1: exec root at height $H ${rt:0:18} differs from ${first:0:18}"; ok=0; }; done -[[ $paid_any == 1 ]] || { echo "FAIL no segment record paid on any box during the run"; ok=0; } -res="FAIL"; [[ $ok == 1 ]] && res="PASS" -echo "$res $(stamp) release ${SHA:0:16} activation ${ACT:-none} boxes ${#BOXES[@]} minutes $MINUTES log $LOG" -python3 - "$ROOT" "$res: ${SHA:0:16} ${ACT:-current object}, ${#BOXES[@]} boxes, $MINUTES min" <<'PY' -import json, sys, os, datetime; p=os.path.join(sys.argv[1], "devnet2-status.json"); d=json.load(open(p)) if os.path.exists(p) else {} -d.update({"last_gate": sys.argv[2], "last_gate_at": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")}); json.dump(d, open(p, "w")) -PY -[[ $ok == 1 ]] diff --git a/tools/fleet/devnet2-gate.sh b/tools/fleet/devnet2-gate.sh old mode 100755 new mode 100644 index ed58b37a4..b2e46c798 --- a/tools/fleet/devnet2-gate.sh +++ b/tools/fleet/devnet2-gate.sh @@ -47,8 +47,9 @@ PY )" echo "GATE boxes=${#BOXES[@]} seed=$SEED_HOST:$SEED_PORT peer=$SEED_PEER" # a reading of every box: height, exec tip and root at a height, rejects, max reorg, version, paid segments -read_box() { # host port -> "height daa exec_tip paid_seg version rejects max_reorg" - SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "^/root/fleet/in/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "reject message|PoW rejected|block rejected|invalid block" /root/fleet/dn2-node.log 2>/dev/null); mr=$(grep -oE "selected-chain reorg: [0-9]+ chain blocks" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null +SINCE="${SINCE:-1970-01-01 00:00:00}" # log lines at or after this UTC stamp count (set at step 1) +read_box() { # host port -> "height daa exec_tip paid_seg version rejects max_reorg", rejects and reorgs since $SINCE + SSH "$1" "$2" 'SINCE="'"$SINCE"'"; B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "^/root/fleet/in/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(awk -v s="$SINCE" "substr(\$0,1,19) >= s" /root/fleet/dn2-node.log 2>/dev/null | grep -cE "reject message|PoW rejected|block rejected|invalid block"); mr=$(awk -v s="$SINCE" "substr(\$0,1,19) >= s" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "selected-chain reorg: [0-9]+ chain blocks" | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null } root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; } @@ -67,7 +68,7 @@ restart_all() { # with the override file already on each box as /root/fleet/in/d SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f '^/root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 SEED=$SEED_PEER NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" done } -echo "GATE step1 $(stamp) restart on the release binary, the current override"; restart_all; sleep 60 +SINCE="$(date -u +'%Y-%m-%d %H:%M:%S')"; echo "GATE step1 $(stamp) restart on the release binary, the current override (rejects and reorgs counted from $SINCE)"; restart_all; sleep 60 for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; v="$(awk '{print $5}' <<< "$r")"; [[ -z "$VERSION_WANT" || "$v" == "$VERSION_WANT" ]] || fail "$1: version $v, want $VERSION_WANT"; done # 2. the activation if [[ $DRY == 0 && -n "$ACT" ]]; then diff --git a/tools/fleet/dn400-worker.sh b/tools/fleet/dn400-worker.sh new file mode 100755 index 000000000..5fc17f66b --- /dev/null +++ b/tools/fleet/dn400-worker.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# The CUDA worker for the class v4 rehearsal miner (igneum-miner-v4 --worker ): the package's igneum-worker-cuda +# needs --pack (igneum-miner export-pack writes the first one) and a device; the v4 miner passes only its own flags. +# IGNEUM_CUDA_DEVICE picks the GPU (default 0). The miner's arguments are logged once and passed through. +D="${IGNEUM_CUDA_DEVICE:-0}"; P=/root/fleet/dn400/packs/live +echo "$(date -u +%FT%TZ) worker args from the miner: $*" >> /root/fleet/out/dn400-worker-args.log +case " $* " in *" --serve "*) S="";; *) S="--serve";; esac +W=/opt/igneum/pkg/bin/igneum-worker-cuda; [ -x /root/fleet/in/igneum-worker-cuda-v4 ] && W=/root/fleet/in/igneum-worker-cuda-v4 # the generator-4 worker (release tree 563485b, zig glibc 2.36) once staged; the 0.3.14 package's refuses generator 4 +exec $W $S --device "$D" --pack "$P" "$@" diff --git a/tools/fleet/lib/box.py b/tools/fleet/lib/box.py index cb8f362c0..02c790483 100644 --- a/tools/fleet/lib/box.py +++ b/tools/fleet/lib/box.py @@ -35,17 +35,17 @@ class Registry: raise KeyError(label) class Box: - def __init__(self, host, port, label="box", iid=None, wallet=None, provider="vast"): - self.host, self.port, self.label, self.iid, self.wallet, self.provider = host, int(port), label, iid, wallet, provider + def __init__(self, host, port, label="box", iid=None, wallet=None, provider="vast", user="root"): + self.host, self.port, self.label, self.iid, self.wallet, self.provider, self.user = host, int(port), label, iid, wallet, provider, user @classmethod def from_registry(cls, label): iid, b = Registry.find(label) if not b.get("ssh_host") or not b.get("ssh_port"): raise SshError(f"{label}: no ssh endpoint in the registry") - return cls(b["ssh_host"], b["ssh_port"], label, iid, b.get("wallet"), b.get("provider", "vast")) + return cls(b["ssh_host"], b["ssh_port"], label, iid, b.get("wallet"), b.get("provider", "vast"), b.get("ssh_user", "root")) # ---- transport ---- def run(self, cmd, timeout=60): try: - r = subprocess.run(["ssh"] + SSH_OPTS + ["-p", str(self.port), f"root@{self.host}", cmd], capture_output=True, text=True, timeout=timeout, stdin=subprocess.DEVNULL) + r = subprocess.run(["ssh"] + SSH_OPTS + ["-p", str(self.port), f"{self.user}@{self.host}", cmd], capture_output=True, text=True, timeout=timeout, stdin=subprocess.DEVNULL) except subprocess.TimeoutExpired: return 124, "", "timeout" return r.returncode, r.stdout or "", r.stderr or "" def check(self, cmd, timeout=60): @@ -54,7 +54,7 @@ class Box: return out def put(self, files, dest, tries=3, timeout=900): for i in range(tries): - r = subprocess.run(["scp"] + SSH_OPTS + ["-P", str(self.port)] + list(files) + [f"root@{self.host}:{dest}"], capture_output=True, text=True, timeout=timeout) + r = subprocess.run(["scp"] + SSH_OPTS + ["-P", str(self.port)] + list(files) + [f"{self.user}@{self.host}:{dest}"], capture_output=True, text=True, timeout=timeout) if r.returncode == 0: return True time.sleep(5) raise SshError(f"{self.label}: scp failed {tries} times: {r.stderr.strip()[:120]}") diff --git a/tools/fleet/lib/standing.py b/tools/fleet/lib/standing.py new file mode 100644 index 000000000..2fa58a1b8 --- /dev/null +++ b/tools/fleet/lib/standing.py @@ -0,0 +1,114 @@ +"""The standing fleet (Josh's ruling, 6 October 2026, 19:50 UK): rented cards that stay up and are never destroyed on a +job's end. A box is standing when its registry row carries standing=true and a role: live (the live devnet: node, miner, +prover, voter) or dn2 (Devnet 2: seed, miner or prover). This module is the Mac side of the rule; box-standing.sh is the +box side (the supervisor loop). Every call goes through lib.box. + + roster() the standing rows with role, card, price and uptime + install(label) puts box-standing.sh (and the recovery recipe) on the box and starts the supervisor once + check() one pass: ssh alive? supervisor up? node synced? exec moving? miner and prover present? + version against the live manifest (dl.igneum.network/dl/public/igneum-downloads.json, + files.miner-hive.version); returns one dict per box and writes ~/Desktop/fleet/standing.jsonl + update(label, pkg) the version follow: the box downloads the manifest's hive package (sha256 checked), unpacks it + to /opt/igneum/pkg.new, swaps it in and writes /root/fleet/standing.node; the supervisor + restarts the node on it (data dir kept, seconds) + rerent(label) the host died (ssh dead for two checks, or the provider says the instance is gone): rent the + same shape (card, VRAM, provider) with the label suffixed "-r", set it up, install the + supervisor, mark the old row destroyed; the Devnet 2 roles take their seed from the registry + loop(every=600) check, then rerent what died and update what is behind, every ten minutes +Nothing here destroys a standing box; destroy() in lib.box stays for the one-shot boxes. +""" +import os, sys, json, time, datetime, subprocess +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from lib.box import Box, Registry, F, SshError +HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +ROOT = os.path.expanduser("~/Desktop/fleet"); MANIFEST = "https://dl.igneum.network/dl/public/igneum-downloads.json" +def now(): return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +def roster(): + reg = Registry.load(); rows = [] + for iid, b in reg.items(): + if not b.get("standing") or b.get("state") == "destroyed": continue + since = b.get("standing_since") or b.get("rented_at") or now() + up_h = (datetime.datetime.now(datetime.timezone.utc) - datetime.datetime.fromisoformat(since.replace("Z", "+00:00"))).total_seconds() / 3600 + rows.append({"iid": iid, "label": b["label"], "role": b.get("role", "live"), "card": b.get("card"), "provider": b.get("provider"), "dph": round(float(b.get("dph") or 0), 3), "uptime_h": round(up_h, 1), "since": since}) + return sorted(rows, key=lambda r: (r["role"], r["label"])) +def manifest(): + try: return json.loads(subprocess.run(["curl", "-fsSL", "-m", "20", MANIFEST], capture_output=True, text=True, timeout=30).stdout)["files"]["miner-hive"] + except Exception: return {} +def install(label, role=None, prover=None, mine=None): + """Mark the row standing and start the supervisor on the box; idempotent (a running supervisor is left alone).""" + iid, b = Registry.find(label); role = role or b.get("role", "live"); b_ = Box.from_registry(label) + prover = "1" if (prover if prover is not None else role == "live" or b.get("dn2_prover")) else "0" + mine = "1" if (mine if mine is not None else not b.get("dn2_seed")) else "0" + b_.put([os.path.join(HERE, "box-standing.sh"), os.path.join(HERE, "box-exec-snapshot.sh")], f"{F}/in/") + hub = next((v for v in Registry.load().values() if v.get("hub") and v.get("state") != "destroyed"), {}) + seed = b.get("dn2_seed_addr", "") + rc, out, err = b_.run(f"cd {F} && chmod +x in/box-standing.sh in/box-exec-snapshot.sh; pgrep -f '^bash in/box-standing.sh' >/dev/null && echo supervisor-already || {{ ROLE={role} LABEL={label} WALLET={b.get('wallet')} PROVER={prover} MINE={mine} HUB_PEER={hub.get('hub_peer','')} SEED={seed} setsid nohup bash in/box-standing.sh /dev/null 2>&1 & sleep 2; echo supervisor-started; }}", 60) + Registry.patch(iid, standing=True, role=role, standing_since=b.get("standing_since") or now(), doing=f"standing {role}: node, {'miner, ' if mine == '1' else ''}{'prover' if prover == '1' else 'no prover'} (supervised)") + return out.strip() +def check_one(r): + b = Box.from_registry(r["label"]); row = dict(r); row["t"] = now() + rc, out, err = b.run("echo alive; pgrep -fc '^bash in/box-standing.sh'; tail -1 /root/fleet/out/standing.log 2>/dev/null | cut -c1-400", 45) + if rc != 0 or "alive" not in out: row.update(alive=False); return row + lines = out.strip().split("\n"); row.update(alive=True, supervisor=int(lines[1] or 0) if len(lines) > 1 else 0, last=lines[2] if len(lines) > 2 else "") + kv = dict(x.split("=", 1) for x in row["last"].split() if "=" in x and not x.startswith("miner=")) + row.update(blocks=int(kv.get("blocks", 0) or 0), synced=kv.get("synced") == "true", exec_tip=int(kv.get("exec", 0) or 0), version=kv.get("version", ""), node_pid=kv.get("node", ""), prover=int(kv.get("prover", 0) or 0), bin=kv.get("bin", "")) + row["bin_sha16"] = os.path.basename(row["bin"]).replace("igneumd-", "") if "igneumd-" in row["bin"] and len(os.path.basename(row["bin"])) == 24 else "" + return row +def check(write=True): + from concurrent.futures import ThreadPoolExecutor + rows = roster(); m = manifest(); want = m.get("version", "") + with ThreadPoolExecutor(16) as ex: res = list(ex.map(check_one, rows)) + reg = Registry.load() + for r in res: + r["manifest_version"] = want; wanted = (reg.get(r["iid"]) or {}).get("node_sha16_wanted") + r["behind"] = bool(wanted and r.get("bin_sha16") and r["bin_sha16"] != wanted) # only an explicit publish sets the want; the package version string is not a node version + if write: + with open(os.path.join(ROOT, "standing.jsonl"), "a") as f: + for r in res: f.write(json.dumps(r) + "\n") + return res +def update(label): + """The version follow: the manifest's hive package onto the box, the node pointer rewritten, the supervisor restarts it.""" + m = manifest(); b = Box.from_registry(label) + if not m: raise SshError("no manifest") + rc, out, err = b.run(f"set -e; cd {F}; curl -fsSL -m 600 -o pkg-{m['version']}.tgz https://dl.igneum.network{m['path']}; echo '{m['sha256']} pkg-{m['version']}.tgz' | sha256sum -c - >/dev/null; rm -rf /opt/igneum/pkg.new; mkdir -p /opt/igneum/pkg.new; tar -C /opt/igneum/pkg.new --strip-components=1 -xzf pkg-{m['version']}.tgz; rm -rf /opt/igneum/pkg.prev; mv /opt/igneum/pkg /opt/igneum/pkg.prev; mv /opt/igneum/pkg.new /opt/igneum/pkg; echo /opt/igneum/pkg/bin/igneumd > {F}/standing.node; pkill -9 -f '^/opt/igneum/pkg/bin/igneumd' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg.prev/bin/igneumd' 2>/dev/null; echo updated-to-{m['version']}", 900) + iid, _ = Registry.find(label); Registry.patch(iid, version_wanted=m["version"], updated_at=now()); return out.strip() +def rerent(label): + """Same shape again on the same provider; the old row is marked destroyed. Returns the new label or None.""" + import fleet + iid, b = Registry.find(label); n = int(b.get("rerents", 0)) + 1; new = f"{label.split('-r')[0]}-r{n}" + if b.get("provider") == "runpod": + import runpod; d = runpod.rent(b.get("gpu_type_id") or f"NVIDIA GeForce {b.get('card')}", new, count=1, disk=40) + if not d.get("id"): return None + reg = fleet.load(); reg[str(d["id"])] = {**{k: b[k] for k in ("card", "num_gpus", "provider", "phase", "wallet", "role", "standing") if k in b}, "label": new, "dph": d.get("costPerHr") or b.get("dph"), "state": "renting", "rented_at": now(), "standing_since": now(), "rerents": n, "rerent_of": label}; fleet.save(reg) + else: + nid = fleet.rent_card(b.get("card"), new, b.get("archs") or [], min_ram=b.get("vram_mb"), phase=b.get("phase", "2")) + if not nid: return None + Registry.patch(str(nid), standing=True, role=b.get("role", "live"), standing_since=now(), rerents=n, rerent_of=label) + Registry.patch(iid, state="destroyed", destroyed_at=now(), destroyed_reason="host died, re-rented as " + new) + return new +def loop(every=600): + dead = {} + while True: + try: + res = check() + for r in res: + if not r["alive"]: + dead[r["label"]] = dead.get(r["label"], 0) + 1 + if dead[r["label"]] >= 2: print(now(), "re-renting", r["label"], "->", rerent(r["label"]), flush=True); dead.pop(r["label"], None) + else: + dead.pop(r["label"], None) + if r["behind"]: print(now(), "behind:", r["label"], r.get("bin_sha16"), "wanted", (Registry.load().get(r["iid"]) or {}).get("node_sha16_wanted"), "(a publish script moves it; the loop only reports)", flush=True) + print(now(), "standing check:", len(res), "boxes,", sum(1 for r in res if r["alive"]), "alive,", sum(1 for r in res if r.get("synced")), "synced,", sum(1 for r in res if r["behind"]), "behind", flush=True) + except Exception as e: print(now(), "loop error", str(e)[:200], flush=True) + time.sleep(every) +if __name__ == "__main__": + a = sys.argv[1:] + if not a or a[0] == "roster": + for r in roster(): print(f"{r['label']:<12} {r['role']:<5} {str(r['card']):<22} {r['provider']:<7} USD {r['dph']:.3f}/h up {r['uptime_h']:.1f} h") + rs = roster(); print(f"{len(rs)} standing boxes, USD {sum(r['dph'] for r in rs):.2f}/h, USD {24*sum(r['dph'] for r in rs):.0f}/day") + elif a[0] == "install": print(install(a[1], *(a[2:3] or [None]))) + elif a[0] == "check": + for r in check(): print({k: r.get(k) for k in ("label", "role", "alive", "supervisor", "blocks", "synced", "exec_tip", "version", "behind", "prover")}) + elif a[0] == "update": print(update(a[1])) + elif a[0] == "rerent": print(rerent(a[1])) + elif a[0] == "loop": loop(int(a[1]) if len(a) > 1 else 600) diff --git a/tools/fleet/page.py b/tools/fleet/page.py index b51ab12a1..014580108 100755 --- a/tools/fleet/page.py +++ b/tools/fleet/page.py @@ -36,10 +36,14 @@ def build(): else: runpod += cost boxes.append({"id": iid, "label": b["label"], "card": b["card"], "vram_gb": round((b.get("vram_mb") or 0) / 1024), "provider": b.get("provider", "vast"), "rate_usd_h": b["dph"], "phase": b.get("phase", "1"), "state": "done" if b.get("state") == "destroyed" else b.get("state", "renting"), + "standing": bool(b.get("standing")), "role": b.get("role", ""), "standing_since": b.get("standing_since"), "uptime_h": round(hours(b.get("standing_since") or b["rented_at"]), 1) if b.get("standing") and b.get("state") != "destroyed" else None, "started_at": b["rented_at"], "ended_at": b.get("destroyed_at"), "hours": round(h, 2), "cost_usd": cost, "doing": (("done, destroyed: " + (b.get("doing") or "its measurement is in")) if b.get("state") == "destroyed" else b.get("doing", ""))}) running = [b for b in boxes if b["state"] not in ("done", "failed")] + stand = [b for b in running if b["standing"]] + standing_block = {"count": len(stand), "usd_h": round(sum(b["rate_usd_h"] for b in stand), 2), "usd_day": round(24 * sum(b["rate_usd_h"] for b in stand)), "roles": {r: sum(1 for b in stand if b["role"] == r) for r in sorted(set(b["role"] for b in stand))}, + "rule": "Standing boxes (Josh, 6 October 2026) stay up and are never destroyed on a job's end: 16 on the live devnet (mining, proving, voting) and 6 on Devnet 2, about USD 8/h; a dead host is re-rented in the same shape, the node runs under a supervisor with the recovery recipe, the version follows the live manifest by the update job, and no standing box joins an experiment (experiments use wave boxes); each should carry a mapped p2p port where the provider allows, so the live devnet stops being a star."} page = {"spend": {"vast_usd": round(vast, 2), "runpod_usd": round(runpod, 2), "cap_vast": 1000, "cap_runpod": 500, "updated_at": now(), "running": len(running), "running_note": open(os.path.join(ROOT, "why.txt")).read().strip() if os.path.exists(os.path.join(ROOT, "why.txt")) else ""}, - "phases": phases, "boxes": boxes, "results": list(res.values()), "night": night, "log": log[-200:], + "phases": phases, "standing": standing_block, "boxes": boxes, "results": list(res.values()), "night": night, "log": log[-200:], "devnet2": jl("devnet2-status.json", {"state": "building", "boxes": 0, "height": None, "last_gate": "none yet", "last_gate_at": None})} json.dump(page, open(OUT, "w"), indent=1); return page def publish(force=False): diff --git a/tools/fleet/publish-0315.py b/tools/fleet/publish-0315.py new file mode 100644 index 000000000..fb26ea989 --- /dev/null +++ b/tools/fleet/publish-0315.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +"""Publish 1 of 0.3.15 on the standing live-devnet boxes (the shipper's set, 6 October 2026): the Linux igneumd 06211d55 +(release-0.3.15-node 713ef876), igneum-miner e827d9db, the generator-4 workers (cuda 97e036e2, opencl ef9fbd03); the override +file stays the live thirteen-field object (digest b18ed271 on the new binary). Per box, through lib.box: the node binary to +/root/fleet/in/igneumd- (sha-checked), the miner and workers over /opt/igneum/pkg/bin (the old ones kept as *.0314), +/root/fleet/standing.node rewritten so the supervisor restarts the node on it (data dir kept), the miner killed once so +the supervisor's loop restarts it on the new miner and worker. Then the read-back table: version line, digest, worker sha. + publish-0315.py [labels...] default: every standing box with role live + publish-0315.py --readback [labels...] +""" +import sys, os, hashlib +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); from lib import Box, Registry, standing +from concurrent.futures import ThreadPoolExecutor +SHIP = "/Users/joshm/Projects/igneum-wt-ship0315" +NODE = (SHIP + "/vendor/igneum-node-0315/target-remote/release/igneumd", "06211d55b8cce1c19b31b218f5b61f42389a5ceaf68e0d27781b2e10e33b3f2a") +MINER = (SHIP + "/vendor/igneum-node-0315/target-remote/release/igneum-miner", "e827d9dbe8533ec8") +CUDA = (SHIP + "/infra/cross/out-workers/igneum-worker-cuda", "97e036e23f4ace66") +OCL = (SHIP + "/infra/cross/out-workers/igneum-worker-opencl", "ef9fbd03e7ce14d1") +def sha16(p): return hashlib.sha256(open(p, "rb").read()).hexdigest()[:16] +def publish(label): + b = Box.from_registry(label) + for p, s in (MINER, CUDA, OCL): + if sha16(p) != s: return label, f"local {os.path.basename(p)} sha {sha16(p)} is not {s}" + path = b.install_payload(NODE[0], NODE[1]) + b.put([MINER[0], CUDA[0], OCL[0]], "/root/fleet/in/") + rc, out, err = b.run(f"""set -e; cd /opt/igneum/pkg/bin; for f in igneum-miner igneum-worker-cuda igneum-worker-opencl; do [ -e $f.0314 ] || cp $f $f.0314; cp /root/fleet/in/$f $f.new && chmod +x $f.new && mv $f.new $f; done +echo {path} > /root/fleet/standing.node +pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve' 2>/dev/null; true +echo swapped""", 120) + return label, out.strip() or err[:120] +def readback(label): + b = Box.from_registry(label) + rc, out, err = b.run(r"""P=$(pgrep -af '^/(opt/igneum/pkg/bin|root/fleet/in)/igneumd[^ ]* ' | head -1 | awk '{print $2}'); echo bin=$P sha=$(sha256sum $P 2>/dev/null | cut -c1-16) version="$($P --version 2>&1 | head -1)" vline=$(grep -oE 'igneumd/2\.1\.0-[0-9a-f]+' /root/fleet/node.log | tail -1) digest=$(grep -o 'digest: [0-9a-f]*' /root/fleet/node.log | tail -1 | awk '{print substr($2,1,16)}') miner=$(sha256sum /opt/igneum/pkg/bin/igneum-miner | cut -c1-16) cuda=$(sha256sum /opt/igneum/pkg/bin/igneum-worker-cuda | cut -c1-16) ocl=$(sha256sum /opt/igneum/pkg/bin/igneum-worker-opencl | cut -c1-16) $(/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|synced=[a-z]+' | tr '\n' ' ') miner_up=$(pgrep -fc '^/opt/igneum/pkg/bin/igneum-miner mine') prover_up=$(pgrep -fc '^python3 -u /root/fleet/in/box-prover.py') rej=$(grep -c 'PoW rejected' /root/fleet/node.log)""", 60) + return label, out.strip() or err[:120] +if __name__ == "__main__": + a = sys.argv[1:]; rb = a and a[0] == "--readback"; a = a[1:] if rb else a + labels = a or [r["label"] for r in standing.roster() if r["role"] == "live"] + with ThreadPoolExecutor(13) as ex: + for l, o in ex.map(readback if rb else publish, labels): print(l, "::", o, flush=True) diff --git a/tools/fleet/rehearsal-1bps.json b/tools/fleet/rehearsal-1bps.json new file mode 100644 index 000000000..1fcb2883d --- /dev/null +++ b/tools/fleet/rehearsal-1bps.json @@ -0,0 +1 @@ +{"difficulty_v2_activation_daa":0,"proving_v0_activation_daa":0,"fees_v1_activation_daa":0,"finality_v3_activation_daa":0,"program_class_v3_activation_daa":0,"proving_v1_activation_daa":0,"proving_v1_segment_blocks":8,"proving_v1_unproven_daa":600,"proving_v1_aggregator_share_bps":1000,"proving_v1_fresh_rule_daa":0,"exec_restart_number":18446744073709551615,"exec_restart_hash":"","exec_restart_trust_daa":18446744073709551615,"pow_epoch_blocks":600,"pow_epoch_lead":100,"program_class_v4_activation_daa":2400,"program_class_v4_signal_window_daa":600} \ No newline at end of file diff --git a/tools/fleet/rehearsal-kill.sh b/tools/fleet/rehearsal-kill.sh new file mode 100644 index 000000000..d645d03b4 --- /dev/null +++ b/tools/fleet/rehearsal-kill.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Stops a rehearsal box's miner loops, miner, worker and sampler but NOT its dn400 node: run as a FILE (an inline pkill +# carrying these words kills the ssh shell that carries them). Then MINER_ONLY=1 bash in/box-rehearsal.sh restarts the miner. +pkill -9 -f '^bash in/box-rehearsal.sh'; pkill -9 -f '^/root/fleet/in/igneum-miner-v4 '; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve .*dn400' +sleep 1; echo "left=$(pgrep -c -f '^bash in/box-rehearsal.sh')+$(pgrep -c -f '^/root/fleet/in/igneum-miner-v4 ') node=$(pgrep -c -f '^/root/fleet/in/igneumd')" diff --git a/tools/fleet/runpod.py b/tools/fleet/runpod.py index 6299664d9..94ad9b141 100755 --- a/tools/fleet/runpod.py +++ b/tools/fleet/runpod.py @@ -27,9 +27,9 @@ def ledger(row): def gpus(): for g in call("GET", "/gputypes"): print(f"{g.get('id'):<34} {g.get('displayName','')!s:<22} {g.get('memoryInGb')}GB secure ${g.get('securePrice')}/h community ${g.get('communityPrice')}/h max {g.get('maxGpuCount')} avail {g.get('lowestPrice', {}) if isinstance(g.get('lowestPrice'), dict) else ''}") -def rent(gpu, label, count=8, disk=80, image=IMAGE, secure=False): +def rent(gpu, label, count=8, disk=80, image=IMAGE, secure=False, ports=("22/tcp",)): body = {"name": label, "imageName": image, "gpuTypeIds": [gpu], "gpuCount": count, "containerDiskInGb": disk, "volumeInGb": 0, - "cloudType": "SECURE" if secure else "COMMUNITY", "ports": ["22/tcp"], "env": {"PUBLIC_KEY": PUB}, "supportPublicIp": True, "computeType": "GPU"} + "cloudType": "SECURE" if secure else "COMMUNITY", "ports": list(ports), "env": {"PUBLIC_KEY": PUB}, "supportPublicIp": True, "computeType": "GPU"} d = call("POST", "/pods", body) row = {"t": now(), "event": "rent", "provider": "runpod", "instance": d.get("id"), "label": label, "gpu": gpu, "num_gpus": count, "dph": d.get("costPerHr"), "disk": disk} ledger(row); print(json.dumps(row)); print(json.dumps(d)[:600]); return d