From 45104c9bacac039a092196f014eea504b079bbfd Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:50:34 +0000 Subject: [PATCH] Devnet 2 (the staging chain): box-dn2.sh (igneum-devnet-2, own genesis bits, every activation low, no exec-restart fields), devnet2-override.json, devnet2-gate.sh (install, activation at tip + margin, N minutes, PASS only on zero rejects, reorg depth at most 3, equal exec roots, a paid segment, equal versions), the page's Devnet 2 panel; the wave, pool and snapshot scripts --- tools/fleet/box-dn2.sh | 32 ++++++++++ tools/fleet/box-exec-snapshot.sh | 34 ++++++++++ tools/fleet/box-matrix.sh | 2 +- tools/fleet/box-pool.sh | 24 +++++++ tools/fleet/box-prover.py | 5 +- tools/fleet/box-prover.sh | 2 +- tools/fleet/box-rig-prover.sh | 1 + tools/fleet/box-segal-host.sh | 19 ++++++ tools/fleet/box-setup.sh | 5 +- tools/fleet/box-wave.sh | 32 ++++++++++ tools/fleet/devnet2-gate.sh | 103 ++++++++++++++++++++++++++++++ tools/fleet/devnet2-override.json | 1 + tools/fleet/fleet.py | 3 + tools/fleet/night.py | 2 +- tools/fleet/page.py | 8 ++- 15 files changed, 263 insertions(+), 10 deletions(-) create mode 100755 tools/fleet/box-dn2.sh create mode 100755 tools/fleet/box-exec-snapshot.sh create mode 100755 tools/fleet/box-pool.sh create mode 100755 tools/fleet/box-segal-host.sh create mode 100755 tools/fleet/box-wave.sh create mode 100755 tools/fleet/devnet2-gate.sh create mode 100644 tools/fleet/devnet2-override.json diff --git a/tools/fleet/box-dn2.sh b/tools/fleet/box-dn2.sh new file mode 100755 index 00000000..291cb2e3 --- /dev/null +++ b/tools/fleet/box-dn2.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# A Devnet 2 box (6 October 2026, the project lead's standing structure: the rented fleet is the staging chain every release and +# activation crosses before the live devnet). The node runs igneum-devnet-2 (--devnet --devnet-suffix=2: own handshake +# magic, own data directory, a live-devnet peer refuses it at the handshake) with /root/fleet/dn2-override.json (its own +# genesis bits, every activation at a low DAA, NO exec-restart fields: a fresh chain executes from genesis), peered with +# the Devnet 2 seed; one miner on card 0 with its vote key; PROVER=1 adds the segment prover loop (box-prover.py with +# EXPORT_FROM=0 and the Devnet 2 chain name). NODE_BIN names the igneumd to run (the gate swaps it). +set -uo pipefail +F=/root/fleet; OUT=$F/out; mkdir -p $F/in $OUT $F/dn2 $F/mine/packs; exec >> $OUT/dn2.log 2>&1 +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +LABEL="${LABEL:-dn2}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}"; SEED="${SEED:-}"; NODE_BIN="${NODE_BIN:-$F/in/igneumd-0313}"; PROVER="${PROVER:-0}" +echo "RESULT dn2_start $(stamp) label=$LABEL node=$(sha256sum $NODE_BIN | cut -c1-16) seed=${SEED:-none} prover=$PROVER" +command -v curl >/dev/null || { apt-get update -qq >/dev/null 2>&1; apt-get install -y -qq curl ca-certificates python3 >/dev/null 2>&1; } +if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then + read -r PKG_PATH PKG_SHA PKG_VER <<< "$(curl -fsSL -m 30 https://dl.igneum.network/dl/public/igneum-downloads.json | python3 -c 'import sys,json; d=json.load(sys.stdin)["files"]["miner-hive"]; print(d["path"], d["sha256"], d["version"])')" + curl -fsSL -o $F/pkg.tgz "https://dl.igneum.network$PKG_PATH" && echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null && mkdir -p /opt/igneum/pkg && tar -C /opt/igneum/pkg --strip-components=1 -xzf $F/pkg.tgz || { echo "RESULT dn2_failed package"; exit 2; } +fi +B=/opt/igneum/pkg/bin; cp $F/in/dn2-override.json $F/dn2-override.json +pkill -f 'igneumd.*devnet-suffix=2' 2>/dev/null; sleep 2 +PEER=""; [ -n "$SEED" ] && PEER="--addpeer=$SEED" +IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host nohup $NODE_BIN --devnet --devnet-suffix=2 --appdir=$F/dn2 --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 $PEER --override-params-file=$F/dn2-override.json --nodnsseed --disable-upnp --nologfiles --yes ${NODE_EXTRA:-} >> $F/dn2-node.log 2>&1 & +sleep 10 +echo "RESULT dn2_node $(stamp) pid=$(pgrep -f 'devnet-suffix=2' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)" +for i in $(seq 1 30); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [ -n "$w" ] && break; sleep 5; done +echo "RESULT dn2_watch $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //')" +cd $F/mine && rm -rf packs/dn2 && $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/dn2 > $OUT/dn2-export-pack.log 2>&1 +( while :; do $B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/dn2" --prepare-packs packs/dn2-prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 30 >> $OUT/dn2-miner.log 2>&1; rc=$?; echo "RESULT dn2_miner_exit $(stamp) rc=$rc" >> $OUT/dn2.log; [ $rc = 42 ] && { rm -rf packs/dn2; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/dn2 >> $OUT/dn2-export-pack.log 2>&1; } || sleep 10; done ) & +echo "RESULT dn2_miner_started $(stamp)" +if [ "$PROVER" = 1 ] && [ -x /opt/igneum-floor/bin/igneum-prove-host ]; then + cd $F && LABEL="$LABEL" WALLET="$WALLET" EXPORT_FROM=0 CHAIN_NAME=igneum-devnet-2 MINER=none RUN_HOURS=48 setsid nohup python3 -u $F/in/box-prover.py > $OUT/dn2-prover-launch.log 2>&1 & + echo "RESULT dn2_prover_started $(stamp)" +fi diff --git a/tools/fleet/box-exec-snapshot.sh b/tools/fleet/box-exec-snapshot.sh new file mode 100755 index 00000000..d639c15b --- /dev/null +++ b/tools/fleet/box-exec-snapshot.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# The shipper's exec recovery (6 October 2026, 16:20Z): a box whose exec layer sits at tip 0 "from snapshot" (the genesis +# replay after the 15:43Z reorg wrote a tip-0 exec-snapshot.bin, which loads first and is served to peers) takes node 1's +# exported snapshot (tip chain block 130,272, sha256 ac101f13...): stop the node, delete the data dir's exec-snapshot.bin +# and .prev.bin, start the node as before plus --igneum-exec-snapshot=,0x; then eth_blockNumber climbs +# from 130,272 to the sink within a minute. SNAP = the file's path on this box. +set -uo pipefail +F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; FLOOR=/opt/igneum-floor; HOST=$FLOOR/bin/igneum-prove-host +SNAP="${SNAP:-$F/node1-copy-snapshot.bin}"; SNAP_SHA=ac101f13576179fd7d7f5e8ee902c9a7b6cc47730e3a3c069f389f0ca46d9221 +HUB_SSH="${HUB_SSH:-213.173.107.74}"; HUB_PORT="${HUB_PORT:-16515}"; HUB_PEER="${HUB_PEER:-213.173.107.74:16516}" +mkdir -p $OUT; exec >> $OUT/exec-snapshot.log 2>&1 +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +rpc() { curl -s -m 8 -X POST -H 'Content-Type: application/json' --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"$1\",\"params\":[]}" http://127.0.0.1:26790/; } +echo "RESULT snap_start $(stamp)" +[ -s "$SNAP" ] || scp -i $F/in/fleet-internal -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P "$HUB_PORT" "root@$HUB_SSH:/root/fleet/share/node1-copy-snapshot.bin" "$SNAP" || { echo "RESULT snap_failed pull"; exit 2; } +[ "$(sha256sum "$SNAP" | cut -c1-64)" = "$SNAP_SHA" ] || { echo "RESULT snap_failed sha256 $(sha256sum "$SNAP" | cut -c1-16)"; exit 2; } +chmod 644 "$SNAP" +bash $F/in/box-kill.sh >/dev/null 2>&1 +pkill -f igneumd-0313; pkill -x igneumd; sleep 4; pkill -9 -f igneumd-0313 2>/dev/null; sleep 1 +rm -f $F/node/igneum-devnet/datadir/evm/exec-snapshot.bin $F/node/igneum-devnet/datadir/evm/exec-snapshot.prev.bin +echo "RESULT snap_files_removed $(stamp) left=$(ls $F/node/igneum-devnet/datadir/evm/ 2>/dev/null | tr '\n' ' ')" +IGNEUM_PROOF_VERIFIER=$HOST nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 \ + --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes "--igneum-exec-snapshot=$SNAP,0x$SNAP_SHA" >> $F/node.log 2>&1 & +t1=$(date +%s); sleep 15 +echo "RESULT node_started $(stamp) pid=$(pgrep -f igneumd-0313 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') snapshot_line=\"$(grep -E 'igneum-exec\].*snapshot' $F/node.log | tail -1 | cut -c1-140)\"" +for i in $(seq 1 40); do + sleep 10; ex="$(rpc igneum_getExecStatus | python3 -c 'import sys,json; r=sys.stdin.read(); d=json.loads(r).get("result",{}) if r.strip() else {}; print(int(d.get("executedTip","0x0"),16), "blocked" if d.get("blocked") else "ok", str(d.get("startedFrom",""))[:30])' 2>/dev/null)" + w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'daa=[0-9]*' | tail -1)" + echo "RESULT climb $(stamp) s=$(( $(date +%s) - t1 )) exec=$ex $w" + n=$(printf '%s' "$ex" | awk '{print $1}'); st="$(rpc igneum_getProvingStatus | python3 -c 'import sys,json; r=sys.stdin.read(); d=json.loads(r).get("result",{}) if r.strip() else {}; print(int(d.get("tipDaa","0x0"),16), d.get("paidShards"))' 2>/dev/null)" + tip=$(printf '%s' "$st" | awk '{print $1}'); daa=${w#daa=} + if [ "${n:-0}" -gt 130272 ] && [ -n "$tip" ] && [ "$tip" -ge $(( ${daa:-0} - 20 )) ]; then echo "RESULT snap_done $(stamp) s=$(( $(date +%s) - t1 )) exec=$n tipDaa=$tip paidShards=$(printf '%s' "$st" | awk '{print $2}')"; exit 0; fi +done +echo "RESULT snap_failed no climb in 400 s: exec=$ex" diff --git a/tools/fleet/box-matrix.sh b/tools/fleet/box-matrix.sh index a5b7a972..ecdb3312 100755 --- a/tools/fleet/box-matrix.sh +++ b/tools/fleet/box-matrix.sh @@ -8,7 +8,7 @@ # line is "RESULT matrix_done" or "RESULT matrix_failed ". set -uo pipefail F=/root/fleet; OUT=$F/out; LOG=$OUT/matrix.log; B=/opt/igneum/pkg/bin; FLOOR=/opt/igneum-floor -for b in igneum-prove-host igneum-prove-export; do for d in $FLOOR/target/release $FLOOR/prove/proving/igneum-prove/target/release; do [ -x $d/$b ] && ln -sfn $d/$b $FLOOR/bin/$b; done; done +for b in igneum-prove-host igneum-prove-export; do for d in $FLOOR/target/release $FLOOR/prove/proving/igneum-prove/target/release /opt/igneum-segal/proving/igneum-prove/target/release; do [ -x $d/$b ] && ln -sfn $d/$b $FLOOR/bin/$b; done; done # the last one that exists wins: the segment host (--mode chain) over the floor host HOST=$FLOOR/bin/igneum-prove-host; FIX=$FLOOR/prove/proving/fixtures V1=$FIX/fees-v1-shards2.json; EMPTY=$FIX/block-72854-empty-block-first.json LABEL="${LABEL:-box}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}" diff --git a/tools/fleet/box-pool.sh b/tools/fleet/box-pool.sh new file mode 100755 index 00000000..6eed92b1 --- /dev/null +++ b/tools/fleet/box-pool.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# The pool host (6 October 2026): the 0.3.13 node on the thirteen-field file peered with the hub and the seed, then +# pool-v0 (igneum-pool, built on the hub) against it on 0.0.0.0:4463 with a throwaway coinbase address, dry-run off, +# payouts at the default minimum. RESULT lines in /root/fleet/out/pool.log; the pool's own log in pool-run.log. +set -uo pipefail +F=/root/fleet; OUT=$F/out; mkdir -p $F/in $OUT $F/node $F/pool; exec >> $OUT/pool.log 2>&1 +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +HUB_PEER="${HUB_PEER:-213.173.107.74:16516}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}" +echo "RESULT pool_host_start $(stamp)" +command -v curl >/dev/null || { apt-get update -qq >/dev/null 2>&1; apt-get install -y -qq curl ca-certificates python3 >/dev/null 2>&1; } +if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then + read -r PKG_PATH PKG_SHA PKG_VER <<< "$(curl -fsSL -m 30 https://dl.igneum.network/dl/public/igneum-downloads.json | python3 -c 'import sys,json; d=json.load(sys.stdin)["files"]["miner-hive"]; print(d["path"], d["sha256"], d["version"])')" + curl -fsSL -o $F/pkg.tgz "https://dl.igneum.network$PKG_PATH" && echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null && mkdir -p /opt/igneum/pkg && tar -C /opt/igneum/pkg --strip-components=1 -xzf $F/pkg.tgz || { echo "RESULT pool_host_failed package"; exit 2; } +fi +B=/opt/igneum/pkg/bin; cp $F/in/igneumd-0313 $B/igneumd-0313; chmod +x $B/igneumd-0313; cp $F/in/ov13.json $F/override.json +pgrep -f igneumd-0313 >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & +sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}')" +for i in $(seq 1 90); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [[ "$w" == *synced=true* ]] && break; sleep 10; done +echo "RESULT synced $(stamp) after $((i*10)) s" +for i in $(seq 1 120); do [ -x $F/in/igneum-pool ] && break; sleep 15; done +[ -x $F/in/igneum-pool ] || { echo "RESULT pool_host_failed no pool binary after 30 min"; exit 2; } +cp $F/in/igneum-pool $F/pool/; cd $F/pool +echo "RESULT pool_help $($F/pool/igneum-pool --help 2>&1 | tr '\n' ' ' | cut -c1-600)" +echo "RESULT pool_binary_ready $(stamp)" diff --git a/tools/fleet/box-prover.py b/tools/fleet/box-prover.py index 29d39980..ffe638fb 100644 --- a/tools/fleet/box-prover.py +++ b/tools/fleet/box-prover.py @@ -24,7 +24,7 @@ import json, os, sys, time, subprocess, datetime, binascii, urllib.request, sign # a rig runs one loop per card under /root/fleet/card/ (FLEET_CARD) F = "/root/fleet"; CARD = os.environ.get("FLEET_CARD"); OUT = f"{F}/card{CARD}/out" if CARD else f"{F}/out"; B = "/opt/igneum/pkg/bin"; FLOOR = "/opt/igneum-floor" HOST = f"{FLOOR}/bin/igneum-prove-host"; EXPORT = f"{FLOOR}/bin/igneum-prove-export" -EVM = "http://127.0.0.1:26790"; GRPC = "grpc://127.0.0.1:26610"; CHAIN = "igneum-devnet" +EVM = "http://127.0.0.1:26790"; GRPC = "grpc://127.0.0.1:26610"; CHAIN = os.environ.get("CHAIN_NAME", "igneum-devnet") # Devnet 2 signs for igneum-devnet-2 LABEL = os.environ.get("LABEL", "box"); WALLET = os.environ.get("WALLET", "0x" + "19" * 20) THRESHOLD = os.environ.get("THRESHOLD", ""); MINER = os.environ.get("MINER", "keep"); RUN_HOURS = float(os.environ.get("RUN_HOURS", "9")) EXPORT_FROM = int(os.environ.get("EXPORT_FROM", "27276")) # the devnet's exec restart block (ov13.json exec_restart_number) when the node does not report one @@ -54,6 +54,7 @@ def kill_server(): MPROC = None def miner_start(): global MPROC + if MINER == "none": return # the box's own miner loop runs outside (Devnet 2 boxes) if MPROC and MPROC.poll() is None: return subprocess.run(f"cd {MINE} && rm -rf packs/devnet && {B}/igneum-miner export-pack {GRPC} packs/devnet > {OUT}/prover-export-pack.log 2>&1", shell=True) MPROC = subprocess.Popen([f"{B}/igneum-miner", "mine", GRPC, "1", "100000000", LABEL, "--worker", f"{B}/igneum-worker-cuda", "--worker-args", f"--device {DEV} --pack packs/devnet", @@ -122,7 +123,7 @@ def candidates(st): return segs, start, n, tip, len(work) while (time.time() - t_run0) / 3600 < RUN_HOURS: state["passes"] += 1; p = state["passes"] - if MPROC and MPROC.poll() is not None and MINER == "keep": say(f"RESULT miner_exit {stamp()} rc={MPROC.returncode}; restarting"); MPROC = None; miner_start() + if MINER != "none" and MPROC and MPROC.poll() is not None and MINER == "keep": say(f"RESULT miner_exit {stamp()} rc={MPROC.returncode}; restarting"); MPROC = None; miner_start() st = rpc("igneum_getProvingStatus", []) if not st or not st.get("v1") or not st["v1"].get("active"): say(f"RESULT pass {p} {stamp()} v1 not active or no status"); time.sleep(20); continue tip = hexi(st["tipDaa"]) diff --git a/tools/fleet/box-prover.sh b/tools/fleet/box-prover.sh index 43e56971..323858de 100755 --- a/tools/fleet/box-prover.sh +++ b/tools/fleet/box-prover.sh @@ -3,7 +3,7 @@ # carry them, the proving agent's note of 6 October 2026), waits for it to be synced again (the data dir is kept, so # seconds), then runs box-prover.py with the card's profile: THRESHOLD and MINER from the card's memory unless given. set -uo pipefail -F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; FLOOR=/opt/igneum-floor; for b in igneum-prove-host igneum-prove-export; do for d in $FLOOR/target/release $FLOOR/prove/proving/igneum-prove/target/release; do [ -x $d/$b ] && ln -sfn $d/$b $FLOOR/bin/$b; done; done +F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; FLOOR=/opt/igneum-floor; for b in igneum-prove-host igneum-prove-export; do for d in $FLOOR/target/release $FLOOR/prove/proving/igneum-prove/target/release /opt/igneum-segal/proving/igneum-prove/target/release; do [ -x $d/$b ] && ln -sfn $d/$b $FLOOR/bin/$b; done; done # the last one that exists wins: the segment host (--mode chain) over the floor host HOST=$FLOOR/bin/igneum-prove-host mkdir -p $OUT; exec >> $OUT/prover-launch.log 2>&1 stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } diff --git a/tools/fleet/box-rig-prover.sh b/tools/fleet/box-rig-prover.sh index 30da8891..10e6277c 100755 --- a/tools/fleet/box-rig-prover.sh +++ b/tools/fleet/box-rig-prover.sh @@ -6,6 +6,7 @@ F=/root/fleet; N=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l) LABEL="${LABEL:-rig}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}" for d in $(seq 0 $((N-1))); do mkdir -p $F/card$d/out $F/card$d/mine/packs + sleep 20 # the SDK starts one sp1-gpu-server per host and waits a fixed time for its socket; eight at once overran it (15:18Z) ( cd $F/card$d && LABEL="$LABEL-gpu$d" WALLET="$WALLET" THRESHOLD="${THRESHOLD:-}" MINER="${MINER:-keep}" RUN_HOURS="${RUN_HOURS:-9}" IGNEUM_CUDA_DEVICE=$d FLEET_CARD=$d setsid nohup python3 -u $F/in/box-prover.py $F/card$d/out/launch.log 2>&1 & ) done echo "RESULT rig_provers_started $(date -u +%Y-%m-%dT%H:%M:%SZ) cards=$N" diff --git a/tools/fleet/box-segal-host.sh b/tools/fleet/box-segal-host.sh new file mode 100755 index 00000000..17f9d92e --- /dev/null +++ b/tools/fleet/box-segal-host.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Builds the segment-aligned prover host (the proving-v1 bundle igneum-prove-wsl2-segal.zip: --mode chain, --save-shards, +# --prev) with the cuda feature into /opt/igneum-segal, and points /opt/igneum-floor/bin/igneum-prove-host at it. The +# prover-floor bundle's host (08:28Z) predates --mode chain (16:20Z, the fleet's chain runs failed on its usage line). +set -uo pipefail +F=/root/fleet; OUT=$F/out; S=/opt/igneum-segal; exec >> $OUT/segal-build.log 2>&1 +export PATH="$HOME/.cargo/bin:$PATH" RUSTUP_TOOLCHAIN=stable; unset CARGO_TARGET_DIR +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +echo "RESULT segal_start $(stamp)" +rm -rf $S && mkdir -p $S && unzip -q -o $F/in/igneum-prove-wsl2-segal.zip -d $F/unz-segal && cp -r $F/unz-segal/igneum-prove-wsl2/package/. $S/ && find $S -type f -exec touch {} + +# the warm target of the floor host saves most of the build (the same crates) +[ -d /opt/igneum-floor/prove/proving/igneum-prove/target ] && cp -r /opt/igneum-floor/prove/proving/igneum-prove/target $S/proving/igneum-prove/target 2>/dev/null +cd $S/proving/igneum-prove && t0=$(date +%s) +cargo build --release -p igneum-prove-export -p igneum-prove-host --features igneum-prove-host/cuda -j $(( $(nproc) > 16 ? 16 : $(nproc) )) > $OUT/segal-cargo.log 2>&1; rc=$? +echo "RESULT segal_build_exit $rc time_s=$(( $(date +%s) - t0 ))" +[ $rc -eq 0 ] || { grep -n -A6 '^error' $OUT/segal-cargo.log | head -30; echo "RESULT segal_failed"; exit 2; } +ln -sfn $S/proving/igneum-prove/target/release/igneum-prove-host /opt/igneum-floor/bin/igneum-prove-host +ln -sfn $S/proving/igneum-prove/target/release/igneum-prove-export /opt/igneum-floor/bin/igneum-prove-export +echo "RESULT segal_done $(stamp) host=$(sha256sum /opt/igneum-floor/bin/igneum-prove-host | cut -c1-16) ids=$(/opt/igneum-floor/bin/igneum-prove-host --mode id 2>&1 | grep -o '0x[0-9a-f]*' | head -2 | tr '\n' ' ') chain_mode=$(/opt/igneum-floor/bin/igneum-prove-host 2>&1 | grep -c chain)" diff --git a/tools/fleet/box-setup.sh b/tools/fleet/box-setup.sh index 7fcdfd46..634c3423 100755 --- a/tools/fleet/box-setup.sh +++ b/tools/fleet/box-setup.sh @@ -46,7 +46,8 @@ fi B=/opt/igneum/pkg/bin echo "RESULT package igneumd=$(sha256sum $B/igneumd | cut -c1-16) miner=$(sha256sum $B/igneum-miner | cut -c1-16) cuda_worker=$(sha256sum $B/igneum-worker-cuda | cut -c1-16) version=$($B/igneumd --version 2>&1 | head -1)" cp $IN/override.json $F/override.json -if ! pgrep -x igneumd >/dev/null; then +if [ "${NO_NODE:-0}" = 1 ]; then echo "RESULT node skipped (NO_NODE=1: a Devnet 2 box runs its own chain's node)" +elif ! pgrep -x igneumd >/dev/null; then say "node" mkdir -p $F/node nohup $B/igneumd --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 \ @@ -109,5 +110,5 @@ HOST=$H/proving/igneum-prove/target/release/igneum-prove-host; EXPORT=$H/proving ln -sfn $HOST /opt/igneum-floor/bin/igneum-prove-host; ln -sfn $EXPORT /opt/igneum-floor/bin/igneum-prove-export echo "RESULT host sha256=$(sha256sum $HOST | cut -c1-16) export=$(sha256sum $EXPORT | cut -c1-16) ids=$($HOST --mode id 2>&1 | grep -o '0x[0-9a-f]*' | tr '\n' ' ')" echo "RESULT fixtures $(ls $H/proving/fixtures | wc -l) files, v1=$(sha256sum $H/proving/fixtures/fees-v1-shards2.json | cut -c1-16) empty=$(sha256sum $H/proving/fixtures/block-72854-empty-block-first.json | cut -c1-16)" -echo "RESULT node_now $($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)" +[ "${NO_NODE:-0}" = 1 ] || echo "RESULT node_now $($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)" echo "RESULT setup_done $(stamp)" diff --git a/tools/fleet/box-wave.sh b/tools/fleet/box-wave.sh new file mode 100755 index 00000000..6b86bcb8 --- /dev/null +++ b/tools/fleet/box-wave.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# A wave miner (6 October 2026): the 0.3.12 Linux package's workers with the 0.3.13 node (igneumd-0313, pushed) on the +# thirteen-field override, peered with the fleet hub and the seed, one igneum-miner with its vote key on (the default); +# no builds, no prover. Prints RESULT lines to /root/fleet/out/wave.log: node digest, sync, the miner's rate. +set -uo pipefail +export DEBIAN_FRONTEND=noninteractive +F=/root/fleet; OUT=$F/out; mkdir -p $F/in $OUT $F/mine/packs $F/node; exec >> $OUT/wave.log 2>&1 +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +LABEL="${LABEL:-wave}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}"; HUB_PEER="${HUB_PEER:-213.173.107.74:16516}" +echo "RESULT wave_start $(stamp) label=$LABEL gpu=$(nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader | head -1)" +command -v curl >/dev/null || { apt-get update -qq >/dev/null 2>&1; apt-get install -y -qq curl ca-certificates python3 >/dev/null 2>&1; } +if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then + # the current hive package from the public index (0.3.12's file left the host when 0.3.13 was published, 16:17Z) + read -r PKG_PATH PKG_SHA PKG_VER <<< "$(curl -fsSL -m 30 https://dl.igneum.network/dl/public/igneum-downloads.json | python3 -c 'import sys,json; d=json.load(sys.stdin)["files"]["miner-hive"]; print(d["path"], d["sha256"], d["version"])')" + curl -fsSL -o $F/pkg.tgz "https://dl.igneum.network$PKG_PATH" || { echo "RESULT wave_failed package $PKG_PATH"; exit 2; } + echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null || { echo "RESULT wave_failed sha256"; exit 2; } + echo "RESULT package $(stamp) $PKG_VER $PKG_PATH ${PKG_SHA:0:16}" + mkdir -p /opt/igneum/pkg && tar -C /opt/igneum/pkg --strip-components=1 -xzf $F/pkg.tgz +fi +B=/opt/igneum/pkg/bin +[ "$(sha256sum $F/in/igneumd-0313 | cut -c1-16)" = d6350586fe837b1f ] || { echo "RESULT wave_failed node binary"; exit 2; } +cp $F/in/igneumd-0313 $B/igneumd-0313; chmod +x $B/igneumd-0313; cp $F/in/ov13.json $F/override.json +ldconfig -p | grep -q libnvrtc.so.12 || echo "RESULT note no libnvrtc.so.12 on the path (the worker dlopens it)" +pgrep -f igneumd-0313 >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 & +sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') pid=$(pgrep -f igneumd-0313 | head -1)" +for i in $(seq 1 90); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [[ "$w" == *synced=true* ]] && break; sleep 10; done +echo "RESULT synced $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //') after $((i*10)) s" +cd $F/mine && rm -rf packs/devnet && $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet > $OUT/export-pack.log 2>&1 +while :; do + $B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/devnet" --prepare-packs packs/prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 30 >> $OUT/miner.log 2>&1; rc=$? + echo "RESULT miner_exit $(stamp) rc=$rc"; [ $rc = 42 ] && { rm -rf packs/devnet; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >> $OUT/export-pack.log 2>&1; } || sleep 10 +done diff --git a/tools/fleet/devnet2-gate.sh b/tools/fleet/devnet2-gate.sh new file mode 100755 index 00000000..f07eb149 --- /dev/null +++ b/tools/fleet/devnet2-gate.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# The Devnet 2 release gate (6 October 2026). Given a release's Linux igneumd (a download URL or a local file) and its +# manifest (for the activation field and value, or --activation name=value), it: (1) installs the binary on every +# Devnet 2 box and restarts each node on the CURRENT override, checking the version; (2) sets the activation at tip + MARGIN +# in a new override, restarts every box on it (the seed first) and waits for the crossing; (3) runs N minutes past it; +# (4) prints PASS only when: zero rejected blocks on every box, no selected-chain reorg over depth 3, exec state roots +# equal on every box at a common height, at least one segment record paid since the start, every node's version equal +# to the release's. FAIL names the box and the line. The one-line status goes to ~/Desktop/fleet/devnet2-status.json. +# +# devnet2-gate.sh --binary --sha256 [--activation proving_v1_fresh_rule_daa=NNN | --activation-field F] +# [--margin 600] [--minutes 20] [--dry-run] (dry-run: steps 1 and 4 against the current object, no activation) +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$HOME/Desktop/fleet" +BIN=""; SHA=""; ACT=""; ACT_FIELD=""; MARGIN=600; MINUTES=20; DRY=0 +while [[ $# -gt 0 ]]; do case "$1" in + --binary) BIN="$2"; shift 2 ;; --sha256) SHA="$2"; shift 2 ;; --activation) ACT="$2"; shift 2 ;; --activation-field) ACT_FIELD="$2"; shift 2 ;; + --margin) MARGIN="$2"; shift 2 ;; --minutes) MINUTES="$2"; shift 2 ;; --dry-run) DRY=1; shift ;; *) echo "unknown $1"; exit 2 ;; esac; done +[[ -n "$BIN" && -n "$SHA" ]] || { echo "usage: $0 --binary --sha256 [--activation name=value] [--minutes N] [--dry-run]"; exit 2; } +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +LOG="$ROOT/devnet2-gate-$(date -u +%Y%m%dT%H%M%SZ).log"; exec > >(tee -a "$LOG") 2>&1 +echo "GATE start $(stamp) binary=$BIN sha256=${SHA:0:16} activation=${ACT:-none} minutes=$MINUTES dry_run=$DRY" +fail() { echo "FAIL $(stamp) $*"; python3 - "$ROOT" "FAIL: $*" <<'PY' +import json, sys, os, datetime; p=os.path.join(sys.argv[1], "devnet2-status.json"); d=json.load(open(p)) if os.path.exists(p) else {} +d.update({"last_gate": sys.argv[2][:200], "last_gate_at": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")}); json.dump(d, open(p, "w")) +PY +exit 1; } +# the binary, local +LOCAL="$ROOT/dn2-gate-igneumd" +if [[ "$BIN" == http* ]]; then curl -fsSL -o "$LOCAL" "$BIN" || fail "download $BIN"; else cp "$BIN" "$LOCAL"; fi +[[ "$(shasum -a 256 "$LOCAL" | cut -c1-64)" == "$SHA" ]] || fail "sha256 of the binary is $(shasum -a 256 "$LOCAL" | cut -c1-16), not ${SHA:0:16}" +VERSION_WANT="$(python3 "$HERE/fleet.py" dn2-version "$LOCAL" 2>/dev/null || true)" +# the boxes +mapfile -t BOXES < <(python3 - <<'PY' +import json, os; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json"))) +for iid,b in reg.items(): + if b.get("devnet2") and b.get("state")!="destroyed" and b.get("ssh_host"): print(f"{b['label']} {b['ssh_host']} {b['ssh_port']} {int(bool(b.get('dn2_seed')))} {b.get('wallet')} {int(bool(b.get('dn2_prover')))}") +PY +) +[[ ${#BOXES[@]} -gt 0 ]] || fail "no Devnet 2 boxes in the registry" +SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } +SCP() { scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P "$2" "${@:3}" "root@$1:/root/fleet/in/" >/dev/null; } +seed_line="$(printf '%s\n' "${BOXES[@]}" | awk '$4==1 {print; exit}')"; SEED_HOST="$(awk '{print $2}' <<< "$seed_line")"; SEED_PORT="$(awk '{print $3}' <<< "$seed_line")" +SEED_PEER="$(python3 - <<'PY' +import json, os; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json"))) +print(next((b.get("dn2_peer","") for b in reg.values() if b.get("dn2_seed") and b.get("state")!="destroyed"), "")) +PY +)" +echo "GATE boxes=${#BOXES[@]} seed=$SEED_HOST:$SEED_PORT peer=$SEED_PEER" +# a reading of every box: height, exec tip and root at a height, rejects, max reorg, version, paid segments +read_box() { # host port -> "height daa exec_tip paid_seg version rejects max_reorg" + SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); h=$(grep -o "blocks=[0-9]*" <<< "$w" | cut -d= -f2); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); v=$(pgrep -fa "devnet-suffix=2" | head -1 | awk "{print \$2}"); ver=$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "reject message|PoW rejected|block rejected|invalid block" /root/fleet/dn2-node.log 2>/dev/null); mr=$(grep -oE "selected-chain reorg: [0-9]+ chain blocks" /root/fleet/dn2-node.log 2>/dev/null | grep -oE "[0-9]+ chain" | awk "{if (\$1>m) m=\$1} END {print m+0}"); echo "${h:-0} ${d:-0} ${e:-0 0} ${ver:-none} ${rej:-0} ${mr:-0}"' 2>/dev/null +} +root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } +echo "GATE baseline $(stamp)"; declare -A PAID0 +for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; PAID0[$1]="$(awk '{print $4}' <<< "$r")"; done +# 1. install the binary on every box, restart on the current override +for l in "${BOXES[@]}"; do set -- $l + SCP "$2" "$3" "$LOCAL" || fail "$1: scp of the binary" + SSH "$2" "$3" "mv /root/fleet/in/dn2-gate-igneumd /root/fleet/in/igneumd-gate && chmod +x /root/fleet/in/igneumd-gate && [ \"\$(sha256sum /root/fleet/in/igneumd-gate | cut -c1-64)\" = $SHA" || fail "$1: the binary's sha256 on the box" +done +restart_all() { # with the override file already on each box as /root/fleet/in/dn2-override.json + for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] || continue + SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f 'igneumd.*devnet-suffix=2'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" + sleep 15; done + for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] && continue + SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f 'igneumd.*devnet-suffix=2'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 SEED=$SEED_PEER NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh /dev/null 2>&1 & echo restarted" || fail "$1: restart" + done +} +echo "GATE step1 $(stamp) restart on the release binary, the current override"; restart_all; sleep 60 +for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r"; v="$(awk '{print $5}' <<< "$r")"; [[ -z "$VERSION_WANT" || "$v" == "$VERSION_WANT" ]] || fail "$1: version $v, want $VERSION_WANT"; done +# 2. the activation +if [[ $DRY == 0 && -n "$ACT" ]]; then + tip="$(read_box "$SEED_HOST" "$SEED_PORT" | awk '{print $2}')"; name="${ACT%%=*}"; val="${ACT#*=}"; [[ "$val" == auto ]] && val=$((tip + MARGIN)) + python3 - "$HERE/devnet2-override.json" "$name" "$val" "$ROOT/dn2-override-next.json" <<'PY' +import json, sys; d=json.load(open(sys.argv[1])); d[sys.argv[2]]=int(sys.argv[3]); json.dump(d, open(sys.argv[4], "w")) +PY + for l in "${BOXES[@]}"; do set -- $l; SCP "$2" "$3" "$ROOT/dn2-override-next.json" && SSH "$2" "$3" "mv /root/fleet/in/dn2-override-next.json /root/fleet/in/dn2-override.json" || fail "$1: override push"; done + echo "GATE step2 $(stamp) activation $name=$val at tip $tip (margin $MARGIN); restart on the new object"; restart_all + for i in $(seq 1 120); do sleep 15; d="$(read_box "$SEED_HOST" "$SEED_PORT" | awk '{print $2}')"; [[ "${d:-0}" -ge "$val" ]] && { echo "GATE crossed $name=$val at $(stamp) (daa $d)"; break; }; done +fi +# 3. run +echo "GATE step3 $(stamp) running $MINUTES min"; sleep $((MINUTES * 60)) +# 4. the checks +echo "GATE step4 $(stamp) checks"; H=""; declare -A ROOTS; ok=1; paid_any=0 +for l in "${BOXES[@]}"; do set -- $l; r="$(read_box "$2" "$3")"; echo " $1: $r" + rej="$(awk '{print $6}' <<< "$r")"; mr="$(awk '{print $7}' <<< "$r")"; v="$(awk '{print $5}' <<< "$r")"; paid="$(awk '{print $4}' <<< "$r")"; et="$(awk '{print $3}' <<< "$r")" + [[ "$rej" == 0 ]] || { echo "FAIL $1: $rej rejected blocks (grep 'reject' /root/fleet/dn2-node.log)"; ok=0; } + [[ "${mr:-0}" -le 3 ]] || { echo "FAIL $1: a selected-chain reorg of depth $mr"; ok=0; } + [[ -z "$VERSION_WANT" || "$v" == "$VERSION_WANT" ]] || { echo "FAIL $1: version $v"; ok=0; } + [[ "${paid:-0}" -gt "${PAID0[$1]:-0}" ]] && paid_any=1 + [[ -z "$H" || "$et" -lt "$H" ]] && H="$et" +done +H=$((H > 20 ? H - 20 : 1)); HX="$(printf '0x%x' "$H")" +for l in "${BOXES[@]}"; do set -- $l; ROOTS[$1]="$(root_at "$2" "$3" "$HX")"; echo " $1 root@$H ${ROOTS[$1]:0:18}"; done +first=""; for k in "${!ROOTS[@]}"; do [[ -z "$first" ]] && first="${ROOTS[$k]}"; [[ "${ROOTS[$k]}" == "$first" ]] || { echo "FAIL $k: exec root at height $H ${ROOTS[$k]:0:18} differs from ${first:0:18}"; ok=0; }; done +[[ $paid_any == 1 ]] || { echo "FAIL no segment record paid on any box during the run"; ok=0; } +res="FAIL"; [[ $ok == 1 ]] && res="PASS" +echo "$res $(stamp) release ${SHA:0:16} activation ${ACT:-none} boxes ${#BOXES[@]} minutes $MINUTES log $LOG" +python3 - "$ROOT" "$res: ${SHA:0:16} ${ACT:-current object}, ${#BOXES[@]} boxes, $MINUTES min" <<'PY' +import json, sys, os, datetime; p=os.path.join(sys.argv[1], "devnet2-status.json"); d=json.load(open(p)) if os.path.exists(p) else {} +d.update({"last_gate": sys.argv[2], "last_gate_at": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")}); json.dump(d, open(p, "w")) +PY +[[ $ok == 1 ]] diff --git a/tools/fleet/devnet2-override.json b/tools/fleet/devnet2-override.json new file mode 100644 index 00000000..1968345c --- /dev/null +++ b/tools/fleet/devnet2-override.json @@ -0,0 +1 @@ +{"genesis_bits":505413632,"difficulty_v2_activation_daa":600,"proving_v0_activation_daa":900,"fees_v1_activation_daa":1200,"finality_v3_activation_daa":1500,"program_class_v3_activation_daa":1800,"proving_v1_activation_daa":1800,"proving_v1_segment_blocks":8,"proving_v1_unproven_daa":600,"proving_v1_aggregator_share_bps":1000,"proving_v1_fresh_rule_daa":2400} diff --git a/tools/fleet/fleet.py b/tools/fleet/fleet.py index 23281710..a0d54b52 100755 --- a/tools/fleet/fleet.py +++ b/tools/fleet/fleet.py @@ -180,5 +180,8 @@ if __name__ == "__main__": b = [v for v in boxes([a[1]]).values()][0]; f = a[2] if len(a) > 2 else "/root/fleet/setup.log"; print(ssh(b, f"tail -n 30 {f}")[1]) elif c == "sh": b = [v for v in boxes([a[1]]).values()][0]; rc, out, err = ssh(b, a[2], timeout=600); print(out, err) + elif c == "dn2-version": # the version string a Linux igneumd prints, read from its strings (the gate compares every box to it) + import re as _re + data = open(a[1], "rb").read(); m = _re.search(rb"igneumd/2\.1\.0-[0-9a-f]{8}", data); print(m.group(0).decode().replace("igneumd/", "igneumd_") if m else "") elif c == "list": for iid, b in load().items(): print(iid, b["label"], b["card"], b.get("state"), f"${b['dph']:.3f}/h", b.get("ssh_host"), b.get("ssh_port")) diff --git a/tools/fleet/night.py b/tools/fleet/night.py index 7a03f4da..bb1a29c0 100644 --- a/tools/fleet/night.py +++ b/tools/fleet/night.py @@ -49,7 +49,7 @@ while True: if time.time() - zero_since[iid] > 150: fleet.ssh(b, f"cd /root/fleet && bash in/box-kill.sh >/dev/null 2>&1; mv out/node-swap.log out/node-swap-reset-$(date +%s).log 2>/dev/null; STEP=file NODE_SHA256={SHA} EXPECT_DIGEST={DIG} HUB_PEER={hub['hub_peer']} HUB_SSH={hub['ssh_host']} HUB_PORT={hub['ssh_port']} setsid nohup in/box-node-swap.sh /dev/null 2>&1 &", timeout=60) log(f"{b['label']}: exec at 0 for 150 s after a completed replay; exec restart re-run"); zero_since.pop(iid, None); launched.discard(iid) - if at_tip and iid not in launched and d["provers"] == 0: + if at_tip and iid not in launched and d["provers"] == 0 and b.get("stage") != "mine-only": if (b.get("num_gpus") or 1) > 1: fleet.ssh(b, f"cd /root/fleet && LABEL={b['label']} WALLET={b['wallet']} setsid nohup in/box-rig-prover.sh out/rig-provers.log 2>&1 &", timeout=60) else: fleet.ssh(b, f"cd /root/fleet && mv out/prover.log out/prover-prev-$(date +%s).log 2>/dev/null; LABEL={b['label']} WALLET={b['wallet']} HUB_PEER={hub['hub_peer']} setsid nohup in/box-prover.sh /dev/null 2>&1 &", timeout=60) launched.add(iid); fleet.patch(iid, stage="prover", state="running", doing=f"phase 2 prover on the executed tip ({d['exec']})"); log(f"{b['label']}: replay at the tip (exec {d['exec']}), prover started") diff --git a/tools/fleet/page.py b/tools/fleet/page.py index 4be2d4c3..b51ab12a 100755 --- a/tools/fleet/page.py +++ b/tools/fleet/page.py @@ -36,9 +36,11 @@ def build(): else: runpod += cost boxes.append({"id": iid, "label": b["label"], "card": b["card"], "vram_gb": round((b.get("vram_mb") or 0) / 1024), "provider": b.get("provider", "vast"), "rate_usd_h": b["dph"], "phase": b.get("phase", "1"), "state": "done" if b.get("state") == "destroyed" else b.get("state", "renting"), - "started_at": b["rented_at"], "ended_at": b.get("destroyed_at"), "hours": round(h, 2), "cost_usd": cost, "doing": b.get("doing", "")}) - page = {"spend": {"vast_usd": round(vast, 2), "runpod_usd": round(runpod, 2), "cap_vast": 900, "cap_runpod": 450, "updated_at": now()}, - "phases": phases, "boxes": boxes, "results": list(res.values()), "night": night, "log": log[-200:]} + "started_at": b["rented_at"], "ended_at": b.get("destroyed_at"), "hours": round(h, 2), "cost_usd": cost, "doing": (("done, destroyed: " + (b.get("doing") or "its measurement is in")) if b.get("state") == "destroyed" else b.get("doing", ""))}) + running = [b for b in boxes if b["state"] not in ("done", "failed")] + page = {"spend": {"vast_usd": round(vast, 2), "runpod_usd": round(runpod, 2), "cap_vast": 1000, "cap_runpod": 500, "updated_at": now(), "running": len(running), "running_note": open(os.path.join(ROOT, "why.txt")).read().strip() if os.path.exists(os.path.join(ROOT, "why.txt")) else ""}, + "phases": phases, "boxes": boxes, "results": list(res.values()), "night": night, "log": log[-200:], + "devnet2": jl("devnet2-status.json", {"state": "building", "boxes": 0, "height": None, "last_gate": "none yet", "last_gate_at": None})} json.dump(page, open(OUT, "w"), indent=1); return page def publish(force=False): stamp = os.path.join(ROOT, ".last-publish")