From 9be8954a2f101cd65a2549b10f987f0abdb11a6a Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:14:57 +0000 Subject: [PATCH] GPU fleet: RunPod client, the phase-1 collector, the phase-2 launcher (node restarted with the proof verifier), parallel status, the toolchain fix --- tools/fleet/box-prover.sh | 22 ++++++++++++++++ tools/fleet/box-setup.sh | 2 +- tools/fleet/collect.py | 53 +++++++++++++++++++++++++++++++++++++ tools/fleet/fleet.py | 24 +++++++++++------ tools/fleet/runpod.py | 55 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 9 deletions(-) create mode 100755 tools/fleet/box-prover.sh create mode 100755 tools/fleet/collect.py create mode 100755 tools/fleet/runpod.py diff --git a/tools/fleet/box-prover.sh b/tools/fleet/box-prover.sh new file mode 100755 index 000000000..c6ac506ea --- /dev/null +++ b/tools/fleet/box-prover.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Phase 2 launcher: restarts the box's node with IGNEUM_PROOF_VERIFIER (so its pool verifies records and its templates +# carry them, the proving agent's note of 6 October 2026), waits for it to be synced again (the data dir is kept, so +# seconds), then runs box-prover.py with the card's profile: THRESHOLD and MINER from the card's memory unless given. +set -uo pipefail +F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; FLOOR=/opt/igneum-floor; HOST=$FLOOR/bin/igneum-prove-host +mkdir -p $OUT; exec >> $OUT/prover-launch.log 2>&1 +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +TOTAL="$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits -i 0 | tr -d ' ')" +if [ -z "${THRESHOLD:-}" ]; then + if [ "$TOTAL" -ge 23000 ]; then THRESHOLD=""; elif [ "$TOTAL" -ge 15000 ]; then THRESHOLD=134217728; else THRESHOLD=67108864; fi +fi +if [ -z "${MINER:-}" ]; then if [ "$TOTAL" -ge 15000 ]; then MINER=keep; else MINER=pause; fi; fi +echo "RESULT launch $(stamp) total_mib=$TOTAL threshold=${THRESHOLD:-default} miner=$MINER" +pkill -f "igneum-miner mine" 2>/dev/null; pkill -f igneum-worker-cuda 2>/dev/null; pkill -f sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock +pkill -x igneumd; sleep 4; pkill -9 -x igneumd 2>/dev/null; sleep 1 +IGNEUM_PROOF_VERIFIER=$HOST nohup $B/igneumd --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 \ + --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes >> $F/node.log 2>&1 & +sleep 10 +echo "RESULT node_restarted $(stamp) pid=$(pgrep -x igneumd | head -1) verifier=$(grep -c 'IGNEUM_PROOF_VERIFIER\|proof verifier' $F/node.log)" +export THRESHOLD MINER +exec python3 -u $F/in/box-prover.py diff --git a/tools/fleet/box-setup.sh b/tools/fleet/box-setup.sh index 1848b480e..af8c142b5 100755 --- a/tools/fleet/box-setup.sh +++ b/tools/fleet/box-setup.sh @@ -56,7 +56,7 @@ fi echo "RESULT node pid=$(pgrep -x igneumd | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | head -1 | awk '{print $2}') fresh=$(grep -c 'fresh-record rule' $F/node.log)" # 3. rust and go -export PATH="$HOME/.cargo/bin:/opt/igneum-floor/go/bin:$PATH" +export PATH="$HOME/.cargo/bin:/opt/igneum-floor/go/bin:$PATH" RUSTUP_TOOLCHAIN=stable # the SP1 repo's rust-toolchain.toml would pull llvm-tools, rustc-dev and clippy from a CDN some hosts reach badly (the 3060 box, 12:00Z); plain stable builds the server if [ ! -x "$HOME/.cargo/bin/cargo" ]; then say "rustup"; curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal >/dev/null 2>&1 || fail rustup; fi if [ ! -x /opt/igneum-floor/go/bin/go ]; then say "go" diff --git a/tools/fleet/collect.py b/tools/fleet/collect.py new file mode 100755 index 000000000..66413ac5f --- /dev/null +++ b/tools/fleet/collect.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Phase 1 collector: reads ~/Desktop/fleet//matrix.json (and ember.json when present) for every phase-1 box, +writes ~/Desktop/fleet/results.json (the fleet page's rows, keyed by card) and prints the markdown table for +docs/analysis/prover-tiers-real-cards.md. The verdict rule, per card (the 9.0 GB mine-and-prove line and the tier +gates of docs/analysis/prover-floor.md), on the card's OWN numbers: + proves alone = a compressed proof of the v1 shard at 2^26 or 2^27 verified with the server's own working set + plus the idle under the card's memory, but mine-and-prove over the line + mines and proves = the compressed v1 shard beside the miner verified (peak under the card's memory) + mines and proves core-only = only the core-only point beside the miner fits + mining only = no proof point verified on the card (every point refused or out of memory) +""" +import json, os, sys, glob +ROOT = os.path.expanduser("~/Desktop/fleet") +reg = json.load(open(f"{ROOT}/boxes.json")) +def gb(mib): return round(mib / 1024, 1) +rows = {}; table = [] +for iid, b in reg.items(): + if not b.get("phase") == "1": continue + mp = f"{ROOT}/{iid}/matrix.json" + if not os.path.exists(mp): continue + m = json.load(open(mp)); pts = {r["name"]: r for r in m["rows"] if r.get("row") == "point"}; miner = next((r for r in m["rows"] if r.get("row") == "miner"), {}) + total = m["total_mib"]; idle = m["idle_mib"] + def ok(n): p = pts.get(n); return p and p.get("verified") == "yes" + def own(n): p = pts.get(n); return p.get("own_mib") if p else None + def secs(n): p = pts.get(n); return p.get("prove_s") if p else None + alone = next((n for n in ("alone-comp-26-v1", "alone-comp-27-v1", "alone-comp-25-v1") if ok(n)), None) + beside = next((n for n in ("miner-comp-26-v1", "miner-comp-27-v1") if ok(n)), None) + core_b = next((n for n in ("miner-core-25-v1", "miner-core-26-v1", "miner-core-24-v1") if ok(n)), None) + stock = pts.get("stock-comp-v1", {}) + if beside: verdict = "mines and proves" + elif alone and core_b: verdict = "mines and proves core-only" + elif alone: verdict = "proves alone" + else: verdict = "mining only" + card = m["card"].replace("NVIDIA GeForce ", "").replace("NVIDIA ", "") + r = {"card": card, "vram_gb": round(total / 1024), "mhs": miner.get("mhs"), "watts": miner.get("watts"), "miner_gb": gb(miner.get("own_mib", 0)), + "stock": "refused: " + (stock.get("err") or "")[:60] if stock.get("verified") != "yes" else f"proved {stock.get('prove_s')} s at {gb(stock.get('peak_mib', 0))} GB", + "prove_alone_gb": gb(own(alone)) if alone else None, "shard_s": secs(alone), "prove_alone_point": alone, + "mine_prove_gb": gb(pts[beside]["peak_mib"]) if beside else None, "mine_prove_s": secs(beside), "core_only_gb": gb(own(core_b)) if core_b else None, "core_only_s": secs(core_b), + "verdict": verdict, "points": {n: {"v": p.get("verified"), "own": p.get("own_mib"), "peak": p.get("peak_mib"), "s": p.get("prove_s"), "err": (p.get("err") or "")[:80]} for n, p in pts.items()}} + r["tier_line"] = {"mines and proves": f"{card} ({r['vram_gb']} GB): mines and proves; the v1 shard beside the miner {r['mine_prove_s']} s at {r['mine_prove_gb']} GB peak", + "mines and proves core-only": f"{card} ({r['vram_gb']} GB): proves alone ({r['shard_s']} s) and mines beside a core-only prover ({r['core_only_gb']} GB own)", + "proves alone": f"{card} ({r['vram_gb']} GB): proves alone, {r['shard_s']} s a v1 shard at {r['prove_alone_gb']} GB own; not beside the miner", + "mining only": f"{card} ({r['vram_gb']} GB): mines only ({r['mhs']} MH/s); no proof point fits"}[verdict] + ep = f"{ROOT}/{iid}/ember.json" + if os.path.exists(ep): + e = json.load(open(ep)); c = e["chosen"] + r.update({"tune_w": c["watts"], "tune_clock_mhz": c["clock_mhz"], "tune_mhs": c["mhs"], "tune_mhw": c["eff"], "ladder": [{"w": s["watts"], "mhs": s["mhs"], "mhw": s["eff"], "step": s["label"]} for s in e["steps"]], "tune_plan": e["plan"]}) + rows[card] = r + table.append(f"| {card} | {r['vram_gb']} | {idle} | {r['mhs']} MH/s at {r['watts']} W, {r['miner_gb']} GB | {r['stock']} | " + + f"{r['prove_alone_gb'] or 'no'} GB, {r['shard_s'] or ''} s ({alone or 'none'}) | {r['mine_prove_gb'] or 'no'} GB peak, {r['mine_prove_s'] or ''} s | {r['core_only_gb'] or 'no'} GB, {r['core_only_s'] or ''} s | {verdict} |") +json.dump(rows, open(f"{ROOT}/results.json", "w"), indent=1) +print("| Card | VRAM GB | Idle MiB | Miner | Stock SP1 6.8.1 | Patched, proves alone (own) | Beside the miner (peak) | Core-only beside the miner (own) | Verdict |\n|---|---|---|---|---|---|---|---|---|") +print("\n".join(sorted(table))) diff --git a/tools/fleet/fleet.py b/tools/fleet/fleet.py index a1bfe7c48..847d0c6fd 100755 --- a/tools/fleet/fleet.py +++ b/tools/fleet/fleet.py @@ -84,20 +84,28 @@ def setup(labels): if not b.get("ssh_ok"): print(b["label"], "no ssh yet"); continue ssh(b, "mkdir -p /root/fleet/in /root/fleet/out") if scp(b, INPUTS + [os.path.join(HERE, "box-setup.sh")], "/root/fleet/in/") != 0: print(b["label"], "scp failed"); continue - rc, out, err = ssh(b, f"cd /root/fleet && chmod +x in/box-setup.sh && if pgrep -f in/box-setup.sh >/dev/null; then echo already; else ARCHS={b['archs']} LABEL={b['label']} WALLET={b['wallet']} setsid nohup in/box-setup.sh /dev/null 2>&1 & echo started; fi", timeout=60) + rc, out, err = ssh(b, f"cd /root/fleet && chmod +x in/box-setup.sh && if pgrep -f '^bash in/box-setup.sh' >/dev/null; then echo already; else RUSTUP_TOOLCHAIN=stable ARCHS={b['archs']} LABEL={b['label']} WALLET={b['wallet']} setsid nohup in/box-setup.sh /dev/null 2>&1 & echo started; fi", timeout=60) print(b["label"], out.strip()[:40]) b["setup_started"] = now(); print(b["label"], "setup started") save(load() | {k: v for k, v in boxes(labels).items()}) def status(labels): reg = load() - for iid, b in boxes(labels, reg).items(): - if not b.get("ssh_ok"): print(f"{b['label']:<14} {iid} no ssh"); continue - try: - rc, out, err = ssh(b, "grep -E '^(RESULT|STAGE)' /root/fleet/setup.log 2>/dev/null | tail -1; grep -c '^RESULT setup_done' /root/fleet/setup.log 2>/dev/null; /opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1; grep -E '^(RESULT|STAGE)' /root/fleet/out/matrix.log 2>/dev/null | tail -1", timeout=60) - lines = out.strip().split("\n"); print(f"{b['label']:<14} {iid} | " + " | ".join(l[:110] for l in lines)) - if any(l.startswith("1") for l in lines[1:2]) and b.get("state") == "installing": b["state"] = "running"; save(reg) - except subprocess.TimeoutExpired: print(f"{b['label']:<14} {iid} ssh timeout") + from concurrent.futures import ThreadPoolExecutor + def one(item): + iid, b = item + if not b.get("ssh_ok"): return f"{b['label']:<14} {iid} no ssh", None + rc, out, err = ssh(b, "grep -E '^(RESULT|STAGE)' /root/fleet/setup.log 2>/dev/null | tail -1; grep -c '^RESULT setup_done' /root/fleet/setup.log 2>/dev/null; /opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1; grep -E '^(RESULT|STAGE)' /root/fleet/out/matrix.log 2>/dev/null | tail -1; grep -E '^RESULT' /root/fleet/out/ember.log 2>/dev/null | tail -1; grep -E '^RESULT' /root/fleet/out/prover.log 2>/dev/null | tail -1", timeout=45) + if rc == 124: return f"{b['label']:<14} {iid} ssh timeout", None + lines = out.strip().split("\n") + done = len(lines) > 1 and lines[1].strip() == "1" + return f"{b['label']:<14} {iid} | " + " | ".join(l[:120] for l in lines if l.strip() and l.strip() not in ("0", "1")), done + items = list(boxes(labels, reg).items()) + with ThreadPoolExecutor(max_workers=16) as ex: results = list(ex.map(one, items)) + for (iid, b), (line, done) in zip(items, results): + print(line) + if done and b.get("state") == "installing": b["state"] = "running" + save(reg) def run(script, labels, env=""): for iid, b in boxes(labels).items(): diff --git a/tools/fleet/runpod.py b/tools/fleet/runpod.py new file mode 100755 index 000000000..bff03da22 --- /dev/null +++ b/tools/fleet/runpod.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""RunPod REST client for the fleet (https://rest.runpod.io/v1, Bearer from ~/.config/runpod/credentials; the GraphQL +endpoint refuses the Bearer header, so nothing here touches account settings: the fleet key goes in per pod through +the PUBLIC_KEY env, which RunPod's images write to authorized_keys). Ledger lines as vast.py. + + runpod.py gpus the GPU types with availability and prices + runpod.py rent