GPU fleet: RunPod client, the phase-1 collector, the phase-2 launcher (node restarted with the proof verifier), parallel status, the toolchain fix
This commit is contained in:
igneum-labs
parent
1b79639577
commit
9be8954a2f
5 changed files with 147 additions and 9 deletions
22
tools/fleet/box-prover.sh
Executable file
22
tools/fleet/box-prover.sh
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
#!/usr/bin/env bash
|
||||
# Phase 2 launcher: restarts the box's node with IGNEUM_PROOF_VERIFIER (so its pool verifies records and its templates
|
||||
# carry them, the proving agent's note of 6 October 2026), waits for it to be synced again (the data dir is kept, so
|
||||
# seconds), then runs box-prover.py with the card's profile: THRESHOLD and MINER from the card's memory unless given.
|
||||
set -uo pipefail
|
||||
F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; FLOOR=/opt/igneum-floor; HOST=$FLOOR/bin/igneum-prove-host
|
||||
mkdir -p $OUT; exec >> $OUT/prover-launch.log 2>&1
|
||||
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
TOTAL="$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits -i 0 | tr -d ' ')"
|
||||
if [ -z "${THRESHOLD:-}" ]; then
|
||||
if [ "$TOTAL" -ge 23000 ]; then THRESHOLD=""; elif [ "$TOTAL" -ge 15000 ]; then THRESHOLD=134217728; else THRESHOLD=67108864; fi
|
||||
fi
|
||||
if [ -z "${MINER:-}" ]; then if [ "$TOTAL" -ge 15000 ]; then MINER=keep; else MINER=pause; fi; fi
|
||||
echo "RESULT launch $(stamp) total_mib=$TOTAL threshold=${THRESHOLD:-default} miner=$MINER"
|
||||
pkill -f "igneum-miner mine" 2>/dev/null; pkill -f igneum-worker-cuda 2>/dev/null; pkill -f sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock
|
||||
pkill -x igneumd; sleep 4; pkill -9 -x igneumd 2>/dev/null; sleep 1
|
||||
IGNEUM_PROOF_VERIFIER=$HOST nohup $B/igneumd --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 \
|
||||
--addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes >> $F/node.log 2>&1 &
|
||||
sleep 10
|
||||
echo "RESULT node_restarted $(stamp) pid=$(pgrep -x igneumd | head -1) verifier=$(grep -c 'IGNEUM_PROOF_VERIFIER\|proof verifier' $F/node.log)"
|
||||
export THRESHOLD MINER
|
||||
exec python3 -u $F/in/box-prover.py
|
||||
|
|
@ -56,7 +56,7 @@ fi
|
|||
echo "RESULT node pid=$(pgrep -x igneumd | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | head -1 | awk '{print $2}') fresh=$(grep -c 'fresh-record rule' $F/node.log)"
|
||||
|
||||
# 3. rust and go
|
||||
export PATH="$HOME/.cargo/bin:/opt/igneum-floor/go/bin:$PATH"
|
||||
export PATH="$HOME/.cargo/bin:/opt/igneum-floor/go/bin:$PATH" RUSTUP_TOOLCHAIN=stable # the SP1 repo's rust-toolchain.toml would pull llvm-tools, rustc-dev and clippy from a CDN some hosts reach badly (the 3060 box, 12:00Z); plain stable builds the server
|
||||
if [ ! -x "$HOME/.cargo/bin/cargo" ]; then say "rustup"; curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal >/dev/null 2>&1 || fail rustup; fi
|
||||
if [ ! -x /opt/igneum-floor/go/bin/go ]; then
|
||||
say "go"
|
||||
|
|
|
|||
53
tools/fleet/collect.py
Executable file
53
tools/fleet/collect.py
Executable file
|
|
@ -0,0 +1,53 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Phase 1 collector: reads ~/Desktop/fleet/<instance>/matrix.json (and ember.json when present) for every phase-1 box,
|
||||
writes ~/Desktop/fleet/results.json (the fleet page's rows, keyed by card) and prints the markdown table for
|
||||
docs/analysis/prover-tiers-real-cards.md. The verdict rule, per card (the 9.0 GB mine-and-prove line and the tier
|
||||
gates of docs/analysis/prover-floor.md), on the card's OWN numbers:
|
||||
proves alone = a compressed proof of the v1 shard at 2^26 or 2^27 verified with the server's own working set
|
||||
plus the idle under the card's memory, but mine-and-prove over the line
|
||||
mines and proves = the compressed v1 shard beside the miner verified (peak under the card's memory)
|
||||
mines and proves core-only = only the core-only point beside the miner fits
|
||||
mining only = no proof point verified on the card (every point refused or out of memory)
|
||||
"""
|
||||
import json, os, sys, glob
|
||||
ROOT = os.path.expanduser("~/Desktop/fleet")
|
||||
reg = json.load(open(f"{ROOT}/boxes.json"))
|
||||
def gb(mib): return round(mib / 1024, 1)
|
||||
rows = {}; table = []
|
||||
for iid, b in reg.items():
|
||||
if not b.get("phase") == "1": continue
|
||||
mp = f"{ROOT}/{iid}/matrix.json"
|
||||
if not os.path.exists(mp): continue
|
||||
m = json.load(open(mp)); pts = {r["name"]: r for r in m["rows"] if r.get("row") == "point"}; miner = next((r for r in m["rows"] if r.get("row") == "miner"), {})
|
||||
total = m["total_mib"]; idle = m["idle_mib"]
|
||||
def ok(n): p = pts.get(n); return p and p.get("verified") == "yes"
|
||||
def own(n): p = pts.get(n); return p.get("own_mib") if p else None
|
||||
def secs(n): p = pts.get(n); return p.get("prove_s") if p else None
|
||||
alone = next((n for n in ("alone-comp-26-v1", "alone-comp-27-v1", "alone-comp-25-v1") if ok(n)), None)
|
||||
beside = next((n for n in ("miner-comp-26-v1", "miner-comp-27-v1") if ok(n)), None)
|
||||
core_b = next((n for n in ("miner-core-25-v1", "miner-core-26-v1", "miner-core-24-v1") if ok(n)), None)
|
||||
stock = pts.get("stock-comp-v1", {})
|
||||
if beside: verdict = "mines and proves"
|
||||
elif alone and core_b: verdict = "mines and proves core-only"
|
||||
elif alone: verdict = "proves alone"
|
||||
else: verdict = "mining only"
|
||||
card = m["card"].replace("NVIDIA GeForce ", "").replace("NVIDIA ", "")
|
||||
r = {"card": card, "vram_gb": round(total / 1024), "mhs": miner.get("mhs"), "watts": miner.get("watts"), "miner_gb": gb(miner.get("own_mib", 0)),
|
||||
"stock": "refused: " + (stock.get("err") or "")[:60] if stock.get("verified") != "yes" else f"proved {stock.get('prove_s')} s at {gb(stock.get('peak_mib', 0))} GB",
|
||||
"prove_alone_gb": gb(own(alone)) if alone else None, "shard_s": secs(alone), "prove_alone_point": alone,
|
||||
"mine_prove_gb": gb(pts[beside]["peak_mib"]) if beside else None, "mine_prove_s": secs(beside), "core_only_gb": gb(own(core_b)) if core_b else None, "core_only_s": secs(core_b),
|
||||
"verdict": verdict, "points": {n: {"v": p.get("verified"), "own": p.get("own_mib"), "peak": p.get("peak_mib"), "s": p.get("prove_s"), "err": (p.get("err") or "")[:80]} for n, p in pts.items()}}
|
||||
r["tier_line"] = {"mines and proves": f"{card} ({r['vram_gb']} GB): mines and proves; the v1 shard beside the miner {r['mine_prove_s']} s at {r['mine_prove_gb']} GB peak",
|
||||
"mines and proves core-only": f"{card} ({r['vram_gb']} GB): proves alone ({r['shard_s']} s) and mines beside a core-only prover ({r['core_only_gb']} GB own)",
|
||||
"proves alone": f"{card} ({r['vram_gb']} GB): proves alone, {r['shard_s']} s a v1 shard at {r['prove_alone_gb']} GB own; not beside the miner",
|
||||
"mining only": f"{card} ({r['vram_gb']} GB): mines only ({r['mhs']} MH/s); no proof point fits"}[verdict]
|
||||
ep = f"{ROOT}/{iid}/ember.json"
|
||||
if os.path.exists(ep):
|
||||
e = json.load(open(ep)); c = e["chosen"]
|
||||
r.update({"tune_w": c["watts"], "tune_clock_mhz": c["clock_mhz"], "tune_mhs": c["mhs"], "tune_mhw": c["eff"], "ladder": [{"w": s["watts"], "mhs": s["mhs"], "mhw": s["eff"], "step": s["label"]} for s in e["steps"]], "tune_plan": e["plan"]})
|
||||
rows[card] = r
|
||||
table.append(f"| {card} | {r['vram_gb']} | {idle} | {r['mhs']} MH/s at {r['watts']} W, {r['miner_gb']} GB | {r['stock']} | " +
|
||||
f"{r['prove_alone_gb'] or 'no'} GB, {r['shard_s'] or ''} s ({alone or 'none'}) | {r['mine_prove_gb'] or 'no'} GB peak, {r['mine_prove_s'] or ''} s | {r['core_only_gb'] or 'no'} GB, {r['core_only_s'] or ''} s | {verdict} |")
|
||||
json.dump(rows, open(f"{ROOT}/results.json", "w"), indent=1)
|
||||
print("| Card | VRAM GB | Idle MiB | Miner | Stock SP1 6.8.1 | Patched, proves alone (own) | Beside the miner (peak) | Core-only beside the miner (own) | Verdict |\n|---|---|---|---|---|---|---|---|---|")
|
||||
print("\n".join(sorted(table)))
|
||||
|
|
@ -84,20 +84,28 @@ def setup(labels):
|
|||
if not b.get("ssh_ok"): print(b["label"], "no ssh yet"); continue
|
||||
ssh(b, "mkdir -p /root/fleet/in /root/fleet/out")
|
||||
if scp(b, INPUTS + [os.path.join(HERE, "box-setup.sh")], "/root/fleet/in/") != 0: print(b["label"], "scp failed"); continue
|
||||
rc, out, err = ssh(b, f"cd /root/fleet && chmod +x in/box-setup.sh && if pgrep -f in/box-setup.sh >/dev/null; then echo already; else ARCHS={b['archs']} LABEL={b['label']} WALLET={b['wallet']} setsid nohup in/box-setup.sh </dev/null >/dev/null 2>&1 & echo started; fi", timeout=60)
|
||||
rc, out, err = ssh(b, f"cd /root/fleet && chmod +x in/box-setup.sh && if pgrep -f '^bash in/box-setup.sh' >/dev/null; then echo already; else RUSTUP_TOOLCHAIN=stable ARCHS={b['archs']} LABEL={b['label']} WALLET={b['wallet']} setsid nohup in/box-setup.sh </dev/null >/dev/null 2>&1 & echo started; fi", timeout=60)
|
||||
print(b["label"], out.strip()[:40])
|
||||
b["setup_started"] = now(); print(b["label"], "setup started")
|
||||
save(load() | {k: v for k, v in boxes(labels).items()})
|
||||
|
||||
def status(labels):
|
||||
reg = load()
|
||||
for iid, b in boxes(labels, reg).items():
|
||||
if not b.get("ssh_ok"): print(f"{b['label']:<14} {iid} no ssh"); continue
|
||||
try:
|
||||
rc, out, err = ssh(b, "grep -E '^(RESULT|STAGE)' /root/fleet/setup.log 2>/dev/null | tail -1; grep -c '^RESULT setup_done' /root/fleet/setup.log 2>/dev/null; /opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1; grep -E '^(RESULT|STAGE)' /root/fleet/out/matrix.log 2>/dev/null | tail -1", timeout=60)
|
||||
lines = out.strip().split("\n"); print(f"{b['label']:<14} {iid} | " + " | ".join(l[:110] for l in lines))
|
||||
if any(l.startswith("1") for l in lines[1:2]) and b.get("state") == "installing": b["state"] = "running"; save(reg)
|
||||
except subprocess.TimeoutExpired: print(f"{b['label']:<14} {iid} ssh timeout")
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
def one(item):
|
||||
iid, b = item
|
||||
if not b.get("ssh_ok"): return f"{b['label']:<14} {iid} no ssh", None
|
||||
rc, out, err = ssh(b, "grep -E '^(RESULT|STAGE)' /root/fleet/setup.log 2>/dev/null | tail -1; grep -c '^RESULT setup_done' /root/fleet/setup.log 2>/dev/null; /opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1; grep -E '^(RESULT|STAGE)' /root/fleet/out/matrix.log 2>/dev/null | tail -1; grep -E '^RESULT' /root/fleet/out/ember.log 2>/dev/null | tail -1; grep -E '^RESULT' /root/fleet/out/prover.log 2>/dev/null | tail -1", timeout=45)
|
||||
if rc == 124: return f"{b['label']:<14} {iid} ssh timeout", None
|
||||
lines = out.strip().split("\n")
|
||||
done = len(lines) > 1 and lines[1].strip() == "1"
|
||||
return f"{b['label']:<14} {iid} | " + " | ".join(l[:120] for l in lines if l.strip() and l.strip() not in ("0", "1")), done
|
||||
items = list(boxes(labels, reg).items())
|
||||
with ThreadPoolExecutor(max_workers=16) as ex: results = list(ex.map(one, items))
|
||||
for (iid, b), (line, done) in zip(items, results):
|
||||
print(line)
|
||||
if done and b.get("state") == "installing": b["state"] = "running"
|
||||
save(reg)
|
||||
|
||||
def run(script, labels, env=""):
|
||||
for iid, b in boxes(labels).items():
|
||||
|
|
|
|||
55
tools/fleet/runpod.py
Executable file
55
tools/fleet/runpod.py
Executable file
|
|
@ -0,0 +1,55 @@
|
|||
#!/usr/bin/env python3
|
||||
"""RunPod REST client for the fleet (https://rest.runpod.io/v1, Bearer from ~/.config/runpod/credentials; the GraphQL
|
||||
endpoint refuses the Bearer header, so nothing here touches account settings: the fleet key goes in per pod through
|
||||
the PUBLIC_KEY env, which RunPod's images write to authorized_keys). Ledger lines as vast.py.
|
||||
|
||||
runpod.py gpus the GPU types with availability and prices
|
||||
runpod.py rent <gpuTypeId> <label> [--count 8] [--disk 80] [--image ...] [--secure]
|
||||
runpod.py list pods: id, name, status, ssh endpoint
|
||||
runpod.py destroy <pod_id>...
|
||||
"""
|
||||
import json, os, sys, time, urllib.request, urllib.error, datetime, argparse
|
||||
KEY = open(os.path.expanduser("~/.config/runpod/credentials")).read().strip()
|
||||
BASE = "https://rest.runpod.io/v1"
|
||||
LEDGER = os.path.expanduser("~/Desktop/fleet/ledger.jsonl")
|
||||
IMAGE = "runpod/pytorch:2.8.0-py3.11-cuda12.8.1-cudnn-devel-ubuntu22.04"
|
||||
PUB = open(os.path.expanduser("~/.ssh/igneum-fleet.pub")).read().strip()
|
||||
def call(method, path, body=None):
|
||||
req = urllib.request.Request(BASE + path, data=(json.dumps(body).encode() if body is not None else None),
|
||||
headers={"Authorization": "Bearer " + KEY, "Content-Type": "application/json"}, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=90) as r:
|
||||
t = r.read(); return json.loads(t) if t else {}
|
||||
except urllib.error.HTTPError as e: raise SystemExit(f"{method} {path}: HTTP {e.code}: {e.read().decode(errors='replace')[:400]}")
|
||||
def now(): return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
def ledger(row):
|
||||
with open(LEDGER, "a") as f: f.write(json.dumps(row) + "\n")
|
||||
def gpus():
|
||||
for g in call("GET", "/gputypes"):
|
||||
print(f"{g.get('id'):<34} {g.get('displayName','')!s:<22} {g.get('memoryInGb')}GB secure ${g.get('securePrice')}/h community ${g.get('communityPrice')}/h max {g.get('maxGpuCount')} avail {g.get('lowestPrice', {}) if isinstance(g.get('lowestPrice'), dict) else ''}")
|
||||
def rent(gpu, label, count=8, disk=80, image=IMAGE, secure=False):
|
||||
body = {"name": label, "imageName": image, "gpuTypeIds": [gpu], "gpuCount": count, "containerDiskInGb": disk, "volumeInGb": 0,
|
||||
"cloudType": "SECURE" if secure else "COMMUNITY", "ports": ["22/tcp"], "env": {"PUBLIC_KEY": PUB}, "supportPublicIp": True, "computeType": "GPU"}
|
||||
d = call("POST", "/pods", body)
|
||||
row = {"t": now(), "event": "rent", "provider": "runpod", "instance": d.get("id"), "label": label, "gpu": gpu, "num_gpus": count, "dph": d.get("costPerHr"), "disk": disk}
|
||||
ledger(row); print(json.dumps(row)); print(json.dumps(d)[:600]); return d
|
||||
def pods():
|
||||
out = call("GET", "/pods")
|
||||
return out if isinstance(out, list) else out.get("pods", [])
|
||||
def ssh_of(p):
|
||||
for m in (p.get("portMappings") or {}).items() if isinstance(p.get("portMappings"), dict) else []:
|
||||
if str(m[0]) == "22": return p.get("publicIp"), m[1]
|
||||
return p.get("publicIp"), None
|
||||
def destroy(pid):
|
||||
d = call("DELETE", f"/pods/{pid}"); ledger({"t": now(), "event": "destroy", "provider": "runpod", "instance": pid}); print("destroyed", pid, d)
|
||||
if __name__ == "__main__":
|
||||
a = sys.argv[1:]
|
||||
if a[0] == "gpus": gpus()
|
||||
elif a[0] == "rent":
|
||||
ap = argparse.ArgumentParser(); ap.add_argument("gpu"); ap.add_argument("label"); ap.add_argument("--count", type=int, default=8); ap.add_argument("--disk", type=int, default=80); ap.add_argument("--image", default=IMAGE); ap.add_argument("--secure", action="store_true")
|
||||
n = ap.parse_args(a[1:]); rent(n.gpu, n.label, n.count, n.disk, n.image, n.secure)
|
||||
elif a[0] == "list":
|
||||
for p in pods(): print(p.get("id"), p.get("name"), p.get("desiredStatus"), p.get("gpuCount"), p.get("machine", {}).get("gpuTypeId") if isinstance(p.get("machine"), dict) else "", f"${p.get('costPerHr')}/h", ssh_of(p), (p.get("portMappings")))
|
||||
elif a[0] == "destroy":
|
||||
for i in a[1:]: destroy(i)
|
||||
elif a[0] == "raw": print(json.dumps(pods(), indent=1)[:3000])
|
||||
Loading…
Reference in a new issue