From 34fc6382662ff8157740b60a98738201981f602f Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:53:23 +0000 Subject: [PATCH] box-prover.py: one instance per box (pid file) and a per-process state tmp (two instances raced and killed each other's GPU server on Devnet 2); dn2-check.py: the Devnet 2 gate reads for the build-server seed Co-Authored-By: Claude Fable 5.1 --- tools/fleet/box-prover.py | 10 +++++++-- tools/fleet/dn2-check.py | 45 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 tools/fleet/dn2-check.py diff --git a/tools/fleet/box-prover.py b/tools/fleet/box-prover.py index 34b3774b6..817f30eb1 100644 --- a/tools/fleet/box-prover.py +++ b/tools/fleet/box-prover.py @@ -109,8 +109,14 @@ state = {"passes": 0, "claimed": 0, "submitted": 0, "paid": 0, "paid_wei": 0, "s attempted = set(); submitted = {}; held = {} # held: first -> {body file, deadline, last} last_seg_secs = 0; t_run0 = time.time() def save_state(): - state["miner_mhs"] = miner_rate(); state["updated"] = stamp(); state["run_min"] = round((time.time() - t_run0) / 60, 1) - json.dump(state, open(f"{OUT}/prover-state.json.tmp", "w"), indent=1); os.replace(f"{OUT}/prover-state.json.tmp", f"{OUT}/prover-state.json") + tmp = f"{OUT}/prover-state.json.{os.getpid()}.tmp" # one tmp per process: two instances racing on one name lost the file (21:4xZ) + json.dump(state, open(tmp, "w"), indent=1); os.replace(tmp, f"{OUT}/prover-state.json") +# one prover per box: a pid file under OUT; a second instance exits at once instead of killing the first's GPU server +_pidf = f"{OUT}/prover.pid" +try: + _old = int(open(_pidf).read().strip()); os.kill(_old, 0); print(f"RESULT refused {stamp()} another box-prover.py runs as pid {_old}; exiting", flush=True); sys.exit(3) +except (FileNotFoundError, ValueError, ProcessLookupError): pass +open(_pidf, "w").write(str(os.getpid())) def submit(method, record, proof_file): proof = "0x" + binascii.hexlify(open(proof_file, "rb").read()).decode() return rpc(method, [{"record": record, "proof": proof}], timeout=180) diff --git a/tools/fleet/dn2-check.py b/tools/fleet/dn2-check.py new file mode 100644 index 000000000..b000a3b94 --- /dev/null +++ b/tools/fleet/dn2-check.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""The Devnet 2 standing gate's reads without the install step (6 October 2026, 21:4xZ; the seed moved to igneum-build-1, +user build, rpc 27610, so devnet2-gate.sh's root@ ssh and standard ports no longer fit the seed). Reads every Devnet 2 box +(registry rows with devnet2 or dn2_seed; the seed through its seed_env ports) and prints one line: PASS only with zero +rejected blocks on every box since SINCE, no selected-chain reorg over depth 3, equal exec state roots at a common height, +at least one paid segment record, every node on one version. Usage: dn2-check.py [--since 2026-10-06T21:00:00Z]""" +import sys, os, json, time, datetime +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); from lib import Box, Registry +from concurrent.futures import ThreadPoolExecutor +def now(): return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") +since = sys.argv[sys.argv.index("--since") + 1] if "--since" in sys.argv else (datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(minutes=30)).strftime("%Y-%m-%dT%H:%M:%SZ") +since_log = since.replace("T", " ").rstrip("Z") +reg = Registry.load() +rows = [(i, b) for i, b in reg.items() if b.get("state") != "destroyed" and b.get("ssh_host") and (b.get("devnet2") or b.get("dn2_seed"))] +def read(item): + i, b = item; env = b.get("seed_env") or {} + log = env.get("SEED_LOG", "/root/fleet/dn2-node.log"); rpc = env.get("SEED_RPC", "27610" if b["label"].startswith("wave") else "26610"); evm = env.get("SEED_EVM", "27790" if b["label"].startswith("wave") else "26790"); miner = env.get("SEED_MINER", "/opt/igneum/pkg/bin/igneum-miner") + if b["label"] == "dn2-seed": rpc, evm = "27610", "27790" # its 26610 is the rehearsal box's live node; the Devnet 2 node sits on the alternate ports + box = Box(b["ssh_host"], b["ssh_port"], b["label"], i, b.get("wallet"), b.get("provider"), b.get("ssh_user", "root")) + cmd = f"""w=$({miner} watch 1 grpc://127.0.0.1:{rpc} 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1); echo W $w +echo REJ $(awk -v s='{since_log}' '($1" "$2) >= s' {log} | grep -c 'PoW rejected') REORG $(awk -v s='{since_log}' '($1" "$2) >= s' {log} | grep -oE 'selected-chain reorg: [0-9]+ chain blocks' | grep -oE '[0-9]+ chain' | awk 'BEGIN{{m=0}} {{if ($1>m) m=$1}} END{{print m}}') +echo EXEC $(curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{{"jsonrpc":"2.0","id":1,"method":"igneum_getExecStatus","params":[]}}' http://127.0.0.1:{evm}/ | grep -oE '"executedTip":"0x[0-9a-f]+"' | grep -oE '0x[0-9a-f]+') +echo PAID $(curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{{"jsonrpc":"2.0","id":1,"method":"igneum_getProvingStatus","params":[]}}' http://127.0.0.1:{evm}/ | grep -oE '"paidSegments":[0-9]+' | grep -oE '[0-9]+') +echo VER $(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' {log} | tail -1) DIGEST $(grep -o 'digest: [0-9a-f]*' {log} | tail -1 | awk '{{print substr($2,1,16)}}')""" + rc, out, err = box.run(cmd, 60); d = {"label": b["label"], "seed": bool(b.get("dn2_seed")), "evm": evm, "box": box} + for ln in out.splitlines(): + p = ln.split() + if not p: continue + if p[0] == "W": d.update({k: v for k, v in (kv.split("=", 1) for kv in p[1:] if "=" in kv)}) + elif p[0] == "REJ" and len(p) >= 4: d["rej"] = int(p[1] or 0); d["reorg"] = int(p[3] or 0) + elif p[0] == "EXEC" and len(p) > 1: d["exec"] = int(p[1], 16) + elif p[0] == "PAID" and len(p) > 1: d["paid"] = int(p[1]) + elif p[0] == "VER": d["ver"] = p[1] if len(p) > 1 and p[1] != "DIGEST" else ""; d["digest"] = p[-1] if "DIGEST" in p else "" + return d +with ThreadPoolExecutor(8) as ex: res = list(ex.map(read, rows)) +# a common height for the state root: the lowest exec tip minus a little +execs = [d.get("exec", 0) for d in res if d.get("exec")]; h = max(0, min(execs) - 2) if execs else 0 +def root(d): + rc, out, err = d["box"].run(f"curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"{hex(h)}\",false]}}' http://127.0.0.1:{d['evm']}/ | grep -oE '\"stateRoot\":\"0x[0-9a-f]+\"' | cut -d'\"' -f4", 30); return out.strip()[:18] +with ThreadPoolExecutor(8) as ex: + for d, r in zip(res, ex.map(root, res)): d["root"] = r +for d in res: print(f" {d['label']:<10} {'seed ' if d['seed'] else ' '} blocks={d.get('blocks','?'):<7} daa={d.get('daa','?'):<7} synced={d.get('synced','?'):<5} exec={d.get('exec','?'):<7} root@{h}={d.get('root','?'):<18} paid={d.get('paid','?'):<4} rej={d.get('rej','?')} reorg={d.get('reorg','?')} {d.get('ver','')} {d.get('digest','')}") +rej = sum(d.get("rej", 0) for d in res); reorg = max((d.get("reorg", 0) for d in res), default=0); roots = {d.get("root") for d in res if d.get("root")}; vers = {d.get("ver") for d in res if d.get("ver")}; paid = max((d.get("paid", 0) for d in res), default=0) +ok = rej == 0 and reorg <= 3 and len(roots) == 1 and paid >= 1 and len(vers) == 1 +print(f"{'PASS' if ok else 'FAIL'} {now()} since {since}: boxes {len(res)}, rejected {rej}, max reorg {reorg}, state roots at {h}: {len(roots)} distinct, paid segments {paid}, versions {sorted(vers)}")