From 76723b32647bc06c6c5d533964ea830c9954612d Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:58:04 +0000 Subject: [PATCH] Fleet: the wave's wind-down by the 10 percent rule (hourly slices sized by weight share, weight_check gated, held under two thirds signing, the lock read after each); the poisoned-peer confirmation script Co-Authored-By: Claude Fable 5.1 --- tools/fleet/canary-poison-confirm.py | 33 +++++++++++++++++++ tools/fleet/winddown.py | 49 ++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 tools/fleet/canary-poison-confirm.py create mode 100644 tools/fleet/winddown.py diff --git a/tools/fleet/canary-poison-confirm.py b/tools/fleet/canary-poison-confirm.py new file mode 100644 index 000000000..9c28b0f8f --- /dev/null +++ b/tools/fleet/canary-poison-confirm.py @@ -0,0 +1,33 @@ +"""The post-deploy confirmation (coordinator + shipper, 21:4xZ): the forced poisoned-peer case against one 0.3.15 standing box, +in the honest layout. Relay R = pool-1 (inbound 213.173.98.36:18451) temporarily on 6615571c (7961c5f1: accepts and relays 1026 +blocks) over its own 1026-block datadir (/root/fleet/node.1026-2109, kept from 20:34Z to 21:09Z). Poisoned miner M = p2-3090-3 on +713ef876 (06211d55, stamps 1026) with a wiped datadir, peered with R only, mining five bounded minutes. Target T = p2-4090-3 on +7f0bde70, which holds R as a peer. Decisive reads every minute: T's log naming version 1026 as rejected from R's address and never +'Accepted ... via relay' for M's hashes; T never rejected or disconnected by the hands or the hub; the hub's count by peer (R's own +line may climb: it is poisoned and old-rule-blind, expected). Then everything wiped and back on 7f0bde70 / 0.3.14 as before. +Usage: poison-confirm.py (after the publish-1 table is in).""" +import sys, os, time, subprocess; sys.path.insert(0, "/Users/joshm/Projects/igneum-wt-gpu-fleet/tools/fleet"); from lib import Box, Registry +def st(): return time.strftime("%H:%M:%SZ", time.gmtime()) +R=Box.from_registry("pool-1"); M=Box.from_registry("p2-3090-3"); T=Box.from_registry("p2-4090-3"); hub=Box.from_registry("hub-1") +_, rM = Registry.find("p2-3090-3"); _, rR = Registry.find("pool-1") +W="/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\\n' ' '" +KILL="pkill -9 -f '^bash in/box-standing.sh'; pkill -9 -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs'; pkill -9 -f '^/(opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir='; sleep 3" +print(st(), "T before:", T.run("echo vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' /root/fleet/node.log | tail -1) rej1026=$(grep -c 'wrong block version' /root/fleet/node.log) got=$(grep -c 'got reject message' /root/fleet/node.log); "+W, 30)[1].replace("\n"," "), flush=True) +# R: the relay on 6615571c over its 1026 datadir (kept aside as node.clean while it runs) +rc,out,err=R.run(f"cd /root/fleet && {KILL}; [ -d node.clean ] || mv node node.clean; rm -rf node; cp -a node.1026-2109 node; setsid nohup in/igneumd-6615571ced35f694 --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:4463 --addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes > node.log 2>&1 & for i in $(seq 1 40); do sleep 10; w=$({W}); case \"$w\" in *synced=true*) break;; esac; done; echo R vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' node.log | tail -1) after=$((i*10))s $w", 500) +print(st(), "relay:", out.replace("\n"," "), err[:80], flush=True) +# M: the 713ef876 miner, wiped datadir, peered with R only; syncs from R then mines +rc,out,err=M.run(f"cd /root/fleet && {KILL}; mv node node.pre-poison-$(date -u +%H%M); mkdir node; setsid nohup in/igneumd-06211d55b8cce1c1 --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=213.173.98.36:18451 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes > node.log 2>&1 & for i in $(seq 1 90); do sleep 10; w=$({W}); case \"$w\" in *synced=true*) break;; esac; done; echo M vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' node.log | tail -1) synced_after=$((i*10))s $w; cd mine && rm -rf packs/devnet && /opt/igneum/pkg/bin/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >/dev/null 2>&1; setsid nohup /opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 p2-3090-3 --worker /opt/igneum/pkg/bin/igneum-worker-cuda --worker-args '--device 0 --pack packs/devnet' --prepare-packs packs/prepare --exit-on-seed-change --evm-address {rM.get('wallet')} --payout-label p2-3090-3 --status-secs 30 > /root/fleet/out/miner-poison.log 2>&1 & echo M mining T0=$(date -u +%T)", 1100) +print(st(), "poisoned miner:", out.replace("\n"," "), err[:80], flush=True) +T0=time.time() +while time.time()-T0 < 300: + time.sleep(60) + m=M.run("grep STATUS /root/fleet/out/miner-poison.log | tail -1 | grep -oE 'accepted=[0-9]+ rejected=[0-9]+'; echo got=$(grep -c 'got reject message' /root/fleet/node.log) "+W, 30)[1].replace("\n"," ") + r=R.run("echo R_wbv=$(grep -c 'wrong block version' /root/fleet/node.log) R_got=$(grep -c 'got reject message' /root/fleet/node.log) R_relayed=$(grep -c 'via relay' /root/fleet/node.log) "+W, 30)[1].replace("\n"," ") + t=T.run("echo T_rej1026=$(grep -c 'wrong block version' /root/fleet/node.log) T_from_R=$(grep 'wrong block version' /root/fleet/node.log | grep -c 213.173.98.36) T_got=$(grep -c 'got reject message' /root/fleet/node.log) T_peers=$(grep -oE 'Connected to (outgoing|incoming) peer [0-9.:]+' /root/fleet/node.log | awk '{print $NF}' | cut -d: -f1 | sort -u | tr '\\n' ','); grep 'wrong block version' /root/fleet/node.log | tail -1 | cut -c12-200", 30)[1].replace("\n"," | ") + h=hub.run("awk '$2 >= \"'$(date -u -d '-6 min' +%H:%M:%S)'\"' /root/fleet/node.log | grep 'wrong block version' | grep -oE 'peer [0-9.]+' | sort | uniq -c | sort -rn | tr '\\n' ';'", 30)[1].strip() + print(st(), f"+{int(time.time()-T0)}s M: {m} | R: {r} | T: {t} | hub by peer: {h}", flush=True) +# restore: M wiped back to 0.3.14 fresh under its supervisor; R wiped back to 7f0bde70 (its standing pointer) under its supervisor +rc,out,err=M.run(f"cd /root/fleet && {KILL}; mv node node.1026-$(date -u +%H%M); mkdir node; echo /opt/igneum/pkg/bin/igneumd-0313 > standing.node; ROLE=live LABEL=p2-3090-3 WALLET={rM.get('wallet')} PROVER=1 MINE=1 setsid nohup bash in/box-standing.sh /dev/null 2>&1 & echo M restored", 60); print(st(), out.strip(), flush=True) +rc,out,err=R.run(f"cd /root/fleet && {KILL}; rm -rf node.1026-replay; mv node node.1026-replay; rm -rf node; mkdir node; cat standing.node; ROLE=live LABEL=pool-1 WALLET={rR.get('wallet')} PROVER=0 MINE=1 P2P_LISTEN=0.0.0.0:4463 setsid nohup bash in/box-standing.sh /dev/null 2>&1 & echo R restored on its pointer with a fresh datadir", 60); print(st(), out.replace("\n"," "), flush=True) +print(st(), "POISON CONFIRM done", flush=True) diff --git a/tools/fleet/winddown.py b/tools/fleet/winddown.py new file mode 100644 index 000000000..66f539649 --- /dev/null +++ b/tools/fleet/winddown.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""The wave's wind-down by the standing rule (6 October 2026, 21:5xZ, the coordinator's order): never remove more than 10 percent of +the live devnet's voter weight in any hour, so finality never sees tonight's departure again. Every hour: read each candidate's +share of the last 2,000 blocks (lib.standing.weight_shares, the conservative proxy for the table), pick the largest slice under +LIMIT (default 9.5 percent) that weight_check allows (it adds what the last hour removed), hold the slice if the hub's last lock +signed under HOLD_PCT of the table after the removal, destroy the slice's pods (RunPod, registry rows marked), then read the lock +line ten minutes later. Candidates: the wave pods, then the 8x rig (one voter), never a standing box or a Devnet 2 box; rz-4090 waits +for the proving agent's word. Log: ~/Desktop/fleet/winddown.log; schedule and finality reads in ~/Desktop/fleet/winddown.jsonl.""" +import sys, os, time, json, datetime +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); from lib import Box, Registry, standing +LIMIT = float(os.environ.get("LIMIT", "0.095")); HOLD_PCT = float(os.environ.get("HOLD_PCT", "66.7")); EVERY = int(os.environ.get("EVERY", "3600")) +ROOT = os.path.expanduser("~/Desktop/fleet"); J = os.path.join(ROOT, "winddown.jsonl") +def st(): return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) +def log(**row): row["t"] = st(); open(J, "a").write(json.dumps(row) + "\n"); print(row["t"], {k: v for k, v in row.items() if k != "t"}, flush=True) +hub = Box.from_registry("hub-1") +def finality(): + rc, out, err = hub.run("grep -E 'Finality: checkpoint [0-9]+ LOCKED:' /root/fleet/node.log | tail -1 | grep -oE 'checkpoint [0-9]+|signed [0-9]+ = [0-9.]+% of active, [0-9.]+% of total' | tr '\\n' ' '; grep -E 'certificate at index [0-9]+ received' /root/fleet/node.log | tail -1 | grep -oE '[0-9]+ of [0-9]+ voters, weight [0-9]+ of [0-9]+'; grep -E 'Finality: checkpoint [0-9]+ LOCKED:' /root/fleet/node.log | tail -1 | cut -c1-19", 40) + s = out.replace("\n", " | ").strip(); import re; m = re.search(r"([0-9.]+)% of total", s); return s, float(m.group(1)) if m else None +def candidates(): + reg = Registry.load(); pods = sorted((i, b) for i, b in reg.items() if b.get("state") != "destroyed" and b.get("ssh_host") and b["label"].startswith("wave")) + rig = [(i, b) for i, b in reg.items() if b.get("state") != "destroyed" and b.get("label") == "rig-4090x8"] + return pods + rig +def keys_of(labels): + rc, out, err = hub.run("for l in " + " ".join(labels) + "; do echo $l $(/opt/igneum/pkg/bin/igneum-miner key-hash $l 2>/dev/null | grep -oE '[0-9a-f]{64}' | head -1); done", 120) + return dict(ln.split() for ln in out.splitlines() if len(ln.split()) == 2) +while True: + cands = candidates() + if not cands: log(event="done", note="no candidates left: the wave and the rig are down; the standing 14 and the Devnet 2 four stay"); break + labels = [b["label"] for _, b in cands]; sh, n = standing.weight_shares(2000); keys = keys_of(labels); share = {l: sh.get(keys.get(l, ""), 0.0) for l in labels} + fin, signed_total = finality() + recent = 0.0 + if os.path.exists(standing.WEIGHT_LOG): + for ln in open(standing.WEIGHT_LOG): + try: j = json.loads(ln) + except Exception: continue + if j.get("ts", 0) >= time.time() - 3600: recent += float(j.get("share", 0)) + budget = LIMIT - recent; slice_, acc = [], 0.0 + for l in sorted(labels, key=lambda x: share[x]): # smallest shares first: more pods per slice, the same weight + if acc + share[l] <= budget: slice_.append(l); acc += share[l] + if signed_total is not None and signed_total - 100 * acc < HOLD_PCT: log(event="hold", reason=f"the slice would take the signed share from {signed_total}% to {signed_total - 100*acc:.1f}%, under {HOLD_PCT}%", finality=fin); time.sleep(EVERY); continue + if not slice_: log(event="hold", reason=f"no pod fits the budget ({100*budget:.1f}% left this hour)", finality=fin); time.sleep(EVERY); continue + v = standing.weight_check(slice_, limit=LIMIT) + if not v["ok"]: log(event="hold", reason="weight_check refused", verdict=v, finality=fin); time.sleep(EVERY); continue + log(event="slice", pods=slice_, share_pct=round(100 * acc, 2), remaining=len(labels) - len(slice_), finality_before=fin, blocks_read=n) + for l in slice_: + try: Box.from_registry(l).destroy(); log(event="destroyed", pod=l, share_pct=round(100 * share[l], 2)) + except Exception as e: log(event="destroy_failed", pod=l, error=str(e)[:120]) + time.sleep(600); fin2, _ = finality(); log(event="finality_after", slice=slice_, finality=fin2) + time.sleep(EVERY - 600)