diff --git a/docs/plans/gpu-fleet.md b/docs/plans/gpu-fleet.md index 4dafc3476..15d469b61 100644 --- a/docs/plans/gpu-fleet.md +++ b/docs/plans/gpu-fleet.md @@ -42,3 +42,18 @@ nodes were igneum-build-1 (26611) and dn2-seed's one mapped port; a true mesh ne - 19:41Z: the 13 live boxes converted (supervisor started, every node synced, every prover up). - Owed: the Devnet 2 six after the block-rate runs; the L4 and the AMD pair when a provider reopens; the ports on re-rent. + +## The 10 percent rule (6 October 2026, 20:00Z, from the finality pause) + +What happened: the class v4 rehearsal job took the GPUs of 13 live-devnet miners between 18:27Z and 18:30Z (their +live nodes stayed up and synced, but a voter's weight is its blue blocks, and a miner that stops mining stops being +a voter), and with seven earlier leavers that was 42.7 percent of the frozen voter table; finality rule v3 then holds +the pause for one full window, and the first lock after 18:39:36Z is expected at about 20:40Z. The fleet's read that +"the boxes never left the devnet" was true of the nodes and wrong about the weight. + +The rule: never remove more than 10 percent of the live devnet's 30-day weight in any hour. Weight is counted by blue +blocks per key over the window, read from the hub; any experiment that borrows live miners does it in slices with an +hour between slices; a standing box's miner is never stopped (or its GPU shared with a second worker) by a job +without that check. `lib/standing.py weight_check ` is the gate: it reads the hub's blue blocks per payout key +over the window, sums the share of the labels asked for, and refuses the job when the share since the last hour's +removals exceeds 10 percent; every fleet job that touches a standing box's miner calls it first. diff --git a/tools/fleet/lib/standing.py b/tools/fleet/lib/standing.py index 2fa58a1b8..c8f120c3d 100644 --- a/tools/fleet/lib/standing.py +++ b/tools/fleet/lib/standing.py @@ -15,6 +15,9 @@ box side (the supervisor loop). Every call goes through lib.box. same shape (card, VRAM, provider) with the label suffixed "-r", set it up, install the supervisor, mark the old row destroyed; the Devnet 2 roles take their seed from the registry loop(every=600) check, then rerent what died and update what is behind, every ten minutes + weight_check(labels) the 10 percent rule (20:00Z): the labels' share of the live devnet's weight (blue blocks per + key over the window, from the hub) plus what was removed in the last hour must stay under + 10 percent, or the job is refused; every job that stops or shares a standing miner calls it Nothing here destroys a standing box; destroy() in lib.box stays for the one-shot boxes. """ import os, sys, json, time, datetime, subprocess @@ -86,6 +89,40 @@ def rerent(label): Registry.patch(str(nid), standing=True, role=b.get("role", "live"), standing_since=now(), rerents=n, rerent_of=label) Registry.patch(iid, state="destroyed", destroyed_at=now(), destroyed_reason="host died, re-rented as " + new) return new +# ---- the 10 percent rule (20:00Z): never remove more than 10 percent of the live devnet's 30-day weight in any hour ---- +WEIGHT_LOG = os.path.join(ROOT, "weight-removals.jsonl") +def weight_shares(window_blocks=30*86400): + """Blue blocks per payout label over the window, from the hub's node (igneum-miner payouts); returns {label: share}.""" + hub = next((v for v in Registry.load().values() if v.get("hub") and v.get("state") != "destroyed"), None) + if not hub: raise SshError("no hub box") + b = Box(hub["ssh_host"], hub["ssh_port"], "hub-1", None, None, hub.get("provider")) + rc, out, err = b.run("/opt/igneum/pkg/bin/igneum-miner payouts grpc://127.0.0.1:26610 2>/dev/null | tail -400", 120) + rows = {} + for ln in out.splitlines(): + parts = ln.split() + nums = [p for p in parts if p.isdigit()] + if len(parts) >= 2 and nums: + rows[parts[0]] = rows.get(parts[0], 0) + int(nums[-1]) + tot = sum(rows.values()) or 1 + return {k: v / tot for k, v in rows.items()}, out[-600:] +def weight_check(labels, limit=0.10, hours=1.0): + """Refuses when the labels' weight plus what was removed in the last `hours` exceeds `limit`. Records an allowed removal.""" + shares, raw = weight_shares() + reg = Registry.load(); keys = {} + for l in labels: + iid, b = Registry.find(l); keys[l] = (b.get("wallet") or "").lower() + want = sum(shares.get(k, 0) for k in keys.values()) + sum(shares.get(l, 0) for l in labels) + since = time.time() - hours * 3600; recent = 0.0 + if os.path.exists(WEIGHT_LOG): + for ln in open(WEIGHT_LOG): + try: j = json.loads(ln) + except Exception: continue + if j.get("ts", 0) >= since: recent += float(j.get("share", 0)) + ok = (want + recent) <= limit + verdict = {"labels": labels, "share": round(want, 4), "removed_last_hour": round(recent, 4), "limit": limit, "ok": ok, "t": now()} + if ok: + with open(WEIGHT_LOG, "a") as f: f.write(json.dumps({"ts": time.time(), "labels": labels, "share": want}) + "\n") + return verdict def loop(every=600): dead = {} while True: @@ -112,3 +149,7 @@ if __name__ == "__main__": elif a[0] == "update": print(update(a[1])) elif a[0] == "rerent": print(rerent(a[1])) elif a[0] == "loop": loop(int(a[1]) if len(a) > 1 else 600) + elif a[0] == "weight_check": + v = weight_check(a[1:]); print(json.dumps(v)); sys.exit(0 if v["ok"] else 1) + elif a[0] == "weights": + sh, raw = weight_shares(); print(json.dumps(dict(sorted(sh.items(), key=lambda kv: -kv[1])[:40]), indent=1))