#!/usr/bin/env python3 """Disk on every live fleet box (6 October 2026, after the hub died on a full disk): one df per box in parallel, written to ~/Desktop/fleet/disk.json ({label: {pct, avail_gb, total_gb, t}}) for the fleet page's disk column; any box at or over 85 percent posts one line to Discord #incidents through tools/community/discord-hooks.mjs (incident open, the fleet's own sentence), at most one line per box per hour (state in ~/Desktop/fleet/disk-alerts.json). Run from page.py's cadence or by hand: disk-sweep.py [--no-alert].""" import sys, os, json, time, subprocess sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); from lib import Box, Registry from concurrent.futures import ThreadPoolExecutor ROOT = os.path.expanduser("~/Desktop/fleet"); OUT = os.path.join(ROOT, "disk.json"); ALERTS = os.path.join(ROOT, "disk-alerts.json") HOOKS = "/Users/joshm/Projects/igneum/tools/community/discord-hooks.mjs" def now(): return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) def one(item): iid, b = item try: box = Box(b["ssh_host"], b["ssh_port"], b["label"], iid, None, b.get("provider"), b.get("ssh_user", "root")) rc, out, err = box.run("df -B1 / | tail -1 | awk '{print $2, $4, $5}'", 25) tot, avail, pct = out.split() return b["label"], {"pct": int(pct.rstrip("%")), "avail_gb": round(int(avail) / 1e9, 1), "total_gb": round(int(tot) / 1e9), "t": now()} except Exception as e: return b["label"], {"pct": None, "error": str(e)[:60], "t": now()} def sweep(alert=True): reg = Registry.load(); live = [(i, b) for i, b in reg.items() if b.get("state") != "destroyed" and b.get("ssh_host")] with ThreadPoolExecutor(20) as ex: res = dict(ex.map(one, live)) json.dump(res, open(OUT, "w"), indent=1) if alert: st = json.load(open(ALERTS)) if os.path.exists(ALERTS) else {} for label, d in res.items(): if d.get("pct") is not None and d["pct"] >= 85 and time.time() - st.get(label, 0) > 3600: what = f"Fleet box {label} disk at {d['pct']} percent ({d['avail_gb']} GB free of {d['total_gb']})" r = subprocess.run(["node", HOOKS, "incident", "open", "--what", what, "--affected", "one rented fleet box; no user-facing effect unless it is the hub (then a slower first join for home miners, no loss)", "--doing", "the standing supervisor prunes the prover's segment exports and trims the node log every ten minutes; the fleet agent reads the box", "--id", f"disk-{label}-{int(time.time())}", "--live"], capture_output=True, text=True, timeout=60) st[label] = time.time(); print("alert", label, d["pct"], "rc", r.returncode, (r.stdout + r.stderr)[-120:].replace("\n", " ")) json.dump(st, open(ALERTS, "w")) return res if __name__ == "__main__": res = sweep(alert="--no-alert" not in sys.argv) for l, d in sorted(res.items(), key=lambda kv: -(kv[1].get("pct") or 0)): print(f"{l:<12} {str(d.get('pct')):>4}% {d.get('avail_gb', '?')} GB free")