36 lines
3 KiB
Python
36 lines
3 KiB
Python
#!/usr/bin/env python3
|
|
"""Disk on every live fleet box (6 October 2026, after the hub died on a full disk): one df per box in parallel, written to
|
|
~/Desktop/fleet/disk.json ({label: {pct, avail_gb, total_gb, t}}) for the fleet page's disk column; any box at or over 85
|
|
percent posts one line to Discord #incidents through tools/community/discord-hooks.mjs (incident open, the fleet's own
|
|
sentence), at most one line per box per hour (state in ~/Desktop/fleet/disk-alerts.json). Run from page.py's cadence or
|
|
by hand: disk-sweep.py [--no-alert]."""
|
|
import sys, os, json, time, subprocess
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); from lib import Box, Registry
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
ROOT = os.path.expanduser("~/Desktop/fleet"); OUT = os.path.join(ROOT, "disk.json"); ALERTS = os.path.join(ROOT, "disk-alerts.json")
|
|
HOOKS = "/Users/joshm/Projects/igneum/tools/community/discord-hooks.mjs"
|
|
def now(): return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
def one(item):
|
|
iid, b = item
|
|
try:
|
|
box = Box(b["ssh_host"], b["ssh_port"], b["label"], iid, None, b.get("provider"), b.get("ssh_user", "root"))
|
|
rc, out, err = box.run("df -B1 / | tail -1 | awk '{print $2, $4, $5}'", 25)
|
|
tot, avail, pct = out.split()
|
|
return b["label"], {"pct": int(pct.rstrip("%")), "avail_gb": round(int(avail) / 1e9, 1), "total_gb": round(int(tot) / 1e9), "t": now()}
|
|
except Exception as e: return b["label"], {"pct": None, "error": str(e)[:60], "t": now()}
|
|
def sweep(alert=True):
|
|
reg = Registry.load(); live = [(i, b) for i, b in reg.items() if b.get("state") != "destroyed" and b.get("ssh_host")]
|
|
with ThreadPoolExecutor(20) as ex: res = dict(ex.map(one, live))
|
|
json.dump(res, open(OUT, "w"), indent=1)
|
|
if alert:
|
|
st = json.load(open(ALERTS)) if os.path.exists(ALERTS) else {}
|
|
for label, d in res.items():
|
|
if d.get("pct") is not None and d["pct"] >= 85 and time.time() - st.get(label, 0) > 3600:
|
|
what = f"Fleet box {label} disk at {d['pct']} percent ({d['avail_gb']} GB free of {d['total_gb']})"
|
|
r = subprocess.run(["node", HOOKS, "incident", "open", "--what", what, "--affected", "one rented fleet box; no user-facing effect unless it is the hub (then a slower first join for home miners, no loss)", "--doing", "the standing supervisor prunes the prover's segment exports and trims the node log every ten minutes; the fleet agent reads the box", "--id", f"disk-{label}-{int(time.time())}", "--live"], capture_output=True, text=True, timeout=60)
|
|
st[label] = time.time(); print("alert", label, d["pct"], "rc", r.returncode, (r.stdout + r.stderr)[-120:].replace("\n", " "))
|
|
json.dump(st, open(ALERTS, "w"))
|
|
return res
|
|
if __name__ == "__main__":
|
|
res = sweep(alert="--no-alert" not in sys.argv)
|
|
for l, d in sorted(res.items(), key=lambda kv: -(kv[1].get("pct") or 0)): print(f"{l:<12} {str(d.get('pct')):>4}% {d.get('avail_gb', '?')} GB free")
|