38 lines
3.7 KiB
Python
38 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Retries the two 8-card rigs on Vast and RunPod every ten minutes until both are held or RIG_RETRY_UNTIL passes."""
|
|
import time, sys, os, secrets, datetime
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
|
import fleet, vast, runpod
|
|
START = ["bash", "-c", "apt-get update -qq >/dev/null 2>&1; DEBIAN_FRONTEND=noninteractive apt-get install -y -qq openssh-server >/dev/null 2>&1; mkdir -p /run/sshd /root/.ssh; echo \"$PUBLIC_KEY\" > /root/.ssh/authorized_keys; chmod 700 /root/.ssh; chmod 600 /root/.ssh/authorized_keys; sed -i 's/^#\\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config; exec /usr/sbin/sshd -D"]
|
|
def log(s): print(datetime.datetime.utcnow().strftime("%H:%M:%SZ"), s, flush=True)
|
|
def have(label): return any(b["label"] == label and b.get("state") != "destroyed" for b in fleet.load().values())
|
|
def try_vast(gpu, label, archs):
|
|
offers = [o for o in vast.search(gpu, n=12, min_cores=16, gpus=8, min_disk=150) if (o.get("cpu_ram") or 0)/1024 >= 96]
|
|
offers.sort(key=lambda o: (o.get("reliability2", 0) < 0.98, o["dph_total"]))
|
|
for o in offers[:3]:
|
|
try: d = vast.call("PUT", f"/v0/asks/{o['id']}/", {"client_id": "me", "image": vast.IMAGE, "disk": 150, "label": label, "runtype": "ssh", "cancel_unavail": True})
|
|
except SystemExit as e: log(f"{label} vast: {str(e)[-60:]}"); return False
|
|
if not d.get("success"): continue
|
|
iid = d["new_contract"]; time.sleep(40)
|
|
if not any(str(i.get("id")) == str(iid) for i in vast.instances()): log(f"{label} vast {iid} cancelled"); continue
|
|
vast.ledger({"t": fleet.now(), "event": "rent", "instance": iid, "offer": o["id"], "label": label, "gpu": gpu, "num_gpus": 8, "dph": o["dph_total"], "disk": 150, "geo": o.get("geolocation")})
|
|
fleet.patch(iid, label=label, card=gpu, vram_mb=o.get("gpu_ram"), archs=archs, offer=o["id"], dph=o["dph_total"], num_gpus=8, provider="vast", phase="3", state="renting", rented_at=fleet.now(), wallet="0x"+secrets.token_hex(20), geo=o.get("geolocation"), doing=f"phase 3: the 8x {gpu.split()[-1]} rig")
|
|
log(f"{label} -> vast {iid} ${o['dph_total']:.2f}/h {o.get('geolocation')}"); return True
|
|
return False
|
|
def try_runpod(gpu, label, archs):
|
|
for cloud in ("SECURE", "COMMUNITY"):
|
|
body = {"name": label, "imageName": "nvidia/cuda:12.8.1-devel-ubuntu24.04", "gpuTypeIds": [gpu], "gpuCount": 8, "containerDiskInGb": 120, "volumeInGb": 0, "cloudType": cloud, "ports": ["22/tcp"], "env": {"PUBLIC_KEY": runpod.PUB}, "supportPublicIp": True, "computeType": "GPU", "dockerStartCmd": START}
|
|
try: d = runpod.call("POST", "/pods", body)
|
|
except SystemExit as e: continue
|
|
iid = d.get("id")
|
|
runpod.ledger({"t": fleet.now(), "event": "rent", "provider": "runpod", "instance": iid, "label": label, "gpu": gpu, "num_gpus": 8, "dph": d.get("costPerHr"), "disk": 120})
|
|
fleet.patch(iid, label=label, card=gpu.replace("NVIDIA GeForce ", ""), vram_mb=32607 if "5090" in gpu else 24564, archs=archs, dph=d.get("costPerHr") or 0, num_gpus=8, provider="runpod", phase="3", state="renting", rented_at=fleet.now(), wallet="0x"+secrets.token_hex(20), cloud=cloud, doing=f"phase 3: the 8x {gpu.split()[-1]} rig")
|
|
log(f"{label} -> runpod {iid} {cloud} ${d.get('costPerHr')}/h"); return True
|
|
return False
|
|
until = float(os.environ.get("RIG_RETRY_UNTIL", time.time() + 6 * 3600))
|
|
while time.time() < until:
|
|
for gpu, label, archs, rpg in (("RTX 4090", "rig-4090x8", "89", "NVIDIA GeForce RTX 4090"), ("RTX 5090", "rig-5090x8", "120", "NVIDIA GeForce RTX 5090")):
|
|
if have(label): continue
|
|
if not try_vast(gpu, label, archs): try_runpod(rpg, label, archs)
|
|
if have("rig-4090x8") and have("rig-5090x8"): log("both rigs held"); break
|
|
time.sleep(600)
|