#!/usr/bin/env python3 """The phase-2 node swap over the fleet, one call per shipper word: swap.py binary on "hands on 0.3.13": every phase-2 box (the hub first) restarts on igneumd-0313 with the ten-field file swap.py file on "hands on b18ed271": the thirteen-field file swapped in and the node restarted; the replay timed swap.py status the swap log's last line and the exec status per box The expected sha256 and digests are pinned here; a box whose binary or digest differs refuses and is reported.""" import sys, os, time, json sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); import fleet from concurrent.futures import ThreadPoolExecutor SHA = "d6350586fe837b1f71696546628ec071b6accce2531aef776cf2b1e5487a8cdc" DIG = {"binary": "7bd98cc4118616455709d5e32a30b799e6e67caa42d2b5d09875cd49848a7ed7", "file": "b18ed271f75dd46406d230f4156c37472127415a4c32c558bac662f6f840e61c"} def targets(): reg = fleet.load() t = [(iid, b) for iid, b in reg.items() if b.get("state") != "destroyed" and b.get("ssh_ok") and (b.get("phase") in ("1", "2") or b.get("hub"))] return sorted(t, key=lambda x: (not x[1].get("hub"), x[1]["label"])) def start(step): hub = next((b for _, b in targets() if b.get("hub")), None) env = f"STEP={step} NODE_SHA256={SHA} EXPECT_DIGEST={DIG[step]} HUB_PEER={hub['hub_peer']} HUB_SSH={hub['ssh_host']} HUB_PORT={hub['ssh_port']}" if hub else f"STEP={step} NODE_SHA256={SHA} EXPECT_DIGEST={DIG[step]}" def one(x): iid, b = x fleet.scp(b, [os.path.join(fleet.HERE, "box-node-swap.sh"), os.path.join(fleet.HERE, "box-kill.sh")], "/root/fleet/in/") r, out, err = fleet.ssh(b, f"cd /root/fleet && chmod +x in/box-node-swap.sh && mv out/node-swap.log out/node-swap-{step}-prev.log 2>/dev/null; {env} setsid nohup in/box-node-swap.sh /dev/null 2>&1 & sleep 1; echo started", timeout=60) fleet.patch(iid, stage="swap-" + step, doing=f"0.3.13 swap, step {step}: restart, digest check, the exec replay timed") return b["label"], out.strip() or err[:60] ts = targets() if hub: # the hub first, then the rest in parallel 20 s later (it is every box's second peer) print(one(next(x for x in ts if x[1].get("hub")))); time.sleep(20) with ThreadPoolExecutor(12) as ex: for lab, out in ex.map(one, [x for x in ts if not x[1].get("hub")]): print(lab, out) def status(): def one(x): iid, b = x r, out, err = fleet.ssh(b, "grep -E '^RESULT (node_started|swap_done|swap_failed|datadir_failed)' /root/fleet/out/node-swap.log 2>/dev/null | tail -2 | cut -c1-200; grep -E '^RESULT replay' /root/fleet/out/node-swap.log 2>/dev/null | tail -1 | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //' | cut -c1-120; grep -E '^RESULT exec_status' /root/fleet/out/node-swap.log 2>/dev/null | tail -1 | cut -c1-200", timeout=40) return b["label"], out.strip().replace("\n", " | ") with ThreadPoolExecutor(12) as ex: for lab, out in ex.map(one, targets()): print(f"{lab:<12} {out}") if __name__ == "__main__": c = sys.argv[1] if c in ("binary", "file"): start(c) elif c == "status": status()