40 lines
3.1 KiB
Python
40 lines
3.1 KiB
Python
#!/usr/bin/env python3
|
|
"""The phase-2 node swap over the fleet, one call per shipper word:
|
|
swap.py binary on "hands on 0.3.13": every phase-2 box (the hub first) restarts on igneumd-0313 with the ten-field file
|
|
swap.py file on "hands on b18ed271": the thirteen-field file swapped in and the node restarted; the replay timed
|
|
swap.py status the swap log's last line and the exec status per box
|
|
The expected sha256 and digests are pinned here; a box whose binary or digest differs refuses and is reported."""
|
|
import sys, os, time, json
|
|
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))); import fleet
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
SHA = "d6350586fe837b1f71696546628ec071b6accce2531aef776cf2b1e5487a8cdc"
|
|
DIG = {"binary": "7bd98cc4118616455709d5e32a30b799e6e67caa42d2b5d09875cd49848a7ed7", "file": "b18ed271f75dd46406d230f4156c37472127415a4c32c558bac662f6f840e61c"}
|
|
def targets():
|
|
reg = fleet.load()
|
|
t = [(iid, b) for iid, b in reg.items() if b.get("state") != "destroyed" and b.get("ssh_ok") and (b.get("phase") in ("1", "2") or b.get("hub"))]
|
|
return sorted(t, key=lambda x: (not x[1].get("hub"), x[1]["label"]))
|
|
def start(step):
|
|
hub = next((b for _, b in targets() if b.get("hub")), None)
|
|
env = f"STEP={step} NODE_SHA256={SHA} EXPECT_DIGEST={DIG[step]} HUB_PEER={hub['hub_peer']} HUB_SSH={hub['ssh_host']} HUB_PORT={hub['ssh_port']}" if hub else f"STEP={step} NODE_SHA256={SHA} EXPECT_DIGEST={DIG[step]}"
|
|
def one(x):
|
|
iid, b = x
|
|
fleet.scp(b, [os.path.join(fleet.HERE, "box-node-swap.sh"), os.path.join(fleet.HERE, "box-kill.sh")], "/root/fleet/in/")
|
|
r, out, err = fleet.ssh(b, f"cd /root/fleet && chmod +x in/box-node-swap.sh && mv out/node-swap.log out/node-swap-{step}-prev.log 2>/dev/null; {env} setsid nohup in/box-node-swap.sh </dev/null >/dev/null 2>&1 & sleep 1; echo started", timeout=60)
|
|
fleet.patch(iid, stage="swap-" + step, doing=f"0.3.13 swap, step {step}: restart, digest check, the exec replay timed")
|
|
return b["label"], out.strip() or err[:60]
|
|
ts = targets()
|
|
if hub: # the hub first, then the rest in parallel 20 s later (it is every box's second peer)
|
|
print(one(next(x for x in ts if x[1].get("hub")))); time.sleep(20)
|
|
with ThreadPoolExecutor(12) as ex:
|
|
for lab, out in ex.map(one, [x for x in ts if not x[1].get("hub")]): print(lab, out)
|
|
def status():
|
|
def one(x):
|
|
iid, b = x
|
|
r, out, err = fleet.ssh(b, "grep -E '^RESULT (node_started|swap_done|swap_failed|datadir_failed)' /root/fleet/out/node-swap.log 2>/dev/null | tail -2 | cut -c1-200; grep -E '^RESULT replay' /root/fleet/out/node-swap.log 2>/dev/null | tail -1 | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //' | cut -c1-120; grep -E '^RESULT exec_status' /root/fleet/out/node-swap.log 2>/dev/null | tail -1 | cut -c1-200", timeout=40)
|
|
return b["label"], out.strip().replace("\n", " | ")
|
|
with ThreadPoolExecutor(12) as ex:
|
|
for lab, out in ex.map(one, targets()): print(f"{lab:<12} {out}")
|
|
if __name__ == "__main__":
|
|
c = sys.argv[1]
|
|
if c in ("binary", "file"): start(c)
|
|
elif c == "status": status()
|