40 lines
7.4 KiB
Python
40 lines
7.4 KiB
Python
"""The 0.3.15 retry in the poisoned-peer form (shipper, 20:4xZ): usage retry-0315-v3.py <igneumd path> <sha256> <igneum-miner path>
|
|
New node p2-4090-3 (wiped datadir, mining; peers the hub, the seed box and pool-1), poisoned peer pool-1 as it stands (1026 blocks,
|
|
6615571c), restart case p2-3090-2 (wiped datadir, restarted at +5 min, the hub its only re-sync peer). Reads every minute for 12:
|
|
p2-4090-3's own log naming version 1026 as rejected and never 'Accepted ... via relay' of a 1026 block, the hands and the hub never
|
|
rejecting or disconnecting p2-4090-3, the hub holding a block by p2-4090-3's key mined in the window, p2-3090-2 back at the tip,
|
|
the hub's 'wrong block version' count split by peer."""
|
|
import sys, os, time, hashlib; sys.path.insert(0, "/Users/joshm/Projects/igneum-wt-gpu-fleet/tools/fleet"); from lib import Box, Registry
|
|
def st(): return time.strftime("%H:%M:%SZ", time.gmtime())
|
|
NODE, SHA, MINER = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
assert hashlib.sha256(open(NODE,"rb").read()).hexdigest()==SHA, "node sha"
|
|
WATCH="/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\\n' ' '"
|
|
KILL="pkill -9 -f '^bash in/box-standing.sh'; pkill -9 -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs'; pkill -9 -f '^/(opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir='; sleep 3"
|
|
hub=Box.from_registry("hub-1"); pool=Box.from_registry("pool-1"); A=Box.from_registry("p2-4090-3"); B=Box.from_registry("p2-3090-2")
|
|
# pool-1: the poisoned peer, as it stands (6615571c, its datadir), on its inbound port
|
|
rc,out,err=pool.run("cd /root/fleet && pgrep -f 'igneumd-6615571ced35f694 --devnet --appdir=' >/dev/null || { setsid nohup /root/fleet/in/igneumd-6615571ced35f694 --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:4463 --addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes </dev/null >> node.log 2>&1 & sleep 15; }; echo pool1 $(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' node.log | tail -1) "+WATCH, 60); print(st(), "poisoned peer:", out.replace("\n"," "), flush=True)
|
|
def hubsplit(): return hub.run("echo total=$(grep -c 'wrong block version' /root/fleet/node.log); awk '$2 >= \"'$(date -u -d '-12 min' +%H:%M:%S)'\"' /root/fleet/node.log | grep 'wrong block version' | grep -oE 'peer [0-9.]+' | sort | uniq -c | sort -rn | tr '\\n' ';'", 30)[1].replace("\n"," ").strip()
|
|
print(st(), "hub before:", hubsplit(), flush=True)
|
|
for box, l, peers in ((A, "p2-4090-3", "--addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --addpeer=213.173.98.36:18451"), (B, "p2-3090-2", "--addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611")):
|
|
path=box.install_payload(NODE, SHA); box.put([MINER], "/root/fleet/in/igneum-miner-new"); iid,r=Registry.find(l)
|
|
rc,out,err=box.run(f"cd /root/fleet && {KILL}; [ -d node ] && mv node node.pre-$(date -u +%H%M); mkdir -p node; mv node.log node.pre-$(date -u +%H%M).log 2>/dev/null; cp in/igneum-miner-new /opt/igneum/pkg/bin/igneum-miner.new && chmod +x /opt/igneum/pkg/bin/igneum-miner.new && mv /opt/igneum/pkg/bin/igneum-miner.new /opt/igneum/pkg/bin/igneum-miner; echo {path} > standing.node; IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host setsid nohup {path} --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 {peers} --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes </dev/null >> node.log 2>&1 & echo started T0=$(date -u +%T)", 90)
|
|
print(st(), l, out.replace("\n"," "), flush=True)
|
|
for i in range(40):
|
|
time.sleep(30); rc,out,err=box.run(WATCH, 30)
|
|
if "synced=true" in out and "blocks=0 " not in out: break
|
|
rc,out,err=box.run(f"cd /root/fleet && echo synced_after={(i+1)*30}s {WATCH} vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' node.log | tail -1) digest=$(grep -o 'digest: [0-9a-f]*' node.log | tail -1 | awk '{{print substr($2,1,8)}}'); cd mine 2>/dev/null || mkdir -p /root/fleet/mine/packs && cd /root/fleet/mine; rm -rf packs/devnet; /opt/igneum/pkg/bin/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >/dev/null 2>&1; setsid nohup /opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 {l} --worker /opt/igneum/pkg/bin/igneum-worker-cuda --worker-args '--device 0 --pack packs/devnet' --prepare-packs packs/prepare --exit-on-seed-change --evm-address {r.get('wallet')} --payout-label {l} --status-secs 60 </dev/null >> /root/fleet/out/miner-0.log 2>&1 & echo miner started; /opt/igneum/pkg/bin/igneum-miner key-hash {l} 2>/dev/null | grep -oE '[0-9a-f]{{64}}' | head -1", 120)
|
|
print(st(), l, "synced + mining:", out.replace("\n"," "), flush=True)
|
|
if l=="p2-4090-3": keyA=out.strip().split()[-1]
|
|
T0=time.time(); restarted=False
|
|
while time.time()-T0 < 720:
|
|
time.sleep(60)
|
|
rc,a,err=A.run("echo rej1026=$(grep -c 'wrong block version' /root/fleet/node.log) got_reject=$(grep -c 'got reject message' /root/fleet/node.log) relayed1026=$(grep -ciE 'version 1026.*(accept|relay)|accepted.*1026' /root/fleet/node.log) peers_now=$(grep -oE 'Connected to (outgoing|incoming) peer [0-9.:]+' /root/fleet/node.log | awk '{print $NF}' | cut -d: -f1 | sort -u | tr '\\n' ','); grep -E 'wrong block version|1026' /root/fleet/node.log | tail -1 | cut -c12-200; "+WATCH, 40)
|
|
rc,b,err=B.run(WATCH+"; echo got_reject=$(grep -c 'got reject message' /root/fleet/node.log)", 30)
|
|
print(st(), f"+{int(time.time()-T0)}s A:", a.replace("\n"," | ").strip(), "| B:", b.replace("\n"," ").strip(), "| hub:", hubsplit(), flush=True)
|
|
if not restarted and time.time()-T0 >= 300:
|
|
rc,out,err=B.run("cd /root/fleet && pkill -9 -f '^/root/fleet/in/igneumd-[0-9a-f]{16} --devnet --appdir='; sleep 20; BIN=$(cat standing.node); IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host setsid nohup $BIN --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=213.173.107.74:16516 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes </dev/null >> node.log 2>&1 & for i in $(seq 1 24); do sleep 5; w=$("+WATCH+"); case \"$w\" in *synced=true*) break;; esac; done; echo resync_after=$((i*5+20))s $w; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve'", 200)
|
|
print(st(), "B restart (hub-only re-sync):", out.replace("\n"," "), flush=True); restarted=True
|
|
hub_pan=hub.run("echo hub_panics=$(grep -ciE 'panicked at consensus' /root/fleet/node.log)", 20)[1].strip(); print(st(), hub_pan, flush=True)
|
|
rc,out,err=hub.run(f"timeout 150 /opt/igneum/pkg/bin/igneum-miner inspect 900 grpc://127.0.0.1:26610 2>&1 | grep -c 'voteKeyHash={keyA}'; awk '$2 >= \"'$(date -u -d '-14 min' +%H:%M:%S)'\"' /root/fleet/node.log | grep -E 'wrong block version|disconnecting' | grep -c '216.232.211.246'", 170)
|
|
print(st(), f"hub holds {out.split()[0] if out.split() else '?'} blocks by p2-4090-3's key {keyA[:16]} in its last 900; hub rejects/disconnects naming p2-4090-3 in the window: {out.split()[1] if len(out.split())>1 else '?'}", flush=True)
|
|
print(st(), "RETRY V3 done", flush=True)
|