diff --git a/tools/fleet/box-node-swap.sh b/tools/fleet/box-node-swap.sh index edecdf771..fd57d1210 100755 --- a/tools/fleet/box-node-swap.sh +++ b/tools/fleet/box-node-swap.sh @@ -19,7 +19,7 @@ echo "$NODE_SHA256 $F/in/igneumd-0313" | sha256sum -c - >/dev/null || { echo "R chmod +x $F/in/igneumd-0313 t0=$(date +%s) bash $F/in/box-kill.sh >/dev/null 2>&1 # every stage process; the node is stopped on the next line -pkill -x igneumd; pkill -f igneumd-0313; sleep 4; pkill -9 -x igneumd 2>/dev/null; pkill -9 -f igneumd-0313 2>/dev/null; sleep 1 +pkill -x igneumd; pkill -f '^/opt/igneum/pkg/bin/igneumd-0313'; sleep 4; pkill -9 -x igneumd 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneumd-0313' 2>/dev/null; sleep 1 TGZ_SHA=e67cc6493cd86dc0a993c3e1005b1df43b661fc4973ef7f0095f602f1536cd85 if [ ! -f $F/node/.full-history ]; then [ "$(sha256sum $F/observer-datadir.tgz 2>/dev/null | cut -c1-64)" = "$TGZ_SHA" ] || rm -f $F/observer-datadir.tgz # a partial pre-pull is not a tarball @@ -38,7 +38,7 @@ IGNEUM_PROOF_VERIFIER=$HOST nohup $B/igneumd-0313 --devnet --appdir=$F/node --rp t1=$(date +%s); sleep 12 d="$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print $2}')" ok=DIGEST_MISMATCH; [ "$d" = "$EXPECT_DIGEST" ] && ok=digest_ok -echo "RESULT node_started $(stamp) step=$STEP pid=$(pgrep -f igneumd-0313 | head -1) version=$($B/igneumd-0313 --version 2>&1 | head -1) digest=${d:0:16} expected=${EXPECT_DIGEST:0:16} $ok exec_restart=\"$(grep -o 'Exec restart from the override file.*' $F/node.log | tail -1 | cut -c1-120)\"" +echo "RESULT node_started $(stamp) step=$STEP pid=$(pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' | head -1) version=$($B/igneumd-0313 --version 2>&1 | head -1) digest=${d:0:16} expected=${EXPECT_DIGEST:0:16} $ok exec_restart=\"$(grep -o 'Exec restart from the override file.*' $F/node.log | tail -1 | cut -c1-120)\"" [ "$ok" = digest_ok ] || { echo "RESULT swap_failed digest mismatch"; exit 2; } watch() { $B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1; } if [ "$STEP" = binary ]; then diff --git a/tools/fleet/canary-next.sh b/tools/fleet/canary-next.sh index 8c68882ef..f070a2bf5 100755 --- a/tools/fleet/canary-next.sh +++ b/tools/fleet/canary-next.sh @@ -29,7 +29,7 @@ hub=next(v for v in reg.values() if v.get("hub") and v.get("state")!="destroyed" PY ) HUB="$(printf '%s\n' "${ROWS[@]}" | awk '$1=="hub-1"')"; HUB_HOST="$(awk '{print $2}' <<< "$HUB")"; HUB_PORT="$(awk '{print $3}' <<< "$HUB")"; HUB_PEER="$(awk '{print $5}' <<< "$HUB")" -SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } +SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } read_box() { SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); ex=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getExecStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"executedTip\",\"0x0\"),16))" 2>/dev/null); v=$(pgrep -fa "^/opt/igneum/pkg/bin/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "got reject message|PoW rejected|block rejected|invalid block" /root/fleet/node.log 2>/dev/null); echo "${d:-0} ${e:-0 0} ${ex:-0} ${ver:-none} ${rej:-0}"' 2>/dev/null; } root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; }; rej0() { cat "$WORK/rej0.$1" 2>/dev/null || echo 0; } @@ -38,7 +38,7 @@ for r in "${ROWS[@]}"; do set -- $r; x="$(read_box "$2" "$3")"; echo " $1: daa= # install: the binary swap with the override file kept (box-node-swap.sh STEP=binary, which refuses on a digest change) T0=$(date +%s) for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue - okc=0; for try in 1 2 3; do scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -P "$3" "$LOCAL" "root@$2:/root/fleet/in/igneumd-0313" >/dev/null 2>&1 && { okc=1; break; }; sleep 5; done # Vast's ssh proxy drops a connection now and then (17:14Z); three tries before a FAIL + okc=0; for try in 1 2 3; do scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -P "$3" "$LOCAL" "root@$2:/root/fleet/in/igneumd-0313" >/dev/null 2>&1 && { okc=1; break; }; sleep 5; done # Vast's ssh proxy drops a connection now and then (17:14Z); three tries before a FAIL [[ $okc == 1 ]] || fail "$1: scp failed three times" oks=0; for try in 1 2 3; do SSH "$2" "$3" "cd /root/fleet && chmod +x in/igneumd-0313 && [ \"\$(sha256sum in/igneumd-0313 | cut -c1-64)\" = $SHA ] && mv out/node-swap.log out/node-swap-canary-prev.log 2>/dev/null; STEP=binary NODE_SHA256=$SHA EXPECT_DIGEST=b18ed271f75dd46406d230f4156c37472127415a4c32c558bac662f6f840e61c HUB_PEER=$HUB_PEER HUB_SSH=$HUB_HOST HUB_PORT=$HUB_PORT setsid nohup in/box-node-swap.sh /dev/null 2>&1 & echo swapped" >/dev/null 2>&1 && { oks=1; break; }; sleep 5; done [[ $oks == 1 ]] || fail "$1: install failed three times" diff --git a/tools/fleet/canary.sh b/tools/fleet/canary.sh index ed8e50076..4061189b3 100755 --- a/tools/fleet/canary.sh +++ b/tools/fleet/canary.sh @@ -29,7 +29,7 @@ hub=next(v for v in reg.values() if v.get("hub") and v.get("state")!="destroyed" PY ) HUB="$(printf '%s\n' "${ROWS[@]}" | awk '$1=="hub-1"')"; HUB_HOST="$(awk '{print $2}' <<< "$HUB")"; HUB_PORT="$(awk '{print $3}' <<< "$HUB")"; HUB_PEER="$(awk '{print $5}' <<< "$HUB")" -SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } +SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } read_box() { SSH "$1" "$2" 'B=/opt/igneum/pkg/bin; w=$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o "blocks=[0-9]*.*synced=[a-z]*" | tail -1); d=$(grep -o "daa=[0-9]*" <<< "$w" | cut -d= -f2); e=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getProvingStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"tipDaa\",\"0x0\"),16), x.get(\"v1\",{}).get(\"paidSegments\",0))" 2>/dev/null); ex=$(curl -s -m 6 -X POST -H "Content-Type: application/json" --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"igneum_getExecStatus\",\"params\":[]}" http://127.0.0.1:26790/ | python3 -c "import sys,json; r=sys.stdin.read(); x=json.loads(r).get(\"result\",{}) if r.strip() else {}; print(int(x.get(\"executedTip\",\"0x0\"),16))" 2>/dev/null); v=$(pgrep -fa "^/opt/igneum/pkg/bin/igneumd" | head -1 | awk "{print \$2}"); ver=$(sha256sum "$v" 2>/dev/null | cut -c1-16)_$($v --version 2>&1 | head -1 | tr " " "_"); rej=$(grep -cE "got reject message|PoW rejected|block rejected|invalid block" /root/fleet/node.log 2>/dev/null); echo "${d:-0} ${e:-0 0} ${ex:-0} ${ver:-none} ${rej:-0}"' 2>/dev/null; } root_at() { SSH "$1" "$2" "curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$3\",false]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; b=json.load(sys.stdin).get(\"result\") or {}; print(b.get(\"stateRoot\",\"none\"))'" 2>/dev/null; } WORK="$(mktemp -d)"; paid0() { cat "$WORK/paid0.$1" 2>/dev/null || echo 0; }; rej0() { cat "$WORK/rej0.$1" 2>/dev/null || echo 0; } @@ -38,7 +38,7 @@ for r in "${ROWS[@]}"; do set -- $r; x="$(read_box "$2" "$3")"; echo " $1: daa= # install: the binary swap with the override file kept (box-node-swap.sh STEP=binary, which refuses on a digest change) T0=$(date +%s) for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue - okc=0; for try in 1 2 3; do scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -P "$3" "$LOCAL" "root@$2:/root/fleet/in/igneumd-0313" >/dev/null 2>&1 && { okc=1; break; }; sleep 5; done # Vast's ssh proxy drops a connection now and then (17:14Z); three tries before a FAIL + okc=0; for try in 1 2 3; do scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -P "$3" "$LOCAL" "root@$2:/root/fleet/in/igneumd-0313" >/dev/null 2>&1 && { okc=1; break; }; sleep 5; done # Vast's ssh proxy drops a connection now and then (17:14Z); three tries before a FAIL [[ $okc == 1 ]] || fail "$1: scp failed three times" oks=0; for try in 1 2 3; do SSH "$2" "$3" "cd /root/fleet && chmod +x in/igneumd-0313 && [ \"\$(sha256sum in/igneumd-0313 | cut -c1-64)\" = $SHA ] && mv out/node-swap.log out/node-swap-canary-prev.log 2>/dev/null; STEP=binary NODE_SHA256=$SHA EXPECT_DIGEST=b18ed271f75dd46406d230f4156c37472127415a4c32c558bac662f6f840e61c HUB_PEER=$HUB_PEER HUB_SSH=$HUB_HOST HUB_PORT=$HUB_PORT setsid nohup in/box-node-swap.sh /dev/null 2>&1 & echo swapped" >/dev/null 2>&1 && { oks=1; break; }; sleep 5; done [[ $oks == 1 ]] || fail "$1: install failed three times" diff --git a/tools/fleet/devnet2-gate.sh b/tools/fleet/devnet2-gate.sh index edcfc8e7b..a6ee990f9 100755 --- a/tools/fleet/devnet2-gate.sh +++ b/tools/fleet/devnet2-gate.sh @@ -37,8 +37,8 @@ for iid,b in reg.items(): PY ) [[ ${#BOXES[@]} -gt 0 ]] || fail "no Devnet 2 boxes in the registry" -SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } -SCP() { scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -P "$2" "${@:3}" "root@$1:/root/fleet/in/" >/dev/null; } +SSH() { ssh -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -o BatchMode=yes -p "$2" "root@$1" "${@:3}"; } +SCP() { scp -i ~/.ssh/igneum-fleet -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=20 -o ServerAliveInterval=10 -o ServerAliveCountMax=3 -P "$2" "${@:3}" "root@$1:/root/fleet/in/" >/dev/null; } seed_line="$(printf '%s\n' "${BOXES[@]}" | awk '$4==1 {print; exit}')"; SEED_HOST="$(awk '{print $2}' <<< "$seed_line")"; SEED_PORT="$(awk '{print $3}' <<< "$seed_line")" SEED_PEER="$(python3 - <<'PY' import json, os; reg=json.load(open(os.path.expanduser("~/Desktop/fleet/boxes.json")))