Merge branch 'gpu-fleet' into HEAD
This commit is contained in:
igneum-josh
commit
a4a27fae09
27 changed files with 51 additions and 51 deletions
|
|
@ -17,14 +17,14 @@ if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then
|
|||
curl -fsSL -o $F/pkg.tgz "https://dl.igneum.network$PKG_PATH" && echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null && mkdir -p /opt/igneum/pkg && tar -C /opt/igneum/pkg --strip-components=1 -xzf $F/pkg.tgz || { echo "RESULT dn2_failed package"; exit 2; }
|
||||
fi
|
||||
B=/opt/igneum/pkg/bin; cp $F/in/dn2-override.json $F/dn2-override.json
|
||||
pkill -9 -f '^/root/fleet/in/igneumd-(0313|[0-9a-f]{16}) ' 2>/dev/null; pkill -9 -f "^/opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:$RPC_PORT " 2>/dev/null; sleep 2 # the Devnet 2 node only, never igneumd-v4 (the rehearsal node beside it, 19:00Z)
|
||||
pkill -9 -f '^[/]root/fleet/in/igneumd-(0313|[0-9a-f]{16}) ' 2>/dev/null; pkill -9 -f "^/opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:$RPC_PORT " 2>/dev/null; sleep 2 # the Devnet 2 node only, never igneumd-v4 (the rehearsal node beside it, 19:00Z)
|
||||
[ "${FRESH:-0}" = 1 ] && { rm -rf $F/dn2; mkdir -p $F/dn2; [ -s $F/dn2-node.log ] && mv $F/dn2-node.log $F/dn2-node.prev.log; echo "RESULT dn2_fresh $(stamp) appdir wiped for a new genesis"; } # never the script's own pattern (17:18Z: dn2-kill.sh killed its caller)
|
||||
PEER=""; for sd in ${SEED//,/ }; do PEER="$PEER --addpeer=$sd"; done # SEED may be a comma list (run A2: two relays per box)
|
||||
# the statement's program ids (proving agent, 22:1xZ): on 4c6b129d they come from the environment or the verifier host, else zero, and a zero id refuses every segment record
|
||||
env ${BPS:+IGNEUMD_DEVNET_BPS=$BPS} IGNEUM_PROOF_PROGRAM_IDS="${PROGRAM_IDS:-0x2b1a81cb413236cf063077b46ed3111628f6c41036bcf6e23ee4cbbf5679ef7a,0x474678f35f7545db28055d5e5bbc308231d84a5a072202087a2a8d5b09123896}" IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host nohup $NODE_BIN --devnet --devnet-suffix=2 --appdir=$F/dn2 --rpclisten=0.0.0.0:$RPC_PORT --evm-rpclisten=127.0.0.1:$EVM_PORT --listen=0.0.0.0:$P2P_PORT $PEER --override-params-file=$F/dn2-override.json --nodnsseed --disable-upnp --nologfiles --yes ${UNSYNCED:+--enable-unsynced-mining} ${NODE_EXTRA:-} >> $F/dn2-node.log 2>&1 &
|
||||
# (--enable-unsynced-mining: a fresh chain's nodes start unsynced and must mine anyway, Reject(IsInIBD) on the seed at 16:52Z; a comment put inside this line at 17:00Z swallowed the redirect and the ampersand, so the node ran in the foreground and the script never reached the miner)
|
||||
sleep 10
|
||||
echo "RESULT dn2_node $(stamp) pid=$(pgrep -f '^/root/fleet/in/igneumd-(0313|[0-9a-f]{16}) ' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)"
|
||||
echo "RESULT dn2_node $(stamp) pid=$(pgrep -f '^[/]root/fleet/in/igneumd-(0313|[0-9a-f]{16}) ' | head -1) version=$($NODE_BIN --version 2>&1 | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/dn2-node.log | tail -1 | awk '{print substr($2,1,16)}') network=$(grep -oiE 'igneum-devnet-2[^ ,]*' $F/dn2-node.log | head -1) genesis=$(grep -oiE 'genesis [0-9a-f]{16}' $F/dn2-node.log | head -1)"
|
||||
for i in $(seq 1 30); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:$RPC_PORT 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [ -n "$w" ] && break; sleep 5; done
|
||||
echo "RESULT dn2_watch $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //')"
|
||||
cd $F/mine && rm -rf packs/dn2 && $B/igneum-miner export-pack grpc://127.0.0.1:$RPC_PORT packs/dn2 > $OUT/dn2-export-pack.log 2>&1
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ rm -rf packs/devnet; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/de
|
|||
nohup $B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/devnet" \
|
||||
--prepare-packs packs/prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 10 > $OUT/ember-miner.log 2>&1 &
|
||||
MPID=$!; cd $F
|
||||
cleanup() { nvidia-smi -i 0 -rgc >/dev/null 2>&1; [ "$PL_OK" = 1 ] && nvidia-smi -i 0 -pl "$PDEF" >/dev/null 2>&1; kill $MPID 2>/dev/null; pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; }
|
||||
cleanup() { nvidia-smi -i 0 -rgc >/dev/null 2>&1; [ "$PL_OK" = 1 ] && nvidia-smi -i 0 -pl "$PDEF" >/dev/null 2>&1; kill $MPID 2>/dev/null; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; }
|
||||
trap cleanup EXIT
|
||||
say "miner warming 90 s"; sleep 90
|
||||
STEPS=$OUT/ember-steps.jsonl; : > $STEPS
|
||||
|
|
|
|||
|
|
@ -16,13 +16,13 @@ echo "RESULT snap_start $(stamp)"
|
|||
[ "$(sha256sum "$SNAP" | cut -c1-64)" = "$SNAP_SHA" ] || { echo "RESULT snap_failed sha256 $(sha256sum "$SNAP" | cut -c1-16)"; exit 2; }
|
||||
chmod 644 "$SNAP"
|
||||
bash $F/in/box-kill.sh >/dev/null 2>&1
|
||||
pkill -f '^/opt/igneum/pkg/bin/igneumd-0313'; pkill -x igneumd; sleep 4; pkill -9 -f '^/opt/igneum/pkg/bin/igneumd-0313' 2>/dev/null; sleep 1
|
||||
pkill -f '^[/]opt/igneum/pkg/bin/igneumd-0313'; pkill -x igneumd; sleep 4; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneumd-0313' 2>/dev/null; sleep 1
|
||||
rm -f $F/node/igneum-devnet/datadir/evm/exec-snapshot.bin $F/node/igneum-devnet/datadir/evm/exec-snapshot.prev.bin
|
||||
echo "RESULT snap_files_removed $(stamp) left=$(ls $F/node/igneum-devnet/datadir/evm/ 2>/dev/null | tr '\n' ' ')"
|
||||
IGNEUM_PROOF_VERIFIER=$HOST nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 \
|
||||
--addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes "--igneum-exec-snapshot=$SNAP,0x$SNAP_SHA" >> $F/node.log 2>&1 &
|
||||
t1=$(date +%s); sleep 15
|
||||
echo "RESULT node_started $(stamp) pid=$(pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') snapshot_line=\"$(grep -E 'igneum-exec\].*snapshot' $F/node.log | tail -1 | cut -c1-140)\""
|
||||
echo "RESULT node_started $(stamp) pid=$(pgrep -f '^[/]opt/igneum/pkg/bin/igneumd-0313' | head -1) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') snapshot_line=\"$(grep -E 'igneum-exec\].*snapshot' $F/node.log | tail -1 | cut -c1-140)\""
|
||||
for i in $(seq 1 40); do
|
||||
sleep 10; ex="$(rpc igneum_getExecStatus | python3 -c 'import sys,json; r=sys.stdin.read(); d=json.loads(r).get("result",{}) if r.strip() else {}; print(int(d.get("executedTip","0x0"),16), "blocked" if d.get("blocked") else "ok", str(d.get("startedFrom",""))[:30])' 2>/dev/null)"
|
||||
w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'daa=[0-9]*' | tail -1)"
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
#!/usr/bin/env bash
|
||||
# Stops every stage process on a box (the matrix, Ember, the prover loop, their miners, workers, samplers and SP1
|
||||
# servers); never the node. Run as a FILE (bash in/box-kill.sh) so that no pattern below can match the shell that runs
|
||||
# it (an inline `pkill -f 'igneum-prove-host'` killed its own ssh shell on 6 October 2026, 12:38Z).
|
||||
# it (an inline `pkill -f '[i]gneum-prove-host'` killed its own ssh shell on 6 October 2026, 12:38Z).
|
||||
for i in 1 2; do
|
||||
pkill -9 -f '^bash in/box-matrix.sh' ; pkill -9 -f '^bash in/box-ember.sh'; pkill -9 -f '^bash in/box-prover.sh'; pkill -9 -f '^python3 -u /root/fleet/in/box-prover.py'
|
||||
pkill -9 -x igneum-miner; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -9 -x sp1-gpu-server; pkill -9 -f '^/opt/igneum-floor/(bin|target|prove)/.*igneum-prove-(host|export)' # pkill -x cannot match a name over 15 characters (igneum-worker-cuda, igneum-prove-host)
|
||||
pkill -9 -f '^nvidia-smi --query'
|
||||
pkill -9 -f '^[b]ash in/box-matrix.sh' ; pkill -9 -f '^[b]ash in/box-ember.sh'; pkill -9 -f '^[b]ash in/box-prover.sh'; pkill -9 -f '^[p]ython3 -u /root/fleet/in/box-prover.py'
|
||||
pkill -9 -x igneum-miner; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -9 -x sp1-gpu-server; pkill -9 -f '^[/]opt/igneum-floor/(bin|target|prove)/.*igneum-prove-(host|export)' # pkill -x cannot match a name over 15 characters (igneum-worker-cuda, igneum-prove-host)
|
||||
pkill -9 -f '^[n]vidia-smi --query'
|
||||
sleep 2
|
||||
done
|
||||
rm -f /tmp/sp1-cuda-*.sock
|
||||
nvidia-smi -i 0 -rgc >/dev/null 2>&1
|
||||
d="$(nvidia-smi --query-gpu=power.default_limit --format=csv,noheader,nounits -i 0 | tr -d ' ' | cut -d. -f1)"; [ -n "$d" ] && nvidia-smi -i 0 -pl "$d" >/dev/null 2>&1
|
||||
cd /root/fleet/out 2>/dev/null && for f in matrix.log ember.log prover.log prover-launch.log rows.jsonl; do [ -f "$f" ] && mv "$f" "$f.$(date +%s).old"; done
|
||||
echo "count=$(pgrep -c -f '^bash in/box-|^python3 -u /root/fleet/in/box-prover')+$(pgrep -c -x igneum-miner)+$(pgrep -c -f '^/opt/igneum/pkg/bin/igneum-worker-cuda')+$(pgrep -c -x sp1-gpu-server)+$(pgrep -c -f '^/opt/igneum-floor/(bin|target|prove)/.*igneum-prove-host') mem=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 0 | tr -d ' ')"
|
||||
echo "count=$(pgrep -c -f '^[b]ash in/box-|^python3 -u /root/fleet/in/box-prover')+$(pgrep -c -x igneum-miner)+$(pgrep -c -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda')+$(pgrep -c -x sp1-gpu-server)+$(pgrep -c -f '^[/]opt/igneum-floor/(bin|target|prove)/.*igneum-prove-host') mem=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 0 | tr -d ' ')"
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ miner_start() {
|
|||
--prepare-packs packs/prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 10 > $OUT/miner.log 2>&1 &
|
||||
MPID=$!; cd $F
|
||||
}
|
||||
miner_stop() { [ -n "$MPID" ] && { kill $MPID 2>/dev/null; sleep 2; kill -9 $MPID 2>/dev/null; }; pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x igneum-miner 2>/dev/null; MPID=""; sleep 3; }
|
||||
miner_stop() { [ -n "$MPID" ] && { kill $MPID 2>/dev/null; sleep 2; kill -9 $MPID 2>/dev/null; }; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x igneum-miner 2>/dev/null; MPID=""; sleep 3; }
|
||||
miner_rate() { # mean of the STATUS now= values in the last N lines
|
||||
grep STATUS $OUT/miner.log | grep -o ' now=[0-9.]*' | tail -n "${1:-10}" | cut -d= -f2 | awk '{s+=$1; n++} END {if (n) printf "%.2f", s/n; else print 0}'
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ echo "$NODE_SHA256 $F/in/igneumd-0313" | sha256sum -c - >/dev/null || { echo "R
|
|||
chmod +x $F/in/igneumd-0313
|
||||
t0=$(date +%s)
|
||||
bash $F/in/box-kill.sh >/dev/null 2>&1 # every stage process; the node is stopped on the next line
|
||||
pkill -x igneumd; pkill -f '^/opt/igneum/pkg/bin/igneumd-0313'; sleep 4; pkill -9 -x igneumd 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneumd-0313' 2>/dev/null; sleep 1
|
||||
pkill -x igneumd; pkill -f '^[/]opt/igneum/pkg/bin/igneumd-0313'; sleep 4; pkill -9 -x igneumd 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneumd-0313' 2>/dev/null; sleep 1
|
||||
TGZ_SHA=e67cc6493cd86dc0a993c3e1005b1df43b661fc4973ef7f0095f602f1536cd85
|
||||
if [ ! -f $F/node/.full-history ]; then
|
||||
[ "$(sha256sum $F/observer-datadir.tgz 2>/dev/null | cut -c1-64)" = "$TGZ_SHA" ] || rm -f $F/observer-datadir.tgz # a partial pre-pull is not a tarball
|
||||
|
|
@ -38,7 +38,7 @@ IGNEUM_PROOF_VERIFIER=$HOST nohup $B/igneumd-0313 --devnet --appdir=$F/node --rp
|
|||
t1=$(date +%s); sleep 12
|
||||
d="$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print $2}')"
|
||||
ok=DIGEST_MISMATCH; [ "$d" = "$EXPECT_DIGEST" ] && ok=digest_ok
|
||||
echo "RESULT node_started $(stamp) step=$STEP pid=$(pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' | head -1) version=$($B/igneumd-0313 --version 2>&1 | head -1) digest=${d:0:16} expected=${EXPECT_DIGEST:0:16} $ok exec_restart=\"$(grep -o 'Exec restart from the override file.*' $F/node.log | tail -1 | cut -c1-120)\""
|
||||
echo "RESULT node_started $(stamp) step=$STEP pid=$(pgrep -f '^[/]opt/igneum/pkg/bin/igneumd-0313' | head -1) version=$($B/igneumd-0313 --version 2>&1 | head -1) digest=${d:0:16} expected=${EXPECT_DIGEST:0:16} $ok exec_restart=\"$(grep -o 'Exec restart from the override file.*' $F/node.log | tail -1 | cut -c1-120)\""
|
||||
[ "$ok" = digest_ok ] || { echo "RESULT swap_failed digest mismatch"; exit 2; }
|
||||
watch() { $B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1; }
|
||||
if [ "$STEP" = binary ]; then
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ if [ ! -x /opt/igneum/pkg/bin/igneum-miner ]; then
|
|||
curl -fsSL -o $F/pkg.tgz "https://dl.igneum.network$PKG_PATH" && echo "$PKG_SHA $F/pkg.tgz" | sha256sum -c - >/dev/null && mkdir -p /opt/igneum/pkg && tar -C /opt/igneum/pkg --strip-components=1 -xzf $F/pkg.tgz || { echo "RESULT pool_host_failed package"; exit 2; }
|
||||
fi
|
||||
B=/opt/igneum/pkg/bin; cp $F/in/igneumd-0313 $B/igneumd-0313; chmod +x $B/igneumd-0313; cp $F/in/ov13.json $F/override.json
|
||||
pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 &
|
||||
pgrep -f '^[/]opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 &
|
||||
sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}')"
|
||||
for i in $(seq 1 90); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [[ "$w" == *synced=true* ]] && break; sleep 10; done
|
||||
echo "RESULT synced $(stamp) after $((i*10)) s"
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ def miner_start():
|
|||
def miner_stop():
|
||||
global MPROC
|
||||
if MPROC: MPROC.terminate(); time.sleep(2); MPROC.kill(); MPROC = None
|
||||
subprocess.run(f"pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --device {DEV} '", shell=True)
|
||||
subprocess.run(f"pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --device {DEV} '", shell=True)
|
||||
def miner_rate(n=6):
|
||||
try:
|
||||
vals = [float(l.split(" now=")[1].split()[0]) for l in open(f"{OUT}/prover-miner.log").read().split("\n") if "STATUS" in l and " now=" in l][-n:]
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ if [ -z "${THRESHOLD:-}" ]; then
|
|||
fi
|
||||
if [ -z "${MINER:-}" ]; then if [ "$TOTAL" -ge 15000 ]; then MINER=keep; else MINER=pause; fi; fi
|
||||
echo "RESULT launch $(stamp) total_mib=$TOTAL threshold=${THRESHOLD:-default} miner=$MINER"
|
||||
pkill -x igneum-miner 2>/dev/null; pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock
|
||||
pkill -x igneum-miner 2>/dev/null; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock
|
||||
pkill -x igneumd; sleep 4; pkill -9 -x igneumd 2>/dev/null; sleep 1
|
||||
HUBARG=""; [ -n "${HUB_PEER:-}" ] && HUBARG="--addpeer=$HUB_PEER" # the fleet's hub box (a second peer beside the seed, 12:35Z)
|
||||
NODE_BIN=$B/igneumd; [ -x $B/igneumd-0313 ] && NODE_BIN=$B/igneumd-0313 # the 0.3.13 swap's binary once box-node-swap.sh has run
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ for d in $(seq 0 $((N-1))); do
|
|||
echo "RESULT miner card=$d mhs=$r watts=$w"; sum=$(awk -v a=$sum -v b=$r 'BEGIN {print a+b}')
|
||||
done
|
||||
echo "RESULT rig_miners $(stamp) cards=$N sum_mhs=$sum watts=$(awk -F', *' '{s+=$2; n++} END {printf "%.0f", s/n*'$N'}' $OUT/rig-samp-mine.csv)"
|
||||
for p in "${pids[@]}"; do kill $p 2>/dev/null; done; pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda'; sleep 3
|
||||
for p in "${pids[@]}"; do kill $p 2>/dev/null; done; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda'; sleep 3
|
||||
fi
|
||||
# C + D. seven core-only provers and one compressing card, in parallel
|
||||
pkill -9 -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock
|
||||
|
|
|
|||
|
|
@ -35,13 +35,13 @@ start_node() {
|
|||
sleep 8; echo "RESULT standing_node_started $(stamp) bin=$nb pid=$(node_pid) digest=$(grep -o 'digest: [0-9a-f]*' $LOG | tail -1 | awk '{print substr($2,1,16)}')"
|
||||
# a standalone igneum-miner keeps a dead template subscription after its node restarts (20:24Z: templates frozen,
|
||||
# fetch_errors climbing, no submits), so the miner and its worker are restarted with the node; the loop brings them back
|
||||
pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null; true
|
||||
pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null; true
|
||||
}
|
||||
HOST_VERIFIER=""
|
||||
miner_loop() {
|
||||
cd $F/mine; [ -d packs/$PACK ] || $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/$PACK > $OUT/export-pack.log 2>&1
|
||||
while :; do
|
||||
pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null # one miner per GPU: a restart race left two on three boxes (21:52Z)
|
||||
pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null # one miner per GPU: a restart race left two on three boxes (21:52Z)
|
||||
$B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/$PACK" --prepare-packs packs/$PACK-prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 60 >> $OUT/miner-0.log 2>&1; rc=$?
|
||||
echo "RESULT standing_miner_exit $(stamp) rc=$rc"; [ $rc = 42 ] && { rm -rf packs/$PACK; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/$PACK >> $OUT/export-pack.log 2>&1; } || sleep 10
|
||||
done
|
||||
|
|
@ -60,17 +60,17 @@ while :; do
|
|||
if [ "$MINE" = 1 ]; then
|
||||
wl="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; wb=$(grep -oE 'blocks=[0-9]+' <<< "$wl" | cut -d= -f2); ws=$(grep -oE 'synced=[a-z]+' <<< "$wl" | cut -d= -f2)
|
||||
if [ "$ws" = true ] && [ "${wb:-0}" -gt 1000 ]; then
|
||||
if ! pgrep -f '^/opt/igneum/pkg/bin/igneum-miner mine' >/dev/null && [ -z "${MINER_LOOP_PID:-}" -o ! -e "/proc/${MINER_LOOP_PID:-0}" ]; then miner_loop & MINER_LOOP_PID=$!; echo "RESULT standing_miner_started $(stamp) loop=$MINER_LOOP_PID"; fi
|
||||
if ! pgrep -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' >/dev/null && [ -z "${MINER_LOOP_PID:-}" -o ! -e "/proc/${MINER_LOOP_PID:-0}" ]; then miner_loop & MINER_LOOP_PID=$!; echo "RESULT standing_miner_started $(stamp) loop=$MINER_LOOP_PID"; fi
|
||||
else
|
||||
# a box never mines from a genesis-only or unsynced view (the block-rate runs' late-pod reorgs; 22:38Z: a miner on an IBD node)
|
||||
if pgrep -f '^/opt/igneum/pkg/bin/igneum-miner mine' >/dev/null || [ -n "${MINER_LOOP_PID:-}" -a -e "/proc/${MINER_LOOP_PID:-0}" ]; then
|
||||
if pgrep -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' >/dev/null || [ -n "${MINER_LOOP_PID:-}" -a -e "/proc/${MINER_LOOP_PID:-0}" ]; then
|
||||
[ -n "${MINER_LOOP_PID:-}" ] && kill -9 "$MINER_LOOP_PID" 2>/dev/null; MINER_LOOP_PID=""
|
||||
pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null
|
||||
pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs' 2>/dev/null
|
||||
echo "RESULT standing_miner_stopped $(stamp) node unsynced ($wl): the miner waits for the tip"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
if [ "$PROVER" = 1 ] && [ -x /opt/igneum-floor/bin/igneum-prove-host ] && ! pgrep -f '^python3 -u /root/fleet/in/box-prover.py' >/dev/null; then
|
||||
if [ "$PROVER" = 1 ] && [ -x /opt/igneum-floor/bin/igneum-prove-host ] && ! pgrep -f '^[p]ython3 -u /root/fleet/in/box-prover.py' >/dev/null; then
|
||||
cd $F && LABEL="$LABEL" WALLET="$WALLET" EXPORT_FROM="${EXPORT_FROM:-0}" CHAIN_NAME=$NET MINER=none RUN_HOURS=240 setsid nohup python3 -u $F/in/box-prover.py </dev/null >> $OUT/prover-launch.log 2>&1 &
|
||||
echo "RESULT standing_prover_started $(stamp)"
|
||||
fi
|
||||
|
|
@ -85,7 +85,7 @@ while :; do
|
|||
ex=$(curl -s -m 5 127.0.0.1:26790 -H 'content-type: application/json' -d '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' | grep -oE '"result":"0x[0-9a-f]+"' | cut -d'"' -f4)
|
||||
exn=$((${ex:-0x0})); blocks=$(printf '%s' "$w" | grep -oE 'blocks=[0-9]+' | cut -d= -f2)
|
||||
m="$(grep STATUS $OUT/miner-0.log 2>/dev/null | tail -1 | grep -oE 'accepted=[0-9]+ rejected=[0-9]+|\([0-9.]+ MH/s inside jobs' | tr '\n' ' ' | tr -d '(')"
|
||||
echo "RESULT standing $(stamp) up=$(( $(date -u +%s) - STARTED ))s role=$ROLE node=$(node_pid) bin=$cur version=$($cur --version 2>&1 | head -1 | awk '{print $2}') $w exec=$exn miner=\"$m\" prover=$(pgrep -c -f '^python3 -u /root/fleet/in/box-prover.py') disk=$DISK gpu=$(nvidia-smi --query-gpu=utilization.gpu,power.draw --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')" >> $OUT/standing.log
|
||||
echo "RESULT standing $(stamp) up=$(( $(date -u +%s) - STARTED ))s role=$ROLE node=$(node_pid) bin=$cur version=$($cur --version 2>&1 | head -1 | awk '{print $2}') $w exec=$exn miner=\"$m\" prover=$(pgrep -c -f '^[p]ython3 -u /root/fleet/in/box-prover.py') disk=$DISK gpu=$(nvidia-smi --query-gpu=utilization.gpu,power.draw --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')" >> $OUT/standing.log
|
||||
# the dead-exec class: synced consensus, exec at 0 from snapshot -> the shipper's recovery recipe (live devnet only)
|
||||
if [ "$ROLE" = live ] && [ "${blocks:-0}" -gt 1000 ] && [ "$exn" = 0 ] && [ -x $F/in/box-exec-snapshot.sh ] && [ $((n / 10)) -gt 2 ]; then
|
||||
echo "RESULT standing_recovery $(stamp) exec=0 blocks=$blocks: running box-exec-snapshot.sh"; HUB_PEER="$HUB_PEER" bash $F/in/box-exec-snapshot.sh; sleep 30
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ set -uo pipefail
|
|||
F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; POOL="${POOL:?host:port}"; LABEL="${LABEL:-wave}"; WALLET="${WALLET:?}"
|
||||
exec >> $OUT/wave.log 2>&1
|
||||
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
pkill -f '^bash in/box-wave.sh'; pkill -x igneum-miner; sleep 2
|
||||
pkill -f '^[b]ash in/box-wave.sh'; pkill -x igneum-miner; sleep 2
|
||||
chmod +x $F/in/igneum-miner-pool
|
||||
echo "RESULT pool_mode $(stamp) pool=$POOL miner=$(sha256sum $F/in/igneum-miner-pool | cut -c1-16)"
|
||||
cd $F/mine
|
||||
|
|
|
|||
|
|
@ -21,8 +21,8 @@ B=/opt/igneum/pkg/bin
|
|||
[ "$(sha256sum $F/in/igneumd-0313 | cut -c1-16)" = d6350586fe837b1f ] || { echo "RESULT wave_failed node binary"; exit 2; }
|
||||
cp $F/in/igneumd-0313 $B/igneumd-0313; chmod +x $B/igneumd-0313; cp $F/in/ov13.json $F/override.json
|
||||
ldconfig -p | grep -q libnvrtc.so.12 || echo "RESULT note no libnvrtc.so.12 on the path (the worker dlopens it)"
|
||||
pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 &
|
||||
sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') pid=$(pgrep -f '^/opt/igneum/pkg/bin/igneumd-0313' | head -1)"
|
||||
pgrep -f '^[/]opt/igneum/pkg/bin/igneumd-0313' >/dev/null || nohup $B/igneumd-0313 --devnet --appdir=$F/node --rpclisten=127.0.0.1:26610 --listen=0.0.0.0:26611 --addpeer=$HUB_PEER --addpeer=188.245.5.161:26611 --override-params-file=$F/override.json --nodnsseed --disable-upnp --nologfiles --yes > $F/node.log 2>&1 &
|
||||
sleep 10; echo "RESULT node $(stamp) digest=$(grep -o 'digest: [0-9a-f]*' $F/node.log | tail -1 | awk '{print substr($2,1,16)}') pid=$(pgrep -f '^[/]opt/igneum/pkg/bin/igneumd-0313' | head -1)"
|
||||
for i in $(seq 1 90); do w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; [[ "$w" == *synced=true* ]] && break; sleep 10; done
|
||||
echo "RESULT synced $(stamp) $(printf '%s' "$w" | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //') after $((i*10)) s"
|
||||
cd $F/mine && rm -rf packs/devnet && $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet > $OUT/export-pack.log 2>&1
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue
|
|||
l="$(SSH "$2" "$3" "grep -E '^RESULT (node_started|swap_failed)' /root/fleet/out/node-swap.log | tail -1")"; echo " $1: ${l:0:160}"; [[ "$l" == *digest_ok* ]] || fail "$1: the swap did not reach digest_ok (${l:0:200})"
|
||||
done
|
||||
# the provers back on (box-prover.sh keeps a running igneumd-0313 with the verifier)
|
||||
for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh </dev/null >/dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done
|
||||
for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh </dev/null >/dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done
|
||||
for p in $MINERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet/mine && pkill -x igneum-miner; sleep 2; rm -rf packs/devnet; /opt/igneum/pkg/bin/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >/dev/null 2>&1; setsid nohup /opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 $1 --worker /opt/igneum/pkg/bin/igneum-worker-cuda --worker-args '--device 0 --pack packs/devnet' --prepare-packs packs/prepare --exit-on-seed-change --evm-address $4 --payout-label $1 --status-secs 30 </dev/null >> /root/fleet/out/mine-only-0.log 2>&1 & echo miner" >/dev/null || fail "$1: miner relaunch"; done
|
||||
fi
|
||||
echo "CANARY running $(stamp) $MINUTES min"; sleep $((MINUTES * 60))
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ def st(): return time.strftime("%H:%M:%SZ", time.gmtime())
|
|||
R=Box.from_registry("pool-1"); M=Box.from_registry("p2-3090-3"); T=Box.from_registry("p2-4090-3"); hub=Box.from_registry("hub-1")
|
||||
_, rM = Registry.find("p2-3090-3"); _, rR = Registry.find("pool-1")
|
||||
W="/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\\n' ' '"
|
||||
KILL="pkill -9 -f '^bash in/box-standing.sh'; pkill -9 -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs'; pkill -9 -f '^/(opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir='; sleep 3"
|
||||
KILL="pkill -9 -f '^[b]ash in/box-standing.sh'; pkill -9 -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs'; pkill -9 -f '^[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir='; sleep 3"
|
||||
print(st(), "T before:", T.run("echo vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' /root/fleet/node.log | tail -1) rej1026=$(grep -c 'wrong block version' /root/fleet/node.log) got=$(grep -c 'got reject message' /root/fleet/node.log); "+W, 30)[1].replace("\n"," "), flush=True)
|
||||
# R: the relay on 6615571c over its 1026 datadir (kept aside as node.clean while it runs)
|
||||
rc,out,err=R.run(f"cd /root/fleet && {KILL}; [ -d node.clean ] || mv node node.clean; rm -rf node; cp -a node.1026-2109 node; setsid nohup in/igneumd-6615571ced35f694 --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:4463 --addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes </dev/null >> node.log 2>&1 & for i in $(seq 1 40); do sleep 10; w=$({W}); case \"$w\" in *synced=true*) break;; esac; done; echo R vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' node.log | tail -1) after=$((i*10))s $w", 500)
|
||||
|
|
|
|||
|
|
@ -9,10 +9,10 @@ def st(): return time.strftime("%H:%M:%SZ", time.gmtime())
|
|||
NODE, SHA, MINER = sys.argv[1], sys.argv[2], sys.argv[3]
|
||||
assert hashlib.sha256(open(NODE,"rb").read()).hexdigest()==SHA, "node sha"
|
||||
WATCH="/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\\n' ' '"
|
||||
KILL="pkill -9 -f '^bash in/box-standing.sh'; pkill -9 -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs'; pkill -9 -f '^/(opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir='; sleep 3"
|
||||
KILL="pkill -9 -f '^[b]ash in/box-standing.sh'; pkill -9 -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve --device 0 --pack packs'; pkill -9 -f '^[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir='; sleep 3"
|
||||
hub=Box.from_registry("hub-1"); pool=Box.from_registry("pool-1"); A=Box.from_registry("p2-4090-3"); B=Box.from_registry("p2-3090-2")
|
||||
# pool-1: the poisoned peer, as it stands (6615571c, its datadir), on its inbound port
|
||||
rc,out,err=pool.run("cd /root/fleet && pgrep -f '^/root/fleet/in/igneumd-6615571ced35f694 ' >/dev/null || { setsid nohup /root/fleet/in/igneumd-6615571ced35f694 --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:4463 --addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes </dev/null >> node.log 2>&1 & sleep 15; }; echo pool1 $(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' node.log | tail -1) "+WATCH, 60); print(st(), "poisoned peer:", out.replace("\n"," "), flush=True)
|
||||
rc,out,err=pool.run("cd /root/fleet && pgrep -f '^[/]root/fleet/in/igneumd-6615571ced35f694 ' >/dev/null || { setsid nohup /root/fleet/in/igneumd-6615571ced35f694 --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:4463 --addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes </dev/null >> node.log 2>&1 & sleep 15; }; echo pool1 $(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' node.log | tail -1) "+WATCH, 60); print(st(), "poisoned peer:", out.replace("\n"," "), flush=True)
|
||||
def hubsplit(): return hub.run("echo total=$(grep -c 'wrong block version' /root/fleet/node.log); awk '$2 >= \"'$(date -u -d '-12 min' +%H:%M:%S)'\"' /root/fleet/node.log | grep 'wrong block version' | grep -oE 'peer [0-9.]+' | sort | uniq -c | sort -rn | tr '\\n' ';'", 30)[1].replace("\n"," ").strip()
|
||||
print(st(), "hub before:", hubsplit(), flush=True)
|
||||
for box, l, peers in ((A, "p2-4090-3", "--addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611 --addpeer=213.173.98.36:18451"), (B, "p2-3090-2", "--addpeer=213.173.107.74:16516 --addpeer=188.245.5.161:26611")):
|
||||
|
|
@ -34,7 +34,7 @@ while time.time()-T0 < 720:
|
|||
rc,b,err=B.run(WATCH+"; echo got_reject=$(grep -c 'got reject message' /root/fleet/node.log)", 30)
|
||||
print(st(), f"+{int(time.time()-T0)}s A:", a.replace("\n"," | ").strip(), "| B:", b.replace("\n"," ").strip(), "| hub:", hubsplit(), flush=True)
|
||||
if not restarted and time.time()-T0 >= 300:
|
||||
rc,out,err=B.run("cd /root/fleet && pkill -9 -f '^/root/fleet/in/igneumd-[0-9a-f]{16} --devnet --appdir='; sleep 20; BIN=$(cat standing.node); IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host setsid nohup $BIN --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=213.173.107.74:16516 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes </dev/null >> node.log 2>&1 & for i in $(seq 1 24); do sleep 5; w=$("+WATCH+"); case \"$w\" in *synced=true*) break;; esac; done; echo resync_after=$((i*5+20))s $w; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve'", 200)
|
||||
rc,out,err=B.run("cd /root/fleet && pkill -9 -f '^[/]root/fleet/in/igneumd-[0-9a-f]{16} --devnet --appdir='; sleep 20; BIN=$(cat standing.node); IGNEUM_PROOF_VERIFIER=/opt/igneum-floor/bin/igneum-prove-host setsid nohup $BIN --devnet --appdir=/root/fleet/node --rpclisten=0.0.0.0:26610 --evm-rpclisten=127.0.0.1:26790 --listen=0.0.0.0:26611 --addpeer=213.173.107.74:16516 --override-params-file=/root/fleet/override.json --nodnsseed --disable-upnp --nologfiles --yes </dev/null >> node.log 2>&1 & for i in $(seq 1 24); do sleep 5; w=$("+WATCH+"); case \"$w\" in *synced=true*) break;; esac; done; echo resync_after=$((i*5+20))s $w; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine'; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve'", 200)
|
||||
print(st(), "B restart (hub-only re-sync):", out.replace("\n"," "), flush=True); restarted=True
|
||||
hub_pan=hub.run("echo hub_panics=$(grep -ciE 'panicked at consensus' /root/fleet/node.log)", 20)[1].strip(); print(st(), hub_pan, flush=True)
|
||||
rc,out,err=hub.run(f"timeout 150 /opt/igneum/pkg/bin/igneum-miner inspect 900 grpc://127.0.0.1:26610 2>&1 | grep -c 'voteKeyHash={keyA}'; awk '$2 >= \"'$(date -u -d '-14 min' +%H:%M:%S)'\"' /root/fleet/node.log | grep -E 'wrong block version|disconnecting' | grep -c '216.232.211.246'", 170)
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ for r in "${ROWS[@]}"; do set -- $r; [[ "$1" == hub-1 ]] && continue
|
|||
l="$(SSH "$2" "$3" "grep -E '^RESULT (node_started|swap_failed)' /root/fleet/out/node-swap.log | tail -1 | cut -c1-140")"; echo " $1: $l"; [[ "$l" == *digest_ok* ]] || fail "$1: the swap did not reach digest_ok ($l)"
|
||||
done
|
||||
# the provers back on (box-prover.sh keeps a running igneumd-0313 with the verifier)
|
||||
for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh </dev/null >/dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done
|
||||
for p in $PROVERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet && pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; sleep 2; mv out/prover.log out/prover-canary-prev.log 2>/dev/null; LABEL=$1 WALLET=$4 HUB_PEER=$HUB_PEER setsid nohup in/box-prover.sh </dev/null >/dev/null 2>&1 & echo prover" >/dev/null || fail "$1: prover relaunch"; done
|
||||
for p in $MINERS; do r="$(printf '%s\n' "${ROWS[@]}" | awk -v l="$p" '$1==l')"; set -- $r; SSH "$2" "$3" "cd /root/fleet/mine && pkill -x igneum-miner; sleep 2; rm -rf packs/devnet; /opt/igneum/pkg/bin/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet >/dev/null 2>&1; setsid nohup /opt/igneum/pkg/bin/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 $1 --worker /opt/igneum/pkg/bin/igneum-worker-cuda --worker-args '--device 0 --pack packs/devnet' --prepare-packs packs/prepare --exit-on-seed-change --evm-address $4 --payout-label $1 --status-secs 30 </dev/null >> /root/fleet/out/mine-only-0.log 2>&1 & echo miner" >/dev/null || fail "$1: miner relaunch"; done
|
||||
echo "CANARY running $(stamp) $MINUTES min"; sleep $((MINUTES * 60))
|
||||
echo "CANARY checks $(stamp)"; ok=1; paid_any=0; H=""
|
||||
|
|
|
|||
|
|
@ -62,10 +62,10 @@ for l in "${BOXES[@]}"; do set -- $l
|
|||
done
|
||||
restart_all() { # with the override file already on each box as /root/fleet/in/dn2-override.json
|
||||
for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] || continue
|
||||
SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f '^/root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh </dev/null >/dev/null 2>&1 & echo restarted" || fail "$1: restart"
|
||||
SSH "$2" "$3" "cd /root/fleet && pkill -f '^[b]ash in/box-dn2.sh'; pkill -f '^[/]root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh </dev/null >/dev/null 2>&1 & echo restarted" || fail "$1: restart"
|
||||
sleep 15; done
|
||||
for l in "${BOXES[@]}"; do set -- $l; [[ "$4" == 1 ]] && continue
|
||||
SSH "$2" "$3" "cd /root/fleet && pkill -f '^bash in/box-dn2.sh'; pkill -f '^/root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 SEED=$SEED_PEER NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh </dev/null >/dev/null 2>&1 & echo restarted" || fail "$1: restart"
|
||||
SSH "$2" "$3" "cd /root/fleet && pkill -f '^[b]ash in/box-dn2.sh'; pkill -f '^[/]root/fleet/in/igneumd'; pkill -x igneum-miner; pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; sleep 3; LABEL=$1 WALLET=$5 SEED=$SEED_PEER NODE_BIN=/root/fleet/in/igneumd-gate PROVER=$6 setsid nohup in/box-dn2.sh </dev/null >/dev/null 2>&1 & echo restarted" || fail "$1: restart"
|
||||
done
|
||||
}
|
||||
SINCE="$(date -u +'%Y-%m-%d %H:%M:%S')"; echo "GATE step1 $(stamp) restart on the release binary, the current override (rejects and reorgs counted from $SINCE)"; restart_all; sleep 60
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
#!/usr/bin/env bash
|
||||
# Stops everything of a Devnet 2 box but sshd: run as a FILE (an inline pkill with "igneumd" in it kills the ssh shell that carries the word).
|
||||
pkill -9 -f '^bash in/box-dn2.sh'; pkill -9 -f '^/root/fleet/in/igneumd'; pkill -9 -f '^/root/fleet/in/igneumd-gate'; pkill -9 -x igneum-miner; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -9 -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -9 -x sp1-gpu-server
|
||||
sleep 2; echo "left=$(pgrep -c -f '^/root/fleet/in/igneumd')+$(pgrep -c -x igneum-miner)+$(pgrep -c -f '^bash in/box-dn2.sh')"
|
||||
pkill -9 -f '^[b]ash in/box-dn2.sh'; pkill -9 -f '^[/]root/fleet/in/igneumd'; pkill -9 -f '^[/]root/fleet/in/igneumd-gate'; pkill -9 -x igneum-miner; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -9 -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -9 -x sp1-gpu-server
|
||||
sleep 2; echo "left=$(pgrep -c -f '^[/]root/fleet/in/igneumd')+$(pgrep -c -x igneum-miner)+$(pgrep -c -f '^[b]ash in/box-dn2.sh')"
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@
|
|||
# Restarts the Devnet 2 prover on a box as ONE instance: run as a FILE (an inline pkill carrying these words kills the ssh shell
|
||||
# that carries them). Env: LABEL WALLET THRESHOLD (optional).
|
||||
F=/root/fleet; cd $F || exit 2
|
||||
pkill -9 -f '^python3 -u (/root/fleet/)?in/box-prover.py'; pkill -9 -x sp1-gpu-server; pkill -9 -f '^/opt/igneum-floor/bin/igneum-prove-host'; pkill -9 -f '^/opt/igneum-segal/proving/igneum-prove/target/release/igneum-prove-host'
|
||||
pkill -9 -f '^[p]ython3 -u (/root/fleet/)?in/box-prover.py'; pkill -9 -x sp1-gpu-server; pkill -9 -f '^[/]opt/igneum-floor/bin/igneum-prove-host'; pkill -9 -f '^[/]opt/igneum-segal/proving/igneum-prove/target/release/igneum-prove-host'
|
||||
sleep 3; rm -f /tmp/sp1-cuda-*.sock $F/out/prover.pid $F/out/prover-state.json.*.tmp
|
||||
LABEL="${LABEL:?}" WALLET="${WALLET:?}" THRESHOLD="${THRESHOLD:-}" EXPORT_FROM=0 CHAIN_NAME=igneum-devnet-2 MINER=none RUN_HOURS=240 setsid nohup python3 -u /root/fleet/in/box-prover.py </dev/null >> $F/out/dn2-prover-launch.log 2>&1 &
|
||||
sleep 25
|
||||
echo "RESULT dn2_prover_restart $(date -u +%FT%TZ) instances=$(pgrep -f '^python3 -u (/root/fleet/)?in/box-prover.py' | wc -l) pid=$(cat $F/out/prover.pid 2>/dev/null) server=$(pgrep -c -x sp1-gpu-server) last=\"$(tail -1 $F/out/prover.log | cut -c1-140)\""
|
||||
echo "RESULT dn2_prover_restart $(date -u +%FT%TZ) instances=$(pgrep -f '^[p]ython3 -u (/root/fleet/)?in/box-prover.py' | wc -l) pid=$(cat $F/out/prover.pid 2>/dev/null) server=$(pgrep -c -x sp1-gpu-server) last=\"$(tail -1 $F/out/prover.log | cut -c1-140)\""
|
||||
|
|
|
|||
|
|
@ -104,7 +104,7 @@ def setup(labels):
|
|||
if not b.get("ssh_ok"): print(b["label"], "no ssh yet"); continue
|
||||
ssh(b, "mkdir -p /root/fleet/in /root/fleet/out")
|
||||
if scp(b, INPUTS + [os.path.join(HERE, "box-setup.sh")], "/root/fleet/in/") != 0: print(b["label"], "scp failed"); continue
|
||||
rc, out, err = ssh(b, f"cd /root/fleet && chmod +x in/box-setup.sh && if pgrep -f '^bash in/box-setup.sh' >/dev/null; then echo already; else RUSTUP_TOOLCHAIN=stable ARCHS={b['archs']} LABEL={b['label']} WALLET={b['wallet']} setsid nohup in/box-setup.sh </dev/null >/dev/null 2>&1 & echo started; fi", timeout=60)
|
||||
rc, out, err = ssh(b, f"cd /root/fleet && chmod +x in/box-setup.sh && if pgrep -f '^[b]ash in/box-setup.sh' >/dev/null; then echo already; else RUSTUP_TOOLCHAIN=stable ARCHS={b['archs']} LABEL={b['label']} WALLET={b['wallet']} setsid nohup in/box-setup.sh </dev/null >/dev/null 2>&1 & echo started; fi", timeout=60)
|
||||
print(b["label"], out.strip()[:40])
|
||||
patch(iid, setup_started=now())
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ KEY = os.path.expanduser("~/.ssh/igneum-fleet")
|
|||
SSH_OPTS = ["-i", KEY, "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR",
|
||||
"-o", "ConnectTimeout=20", "-o", "ServerAliveInterval=10", "-o", "ServerAliveCountMax=3", "-o", "BatchMode=yes"]
|
||||
B = "/opt/igneum/pkg/bin"; F = "/root/fleet"
|
||||
NODE_PATH_PATTERN = "^/root/fleet/in/igneumd|^/opt/igneum/pkg/bin/igneumd"
|
||||
NODE_PATH_PATTERN = "^[/]root/fleet/in/igneumd|^[/]opt/igneum/pkg/bin/igneumd" # the bracket form: the shell that runs a pgrep/pkill with this pattern never matches itself
|
||||
def now(): return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
class SshError(RuntimeError): pass
|
||||
def hexi(v):
|
||||
|
|
@ -132,11 +132,11 @@ class Box:
|
|||
log = log or f"{F}/out/miner-{device}.log"
|
||||
self.check(f"cd {F}/mine 2>/dev/null || mkdir -p {F}/mine/packs && cd {F}/mine; [ -d packs/{pack} ] || {B}/igneum-miner export-pack {grpc} packs/{pack} >/dev/null 2>&1; setsid nohup {B}/igneum-miner mine {grpc} 1 100000000 {shlex.quote(label)} --worker {B}/igneum-worker-cuda --worker-args '--device {device} --pack packs/{pack}' --prepare-packs packs/prepare-{device} --exit-on-seed-change --evm-address {wallet} --payout-label {shlex.quote(label)} --status-secs 30 </dev/null >> {log} 2>&1 & echo $!", 90)
|
||||
return int(self.run("pgrep -x igneum-miner | tail -1", 20)[1].strip() or 0)
|
||||
def stop_miners(self): self.run("pkill -x igneum-miner; pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda'; true", 30)
|
||||
def stop_miners(self): self.run("pkill -x igneum-miner; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda'; true", 30)
|
||||
def start_prover(self, label, wallet, hub_peer="", threshold="", miner="keep"):
|
||||
self.check(f"cd {F} && chmod +x in/box-prover.sh && LABEL={shlex.quote(label)} WALLET={wallet} HUB_PEER={hub_peer} THRESHOLD={threshold} MINER={miner} setsid nohup in/box-prover.sh </dev/null >/dev/null 2>&1 & echo started", 60)
|
||||
def stop_all(self):
|
||||
self.run(f"bash {F}/in/box-kill.sh >/dev/null 2>&1; pkill -f '^python3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -x sp1-gpu-server; true", 60)
|
||||
self.run(f"bash {F}/in/box-kill.sh >/dev/null 2>&1; pkill -f '^[p]ython3 -u /root/fleet/in/box-prover.py'; pkill -x igneum-miner; pkill -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda'; pkill -x sp1-gpu-server; true", 60)
|
||||
# ---- provider ----
|
||||
def destroy(self):
|
||||
import sys; sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ def install(label, role=None, prover=None, mine=None):
|
|||
b_.put([os.path.join(HERE, "box-standing.sh"), os.path.join(HERE, "box-exec-snapshot.sh")], f"{F}/in/")
|
||||
hub = next((v for v in Registry.load().values() if v.get("hub") and v.get("state") != "destroyed"), {})
|
||||
seed = b.get("dn2_seed_addr", "")
|
||||
rc, out, err = b_.run(f"cd {F} && chmod +x in/box-standing.sh in/box-exec-snapshot.sh; pgrep -f '^bash in/box-standing.sh' >/dev/null && echo supervisor-already || {{ ROLE={role} LABEL={label} WALLET={b.get('wallet')} PROVER={prover} MINE={mine} HUB_PEER={hub.get('hub_peer','')} SEED={seed} setsid nohup bash in/box-standing.sh </dev/null >/dev/null 2>&1 & sleep 2; echo supervisor-started; }}", 60)
|
||||
rc, out, err = b_.run(f"cd {F} && chmod +x in/box-standing.sh in/box-exec-snapshot.sh; pgrep -f '^[b]ash in/box-standing.sh' >/dev/null && echo supervisor-already || {{ ROLE={role} LABEL={label} WALLET={b.get('wallet')} PROVER={prover} MINE={mine} HUB_PEER={hub.get('hub_peer','')} SEED={seed} setsid nohup bash in/box-standing.sh </dev/null >/dev/null 2>&1 & sleep 2; echo supervisor-started; }}", 60)
|
||||
Registry.patch(iid, standing=True, role=role, standing_since=b.get("standing_since") or now(), doing=f"standing {role}: node, {'miner, ' if mine == '1' else ''}{'prover' if prover == '1' else 'no prover'} (supervised)")
|
||||
return out.strip()
|
||||
DIGEST_ALERTS = os.path.join(ROOT, "digest-alerts.json")
|
||||
|
|
@ -84,7 +84,7 @@ def update(label):
|
|||
"""The version follow: the manifest's hive package onto the box, the node pointer rewritten, the supervisor restarts it."""
|
||||
m = manifest(); b = Box.from_registry(label)
|
||||
if not m: raise SshError("no manifest")
|
||||
rc, out, err = b.run(f"set -e; cd {F}; curl -fsSL -m 600 -o pkg-{m['version']}.tgz https://dl.igneum.network{m['path']}; echo '{m['sha256']} pkg-{m['version']}.tgz' | sha256sum -c - >/dev/null; rm -rf /opt/igneum/pkg.new; mkdir -p /opt/igneum/pkg.new; tar -C /opt/igneum/pkg.new --strip-components=1 -xzf pkg-{m['version']}.tgz; rm -rf /opt/igneum/pkg.prev; mv /opt/igneum/pkg /opt/igneum/pkg.prev; mv /opt/igneum/pkg.new /opt/igneum/pkg; echo /opt/igneum/pkg/bin/igneumd > {F}/standing.node; pkill -9 -f '^/opt/igneum/pkg/bin/igneumd' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg.prev/bin/igneumd' 2>/dev/null; echo updated-to-{m['version']}", 900)
|
||||
rc, out, err = b.run(f"set -e; cd {F}; curl -fsSL -m 600 -o pkg-{m['version']}.tgz https://dl.igneum.network{m['path']}; echo '{m['sha256']} pkg-{m['version']}.tgz' | sha256sum -c - >/dev/null; rm -rf /opt/igneum/pkg.new; mkdir -p /opt/igneum/pkg.new; tar -C /opt/igneum/pkg.new --strip-components=1 -xzf pkg-{m['version']}.tgz; rm -rf /opt/igneum/pkg.prev; mv /opt/igneum/pkg /opt/igneum/pkg.prev; mv /opt/igneum/pkg.new /opt/igneum/pkg; echo /opt/igneum/pkg/bin/igneumd > {F}/standing.node; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneumd' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg.prev/bin/igneumd' 2>/dev/null; echo updated-to-{m['version']}", 900)
|
||||
iid, _ = Registry.find(label); Registry.patch(iid, version_wanted=m["version"], updated_at=now()); return out.strip()
|
||||
def rerent(label):
|
||||
"""Same shape again on the same provider; the old row is marked destroyed. Returns the new label or None."""
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ PROBE = r"""grep -E '^RESULT (swap_done|swap_failed)' /root/fleet/out/node-swap.
|
|||
curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"igneum_getExecStatus","params":[]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; r=sys.stdin.read(); d=json.loads(r).get("result",{}) if r.strip() else {}; print("exec", int(d.get("executedTip","0x0"),16), "blocked", "yes" if d.get("blocked") else "no", "from", str(d.get("startedFrom",""))[:30])' 2>/dev/null
|
||||
curl -s -m 6 -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"igneum_getProvingStatus","params":[]}' http://127.0.0.1:26790/ | python3 -c 'import sys,json; r=sys.stdin.read(); d=json.loads(r).get("result",{}) if r.strip() else {}; v=d.get("v1",{}); w=v.get("segmentsInWindow",{}); print("tipDaa", int(d.get("tipDaa","0x0"),16), "fresh", v.get("freshRuleActive"), "paidShards", d.get("paidShards"), "pending", w.get("pending"), "proven", w.get("proven"), "unproven", w.get("unproven"), "paidSeg", v.get("paidSegments"))' 2>/dev/null
|
||||
/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'daa=[0-9]*' | tail -1
|
||||
pgrep -c -f '^python3 -u /root/fleet/in/box-prover.py'
|
||||
pgrep -c -f '^[p]ython3 -u /root/fleet/in/box-prover.py'
|
||||
for f in /root/fleet/out/prover-state.json /root/fleet/card*/out/prover-state.json; do [ -f $f ] && python3 -c 'import json,sys; s=json.load(open(sys.argv[1])); print("ps", s.get("claimed",0), s.get("submitted",0), s.get("paid",0), s.get("shards_accepted",0), s.get("shards_refused",0), s.get("segment_refused",0), s.get("held",0), s.get("miner_mhs",0))' $f; done; true"""
|
||||
def probe(b):
|
||||
r, out, err = fleet.ssh(b, PROBE, timeout=45)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ def publish(label):
|
|||
b.put([MINER[0], CUDA[0], OCL[0]], "/root/fleet/in/")
|
||||
rc, out, err = b.run(f"""set -e; cd /opt/igneum/pkg/bin; for f in igneum-miner igneum-worker-cuda igneum-worker-opencl; do [ -e $f.0314 ] || cp $f $f.0314; cp /root/fleet/in/$f $f.new && chmod +x $f.new && mv $f.new $f; done
|
||||
echo {path} > /root/fleet/standing.node
|
||||
pkill -9 -f '^/(opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir=' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve' 2>/dev/null; true # the supervisor restarts the node on the new pointer within a minute, and the miner with it
|
||||
pkill -9 -f '^[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir=' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' 2>/dev/null; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve' 2>/dev/null; true # the supervisor restarts the node on the new pointer within a minute, and the miner with it
|
||||
echo swapped""", 120)
|
||||
return label, out.strip() or err[:120]
|
||||
def readback(label):
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ live=[r["label"] for r in standing.roster() if r["role"]=="live"]
|
|||
def move(l):
|
||||
try:
|
||||
b=Box.from_registry(l); b.put([F16], "/root/fleet/override-16.json")
|
||||
rc,out,err=b.run("cd /root/fleet && cp override.json override-13.json && cp override-16.json override.json && python3 -c \"import json; d=json.load(open('/root/fleet/override.json')); print('fields', len(d))\" && pkill -9 -f '^/(opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir=' ; echo killed-for-restart $(date -u +%T)", 60)
|
||||
rc,out,err=b.run("cd /root/fleet && cp override.json override-13.json && cp override-16.json override.json && python3 -c \"import json; d=json.load(open('/root/fleet/override.json')); print('fields', len(d))\" && pkill -9 -f '^[/](opt/igneum/pkg/bin|root/fleet/in)/igneumd(-0313|-[0-9a-f]{16})? --devnet --appdir=' ; echo killed-for-restart $(date -u +%T)", 60)
|
||||
return l, out.replace("\n"," ").strip()+(" ERR "+err[:60] if err.strip() else "")
|
||||
except Exception as e: return l, "EXC "+str(e)[:60]
|
||||
print(st(), "moving", len(live), "boxes", flush=True)
|
||||
|
|
@ -18,7 +18,7 @@ with ThreadPoolExecutor(14) as ex:
|
|||
time.sleep(150)
|
||||
def read(l):
|
||||
try:
|
||||
rc,out,err=Box.from_registry(l).run("echo digest=$(grep -o 'digest: [0-9a-f]*' /root/fleet/node.log | tail -1 | awk '{print substr($2,1,16)}') vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' /root/fleet/node.log | tail -1) fields=$(python3 -c \"import json; print(len(json.load(open('/root/fleet/override.json'))))\") $(/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\\n' ' ') miner=$(pgrep -f '^/opt/igneum/pkg/bin/igneum-miner mine' | wc -l) rej_since=$(awk '$2 >= \"22:47:00\"' /root/fleet/node.log | grep -c 'got reject message') v4=\"$(grep -E 'Program class v4 (signal window|from the override)' /root/fleet/node.log | tail -2 | sed -E 's/^[0-9: .+-]*//' | cut -c1-70 | tr '\\n' ';')\"", 60)
|
||||
rc,out,err=Box.from_registry(l).run("echo digest=$(grep -o 'digest: [0-9a-f]*' /root/fleet/node.log | tail -1 | awk '{print substr($2,1,16)}') vline=$(grep -oE 'igneumd/2\\.1\\.0-[0-9a-f]+' /root/fleet/node.log | tail -1) fields=$(python3 -c \"import json; print(len(json.load(open('/root/fleet/override.json'))))\") $(/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\\n' ' ') miner=$(pgrep -f '^[/]opt/igneum/pkg/bin/igneum-miner mine' | wc -l) rej_since=$(awk '$2 >= \"22:47:00\"' /root/fleet/node.log | grep -c 'got reject message') v4=\"$(grep -E 'Program class v4 (signal window|from the override)' /root/fleet/node.log | tail -2 | sed -E 's/^[0-9: .+-]*//' | cut -c1-70 | tr '\\n' ';')\"", 60)
|
||||
return l, out.strip().split("\n")[-1]
|
||||
except Exception as e: return l, "EXC "+str(e)[:60]
|
||||
with ThreadPoolExecutor(14) as ex: res=list(ex.map(read, live))
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
#!/usr/bin/env bash
|
||||
# Stops a rehearsal box's miner loops, miner, worker and sampler but NOT its dn400 node: run as a FILE (an inline pkill
|
||||
# carrying these words kills the ssh shell that carries them). Then MINER_ONLY=1 bash in/box-rehearsal.sh restarts the miner.
|
||||
pkill -9 -f '^bash in/box-rehearsal.sh'; pkill -9 -f '^/root/fleet/in/igneum-miner-v4 '; pkill -9 -f '^/opt/igneum/pkg/bin/igneum-worker-cuda --serve .*dn400'
|
||||
sleep 1; echo "left=$(pgrep -c -f '^bash in/box-rehearsal.sh')+$(pgrep -c -f '^/root/fleet/in/igneum-miner-v4 ') node=$(pgrep -c -f '^/root/fleet/in/igneumd')"
|
||||
pkill -9 -f '^[b]ash in/box-rehearsal.sh'; pkill -9 -f '^[/]root/fleet/in/igneum-miner-v4 '; pkill -9 -f '^[/]opt/igneum/pkg/bin/igneum-worker-cuda --serve .*dn400'
|
||||
sleep 1; echo "left=$(pgrep -c -f '^[b]ash in/box-rehearsal.sh')+$(pgrep -c -f '^[/]root/fleet/in/igneum-miner-v4 ') node=$(pgrep -c -f '^[/]root/fleet/in/igneumd')"
|
||||
|
|
|
|||
Loading…
Reference in a new issue