From 8e010d84d6f2ddd956672d8a81d0cd44e907a192 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:45:32 +0000 Subject: [PATCH] Standing supervisor: prunes the prover's segment exports and trims the node log (the hub died on a full disk at 20:42Z); disk in the status line Co-Authored-By: Claude Fable 5.1 --- tools/fleet/box-standing.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tools/fleet/box-standing.sh b/tools/fleet/box-standing.sh index 0a32f05b6..353b7228b 100755 --- a/tools/fleet/box-standing.sh +++ b/tools/fleet/box-standing.sh @@ -60,11 +60,16 @@ while :; do fi n=$((n+1)) if [ $((n % 10)) = 1 ]; then + # disk: the prover's segment exports (/root/fleet/out/segs, 50 to 500 MB each) filled 60 GB boxes in nine hours (20:42Z: the + # hub's node died on "No space left on device"); keep the last 20 minutes of them and the node log under 300 MB + [ -d $OUT/segs ] && find $OUT/segs -mindepth 1 -maxdepth 1 -mmin +20 -exec rm -rf {} + 2>/dev/null + [ "$(stat -c %s $LOG 2>/dev/null || echo 0)" -gt 300000000 ] && { tail -c 100000000 $LOG > $LOG.tail && mv $LOG.tail $LOG; echo "RESULT standing_log_trimmed $(stamp)"; } + DISK="$(df -h / | tail -1 | awk '{print $4"/"$5}')" w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1 | sed -E 's/difficulty=[0-9.]* sink=[0-9a-f]* //')" ex=$(curl -s -m 5 127.0.0.1:26790 -H 'content-type: application/json' -d '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' | grep -oE '"result":"0x[0-9a-f]+"' | cut -d'"' -f4) exn=$((${ex:-0x0})); blocks=$(printf '%s' "$w" | grep -oE 'blocks=[0-9]+' | cut -d= -f2) m="$(grep STATUS $OUT/miner-0.log 2>/dev/null | tail -1 | grep -oE 'accepted=[0-9]+ rejected=[0-9]+|\([0-9.]+ MH/s inside jobs' | tr '\n' ' ' | tr -d '(')" - echo "RESULT standing $(stamp) up=$(( $(date -u +%s) - STARTED ))s role=$ROLE node=$(node_pid) bin=$cur version=$($cur --version 2>&1 | head -1 | awk '{print $2}') $w exec=$exn miner=\"$m\" prover=$(pgrep -c -f '^python3 -u /root/fleet/in/box-prover.py') gpu=$(nvidia-smi --query-gpu=utilization.gpu,power.draw --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')" >> $OUT/standing.log + echo "RESULT standing $(stamp) up=$(( $(date -u +%s) - STARTED ))s role=$ROLE node=$(node_pid) bin=$cur version=$($cur --version 2>&1 | head -1 | awk '{print $2}') $w exec=$exn miner=\"$m\" prover=$(pgrep -c -f '^python3 -u /root/fleet/in/box-prover.py') disk=$DISK gpu=$(nvidia-smi --query-gpu=utilization.gpu,power.draw --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')" >> $OUT/standing.log # the dead-exec class: synced consensus, exec at 0 from snapshot -> the shipper's recovery recipe (live devnet only) if [ "$ROLE" = live ] && [ "${blocks:-0}" -gt 1000 ] && [ "$exn" = 0 ] && [ -x $F/in/box-exec-snapshot.sh ] && [ $((n / 10)) -gt 2 ]; then echo "RESULT standing_recovery $(stamp) exec=0 blocks=$blocks: running box-exec-snapshot.sh"; HUB_PEER="$HUB_PEER" bash $F/in/box-exec-snapshot.sh; sleep 30