diff --git a/tools/fleet/box-standing.sh b/tools/fleet/box-standing.sh index cd2eb1c12..f82945240 100755 --- a/tools/fleet/box-standing.sh +++ b/tools/fleet/box-standing.sh @@ -49,7 +49,13 @@ miner_loop() { n=0 while :; do cur=$(node_bin); rb=$(node_running_bin) - if [ -z "$rb" ]; then start_node + if [ -z "$rb" ]; then + # a crash loop on a corrupt datadir (22:10Z: p2-4090-1b, "Corruption: Can't access /029017.sst", 24 restarts): three starts in five + # minutes with a Corruption or "No space left" line in the log's tail move the datadir aside and the node joins fresh + if [ "$(grep -c standing_node_started $OUT/standing-supervisor.log 2>/dev/null)" -ge 3 ] && tail -40 $LOG 2>/dev/null | grep -qE 'Corruption|No such file or directory: while stat' && [ "$(find $OUT/standing-supervisor.log -mmin -5 | wc -l)" = 1 ] && [ "$(grep standing_node_started $OUT/standing-supervisor.log | tail -3 | awk '{print $3}' | sort -u | wc -l)" = 3 ]; then + mv $APP $APP.corrupt-$(date -u +%H%M); mkdir -p $APP; echo "RESULT standing_datadir_reset $(stamp) corrupt datadir moved to $APP.corrupt-$(date -u +%H%M); the node joins fresh" + fi + start_node elif [ -n "$cur" ] && [ "$rb" != "$cur" ] && [ -s $F/standing.node ]; then echo "RESULT standing_node_update $(stamp) from=$rb to=$cur"; start_node; fi if [ "$MINE" = 1 ] && ! pgrep -f '^/opt/igneum/pkg/bin/igneum-miner mine' >/dev/null && [ -z "${MINER_LOOP_PID:-}" -o ! -e "/proc/${MINER_LOOP_PID:-0}" ]; then w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'synced=[a-z]*' | tail -1)"