From a86b53273606eccee0fdc4560fce1620bdcb88e5 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 22:13:09 +0000 Subject: [PATCH] Standing supervisor: a crash loop on a corrupt datadir moves it aside and rejoins fresh (p2-4090-1b, 24 restarts on a missing sst) Co-Authored-By: Claude Fable 5.1 --- tools/fleet/box-standing.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/fleet/box-standing.sh b/tools/fleet/box-standing.sh index cd2eb1c12..f82945240 100755 --- a/tools/fleet/box-standing.sh +++ b/tools/fleet/box-standing.sh @@ -49,7 +49,13 @@ miner_loop() { n=0 while :; do cur=$(node_bin); rb=$(node_running_bin) - if [ -z "$rb" ]; then start_node + if [ -z "$rb" ]; then + # a crash loop on a corrupt datadir (22:10Z: p2-4090-1b, "Corruption: Can't access /029017.sst", 24 restarts): three starts in five + # minutes with a Corruption or "No space left" line in the log's tail move the datadir aside and the node joins fresh + if [ "$(grep -c standing_node_started $OUT/standing-supervisor.log 2>/dev/null)" -ge 3 ] && tail -40 $LOG 2>/dev/null | grep -qE 'Corruption|No such file or directory: while stat' && [ "$(find $OUT/standing-supervisor.log -mmin -5 | wc -l)" = 1 ] && [ "$(grep standing_node_started $OUT/standing-supervisor.log | tail -3 | awk '{print $3}' | sort -u | wc -l)" = 3 ]; then + mv $APP $APP.corrupt-$(date -u +%H%M); mkdir -p $APP; echo "RESULT standing_datadir_reset $(stamp) corrupt datadir moved to $APP.corrupt-$(date -u +%H%M); the node joins fresh" + fi + start_node elif [ -n "$cur" ] && [ "$rb" != "$cur" ] && [ -s $F/standing.node ]; then echo "RESULT standing_node_update $(stamp) from=$rb to=$cur"; start_node; fi if [ "$MINE" = 1 ] && ! pgrep -f '^/opt/igneum/pkg/bin/igneum-miner mine' >/dev/null && [ -z "${MINER_LOOP_PID:-}" -o ! -e "/proc/${MINER_LOOP_PID:-0}" ]; then w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'synced=[a-z]*' | tail -1)"