igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

Standing supervisor: a crash loop on a corrupt datadir moves it aside and rejoins fresh (p2-4090-1b, 24 restarts on a missing sst)

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-06 22:13:09 +00:00
parent 552ef811f9
commit a86b532736
1 changed files with 7 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
tools/fleet/box-standing.sh
View file

@ -49,7 +49,13 @@ miner_loop() {
n=0
while :; do
cur=$(node_bin); rb=$(node_running_bin)
if [ -z "$rb" ]; then start_node
if [ -z "$rb" ]; then
# a crash loop on a corrupt datadir (22:10Z: p2-4090-1b, "Corruption: Can't access /029017.sst", 24 restarts): three starts in five
# minutes with a Corruption or "No space left" line in the log's tail move the datadir aside and the node joins fresh
if [ "$(grep -c standing_node_started $OUT/standing-supervisor.log 2>/dev/null)" -ge 3 ] && tail -40 $LOG 2>/dev/null | grep -qE 'Corruption|No such file or directory: while stat' && [ "$(find $OUT/standing-supervisor.log -mmin -5 | wc -l)" = 1 ] && [ "$(grep standing_node_started $OUT/standing-supervisor.log | tail -3 | awk '{print $3}' | sort -u | wc -l)" = 3 ]; then
mv $APP $APP.corrupt-$(date -u +%H%M); mkdir -p $APP; echo "RESULT standing_datadir_reset $(stamp) corrupt datadir moved to $APP.corrupt-$(date -u +%H%M); the node joins fresh"
fi
start_node
elif [ -n "$cur" ] && [ "$rb" != "$cur" ] && [ -s $F/standing.node ]; then echo "RESULT standing_node_update $(stamp) from=$rb to=$cur"; start_node; fi
if [ "$MINE" = 1 ] && ! pgrep -f '^/opt/igneum/pkg/bin/igneum-miner mine' >/dev/null && [ -z "${MINER_LOOP_PID:-}" -o ! -e "/proc/${MINER_LOOP_PID:-0}" ]; then
w="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'synced=[a-z]*' | tail -1)"