igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

Jobs reader: the app's closing report counts as final and error lines are collected; prove-shard.sh fails the job when a stage fails

Browse source 
Three watchers never saw a job finish because the closing upload starts with the app header, not the SUMMARY line,
and a shard run whose stages failed still exited 0.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-04 18:59:13 +00:00
parent 48868dbb03
commit 11b01cbabd
2 changed files with 17 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
proving/windows-wsl2/prove-shard.sh
View file

@ -50,18 +50,19 @@ if ! cargo build --release -p igneum-prove-host --features igneum-prove-host/cud
fi
HOST="$DEST/proving/igneum-prove/target/release/igneum-prove-host"
FIXDIR="$DEST/proving/fixtures"
FAILED=""
mkdir -p "$HERE/results" "$DEST/results" # proofs are written on the Linux side (ext4) and copied to the Windows folder once per stage: every byte through /mnt/c costs a bridge round trip
echo "=== GPU shard run: SP1_PROVER=cuda, $SHARD_FIXTURE, shard 0: execute + core + compressed (the first run downloads sp1-gpu-server, about 134 MB) ==="
SP1_PROVER=cuda RUST_LOG=info "$HOST" "$FIXDIR/$SHARD_FIXTURE.json" --mode shard --shard 0 --out "$DEST/results/$SHARD_FIXTURE-cuda-$STAMP.json"
echo "gpu shard run exit $? at $(date -u +%FT%TZ)"
rc=$?; echo "gpu shard run exit $rc at $(date -u +%FT%TZ)"; [ "$rc" = 0 ] || FAILED="$FAILED shard-run"
cp -f "$DEST/results/"* "$HERE/results/" 2>/dev/null
upload
for F in $BLOCK_FIXTURES; do
echo "=== GPU block run: SP1_PROVER=cuda, $F: compressed proof per shard + aggregation ==="
SP1_PROVER=cuda RUST_LOG=info "$HOST" "$FIXDIR/$F.json" --mode block --out "$DEST/results/$F-cuda-$STAMP.json"
echo "gpu block run exit $? at $(date -u +%FT%TZ)"
rc=$?; echo "gpu block run exit $rc at $(date -u +%FT%TZ)"; [ "$rc" = 0 ] || FAILED="$FAILED block-$F"
cp -f "$DEST/results/"* "$HERE/results/" 2>/dev/null
upload
done
@ -70,3 +71,4 @@ echo "=== SUMMARY (RESULT lines) ==="
grep -h '^RESULT\|^STAGE\|^fixture\|exit' "$LOG" | sed 's/^/ /'
upload
echo "log: $LOG (uploaded as run_id $RUN_ID; on the Mac: node tools/logs.mjs $RUN_ID)"
if [ -n "${FAILED:-}" ]; then echo "FAILED stages:$FAILED"; exit 1; fi

14
tools/jobs.mjs
View file

@ -28,7 +28,19 @@ function db() {
};
}
const when = ts => new Date(ts).toISOString().replace('T', ' ').slice(0, 19) + ' UTC';
const summaryOf = lines => { const l = (lines || '').split('\n')[0]; if (!l.startsWith('SUMMARY ')) return null; try { return JSON.parse(l.slice(8)); } catch { return null; } };
// The SUMMARY line is the first line of a progress upload but the app's closing report starts with its IGNEUM-APP
// header (4 October 2026: three watchers never saw a job finish). Scan the first lines for it, and read the closing
// 'job <id>: <status> (exit N)' line as final too.
const summaryOf = lines => {
const ls = (lines || '').split('\n');
let s = null;
for (const l of ls.slice(0, 12)) { if (l.startsWith('SUMMARY ')) { try { s = JSON.parse(l.slice(8)); } catch { s = null; } break; } }
const close = ls.slice(-40).map(l => /^job \S+: (done|failed|timeout|aborted) \(exit (-?\d+)\) after (\d+) s: (.*)$/.exec(l)).filter(Boolean).pop();
if (close) { s = s || {}; s.status = close[1]; s.exit = Number(close[2]); s.duration_s = Number(close[3]); s.summary = close[4]; s.finished_at = s.finished_at || 'closed'; }
else if (s && !s.finished_at) s.status = 'running';
if (s) s.errors = ls.filter(l => /^(Error:|BUILD FAILED|GATE FAILED)|\bFAILED\b|panicked at/.test(l)).slice(0, 5);
return s;
};
const [a, b] = process.argv.slice(2);