From a36ea68b45311a7a2f7dbcaa034371210b53f82a Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Sun, 4 Oct 2026 18:59:13 +0000 Subject: [PATCH] Jobs reader: the app's closing report counts as final and error lines are collected; prove-shard.sh fails the job when a stage fails Three watchers never saw a job finish because the closing upload starts with the app header, not the SUMMARY line, and a shard run whose stages failed still exited 0. Co-Authored-By: Claude Fable 5.1 --- proving/windows-wsl2/prove-shard.sh | 6 ++++-- tools/jobs.mjs | 14 +++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/proving/windows-wsl2/prove-shard.sh b/proving/windows-wsl2/prove-shard.sh index 3981e1bc..71535c7a 100755 --- a/proving/windows-wsl2/prove-shard.sh +++ b/proving/windows-wsl2/prove-shard.sh @@ -50,18 +50,19 @@ if ! cargo build --release -p igneum-prove-host --features igneum-prove-host/cud fi HOST="$DEST/proving/igneum-prove/target/release/igneum-prove-host" FIXDIR="$DEST/proving/fixtures" +FAILED="" mkdir -p "$HERE/results" "$DEST/results" # proofs are written on the Linux side (ext4) and copied to the Windows folder once per stage: every byte through /mnt/c costs a bridge round trip echo "=== GPU shard run: SP1_PROVER=cuda, $SHARD_FIXTURE, shard 0: execute + core + compressed (the first run downloads sp1-gpu-server, about 134 MB) ===" SP1_PROVER=cuda RUST_LOG=info "$HOST" "$FIXDIR/$SHARD_FIXTURE.json" --mode shard --shard 0 --out "$DEST/results/$SHARD_FIXTURE-cuda-$STAMP.json" -echo "gpu shard run exit $? at $(date -u +%FT%TZ)" +rc=$?; echo "gpu shard run exit $rc at $(date -u +%FT%TZ)"; [ "$rc" = 0 ] || FAILED="$FAILED shard-run" cp -f "$DEST/results/"* "$HERE/results/" 2>/dev/null upload for F in $BLOCK_FIXTURES; do echo "=== GPU block run: SP1_PROVER=cuda, $F: compressed proof per shard + aggregation ===" SP1_PROVER=cuda RUST_LOG=info "$HOST" "$FIXDIR/$F.json" --mode block --out "$DEST/results/$F-cuda-$STAMP.json" - echo "gpu block run exit $? at $(date -u +%FT%TZ)" + rc=$?; echo "gpu block run exit $rc at $(date -u +%FT%TZ)"; [ "$rc" = 0 ] || FAILED="$FAILED block-$F" cp -f "$DEST/results/"* "$HERE/results/" 2>/dev/null upload done @@ -70,3 +71,4 @@ echo "=== SUMMARY (RESULT lines) ===" grep -h '^RESULT\|^STAGE\|^fixture\|exit' "$LOG" | sed 's/^/ /' upload echo "log: $LOG (uploaded as run_id $RUN_ID; on the Mac: node tools/logs.mjs $RUN_ID)" +if [ -n "${FAILED:-}" ]; then echo "FAILED stages:$FAILED"; exit 1; fi diff --git a/tools/jobs.mjs b/tools/jobs.mjs index 1b7c9381..8034bba8 100755 --- a/tools/jobs.mjs +++ b/tools/jobs.mjs @@ -28,7 +28,19 @@ function db() { }; } const when = ts => new Date(ts).toISOString().replace('T', ' ').slice(0, 19) + ' UTC'; -const summaryOf = lines => { const l = (lines || '').split('\n')[0]; if (!l.startsWith('SUMMARY ')) return null; try { return JSON.parse(l.slice(8)); } catch { return null; } }; +// The SUMMARY line is the first line of a progress upload but the app's closing report starts with its IGNEUM-APP +// header (4 October 2026: three watchers never saw a job finish). Scan the first lines for it, and read the closing +// 'job : (exit N)' line as final too. +const summaryOf = lines => { + const ls = (lines || '').split('\n'); + let s = null; + for (const l of ls.slice(0, 12)) { if (l.startsWith('SUMMARY ')) { try { s = JSON.parse(l.slice(8)); } catch { s = null; } break; } } + const close = ls.slice(-40).map(l => /^job \S+: (done|failed|timeout|aborted) \(exit (-?\d+)\) after (\d+) s: (.*)$/.exec(l)).filter(Boolean).pop(); + if (close) { s = s || {}; s.status = close[1]; s.exit = Number(close[2]); s.duration_s = Number(close[3]); s.summary = close[4]; s.finished_at = s.finished_at || 'closed'; } + else if (s && !s.finished_at) s.status = 'running'; + if (s) s.errors = ls.filter(l => /^(Error:|BUILD FAILED|GATE FAILED)|\bFAILED\b|panicked at/.test(l)).slice(0, 5); + return s; +}; const [a, b] = process.argv.slice(2);