From 73e6d5766d1ff4fee0d03334a3d3adabffd7dfd4 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:53:54 +0000 Subject: [PATCH] bench/proof-systems: a wall-clock cap per prove run (the patched server hangs on a card too small, the fleet finding), --r0-po2 Co-Authored-By: Claude Fable 5.1 --- bench/proof-systems/run.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bench/proof-systems/run.sh b/bench/proof-systems/run.sh index 2bc93afd0..e712e5a1b 100755 --- a/bench/proof-systems/run.sh +++ b/bench/proof-systems/run.sh @@ -7,14 +7,18 @@ HERE="$(cd "$(dirname "$0")" && pwd)"; ROOT="$(cd "$HERE/../.." && pwd)" CANDS="sp1,r0"; BACKEND=""; FIXTURE="$ROOT/proving/fixtures/fees-v1-shards2.json"; SHARD=0; LABEL=""; OUT="$HERE/rows" SP1_HOST="$ROOT/proving/igneum-prove/target/release/igneum-prove-host" R0_HOST="$ROOT/proving/igneum-prove-r0/target/release/igneum-prove-r0-host" -PATCHED_HOME="/opt/igneum-floor/home"; PATCHED_ENV="" +PATCHED_HOME="/opt/igneum-floor/home"; PATCHED_ENV=""; R0_PO2="" # --label: a note on the row ("beside the miner", "alone"); --sp1-host/--r0-host/--out: paths on a machine where the # tree is not checked out (PC 2's WSL: /opt/igneum-segal/igneum-prove-host); --patched-home: the patched server's HOME -while [ $# -gt 0 ]; do case "$1" in --candidates) CANDS="$2"; shift 2;; --backend) BACKEND="$2"; shift 2;; --fixture) FIXTURE="$2"; shift 2;; --shard) SHARD="$2"; shift 2;; --label) LABEL="$2"; shift 2;; --sp1-host) SP1_HOST="$2"; shift 2;; --r0-host) R0_HOST="$2"; shift 2;; --out) OUT="$2"; shift 2;; --patched-home) PATCHED_HOME="$2"; shift 2;; --patched-env) PATCHED_ENV="$2"; shift 2;; *) echo "unknown $1"; exit 2;; esac; done +while [ $# -gt 0 ]; do case "$1" in --candidates) CANDS="$2"; shift 2;; --backend) BACKEND="$2"; shift 2;; --fixture) FIXTURE="$2"; shift 2;; --shard) SHARD="$2"; shift 2;; --label) LABEL="$2"; shift 2;; --sp1-host) SP1_HOST="$2"; shift 2;; --r0-host) R0_HOST="$2"; shift 2;; --out) OUT="$2"; shift 2;; --patched-home) PATCHED_HOME="$2"; shift 2;; --patched-env) PATCHED_ENV="$2"; shift 2;; --r0-po2) R0_PO2="$2"; shift 2;; *) echo "unknown $1"; exit 2;; esac; done MACHINE="$(hostname -s)"; DATE="$(date -u +%Y%m%d-%H%M)"; mkdir -p "$OUT" if [ -z "$BACKEND" ]; then if command -v nvidia-smi >/dev/null 2>&1; then BACKEND=cuda; elif [ "$(uname)" = Darwin ]; then BACKEND=metal; else BACKEND=cpu; fi; fi # the time tool: the Mac's /usr/bin/time -l prints "maximum resident set size" in bytes, GNU time -v in KB if [ "$(uname)" = Darwin ]; then TIME="/usr/bin/time -l"; else TIME="/usr/bin/time -v"; fi +# the wall-clock cap (the fleet finding, 6 October 2026: on a card too small for a profile the patched sp1-gpu-server +# hangs at the card's limit with 0% utilisation instead of failing); a run past it is a row that says "timed out" +CAP_S="${BENCH_CAP_S:-900}" +if command -v timeout >/dev/null 2>&1; then CAP="timeout -k 10 $CAP_S"; elif command -v gtimeout >/dev/null 2>&1; then CAP="gtimeout -k 10 $CAP_S"; else CAP=""; fi peak_mib() { if [ "$(uname)" = Darwin ]; then awk '/maximum resident set size/ {printf "%d", $1/1048576}' "$1"; else awk '/Maximum resident set size/ {printf "%d", $NF/1024}' "$1"; fi; } row() { # candidate backend host mode-env verify-mode local cand="$1" backend="$2" host="$3" env="$4"; local name="$cand" @@ -22,8 +26,10 @@ row() { # candidate backend host mode-env verify-mode if [ ! -x "$host" ]; then printf '{"candidate":"%s","machine":"%s","backend":"%s","fixture":"%s","shard":%s,"na":"host not built: %s","date":"%s"}\n' "$cand" "$MACHINE" "$backend" "$(basename "$FIXTURE")" "$SHARD" "$host" "$DATE" > "$OUT/$DATE-$MACHINE-$cand.json"; echo "row $cand: n/a (no host)"; return; fi local smi=""; if [ "$backend" = cuda ]; then nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -l 1 > "$dir/smi.csv" 2>/dev/null & smi=$!; fi local t0; t0=$(date +%s.%N) - env $env $TIME "$host" "$FIXTURE" --mode compressed --shard "$SHARD" --prover 0xCAfc6e74000000000000000000000000000000c2 --out "$res" > "$plog" 2> "$tlog" - local rc=$?; local wall; wall=$(python3 -c "import time; print(round(time.time() - $t0, 2))") + local extra=""; if [ "$cand" = r0 ] && [ -n "$R0_PO2" ]; then extra="--po2 $R0_PO2"; fi + env $env $CAP $TIME "$host" "$FIXTURE" --mode compressed --shard "$SHARD" --prover 0xCAfc6e74000000000000000000000000000000c2 $extra --out "$res" > "$plog" 2> "$tlog" + local rc=$?; [ $rc -eq 124 ] || [ $rc -eq 137 ] && echo "RESULT timed out after $CAP_S s" >> "$plog" + pkill -f sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock local wall; wall=$(python3 -c "import time; print(round(time.time() - $t0, 2))") [ -n "$smi" ] && { kill $smi 2>/dev/null; sleep 1; } local peak; if [ "$backend" = cuda ] && [ -s "$dir/smi.csv" ]; then peak=$(sort -n "$dir/smi.csv" | tail -1); else peak=$(peak_mib "$tlog"); fi if [ $rc -ne 0 ] || [ ! -f "$res" ]; then printf '{"candidate":"%s","machine":"%s","backend":"%s","fixture":"%s","shard":%s,"na":"the prove run failed (exit %s): %s","date":"%s"}\n' "$cand" "$MACHINE" "$backend" "$(basename "$FIXTURE")" "$SHARD" "$rc" "$(grep -E 'RESULT|rror' "$plog" "$tlog" | tail -1 | tr -d '"' | cut -c1-200)" "$DATE" > "$OUT/$DATE-$MACHINE-$cand.json"; echo "row $cand: failed"; return; fi