From c6dacb56902a9399eb69b309d4e16ae3be0f67b9 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 11:56:22 +0000 Subject: [PATCH] bench/proof-systems: run.sh portable (GNU time on Linux, host and out paths, a row label, the patched server's HOME), the table keys on the label and fixture Co-Authored-By: Claude Fable 5.1 --- bench/proof-systems/TABLE.md | 2 +- bench/proof-systems/run.sh | 29 +++++++++++++++++------------ bench/proof-systems/table.mjs | 6 +++--- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/bench/proof-systems/TABLE.md b/bench/proof-systems/TABLE.md index 1b658336f..a16858ece 100644 --- a/bench/proof-systems/TABLE.md +++ b/bench/proof-systems/TABLE.md @@ -1,4 +1,4 @@ -| Candidate | Machine, backend | Fixture | Cycles | Prove s | Wall s | Peak memory MiB | Proof bytes | Verify s (one core) | Recursion | Audit | Verified | Date | +| Candidate | Machine, backend, note | Fixture | Cycles | Prove s | Wall s | Peak memory MiB | Proof bytes | Verify s (one core) | Recursion | Audit | Verified | Date | |---|---|---|---|---|---|---|---|---|---|---|---|---| | jolt | MacBook-Pro, cpu | n/a: no guest for our shard statement and no shipped recursion (proving-methods.md route F) | | | | | | | | | | 20261006-1146 | | sp1-patched | MacBook-Pro, cpu | n/a: the patched server exists only on PC 2 (/opt/igneum-floor/home) | | | | | | | | | | 20261006-1146 | diff --git a/bench/proof-systems/run.sh b/bench/proof-systems/run.sh index 28e68ba51..c2b737783 100755 --- a/bench/proof-systems/run.sh +++ b/bench/proof-systems/run.sh @@ -4,21 +4,25 @@ # Run it through the measure lock on the Mac (tools/lock/with-lock.sh measure); on a CUDA box set --backend cuda. set -uo pipefail HERE="$(cd "$(dirname "$0")" && pwd)"; ROOT="$(cd "$HERE/../.." && pwd)" -CANDS="sp1,r0"; BACKEND=""; FIXTURE="$ROOT/proving/fixtures/fees-v1-shards2.json"; SHARD=0 -while [ $# -gt 0 ]; do case "$1" in --candidates) CANDS="$2"; shift 2;; --backend) BACKEND="$2"; shift 2;; --fixture) FIXTURE="$2"; shift 2;; --shard) SHARD="$2"; shift 2;; *) echo "unknown $1"; exit 2;; esac; done -MACHINE="$(hostname -s)"; DATE="$(date -u +%Y%m%d-%H%M)"; OUT="$HERE/rows"; mkdir -p "$OUT" -if [ -z "$BACKEND" ]; then if command -v nvidia-smi >/dev/null 2>&1; then BACKEND=cuda; elif [ "$(uname)" = Darwin ]; then BACKEND=metal; else BACKEND=cpu; fi; fi +CANDS="sp1,r0"; BACKEND=""; FIXTURE="$ROOT/proving/fixtures/fees-v1-shards2.json"; SHARD=0; LABEL=""; OUT="$HERE/rows" SP1_HOST="$ROOT/proving/igneum-prove/target/release/igneum-prove-host" R0_HOST="$ROOT/proving/igneum-prove-r0/target/release/igneum-prove-r0-host" -peak_mib() { # $1 = time log; the Mac's /usr/bin/time -l prints "maximum resident set size" in bytes, Linux's in KB - if [ "$(uname)" = Darwin ]; then awk '/maximum resident set size/ {printf "%d", $1/1048576}' "$1"; else awk '/Maximum resident set size/ {printf "%d", $NF/1024}' "$1"; fi; } +PATCHED_HOME="/opt/igneum-floor/home" +# --label: a note on the row ("beside the miner", "alone"); --sp1-host/--r0-host/--out: paths on a machine where the +# tree is not checked out (PC 2's WSL: /opt/igneum-segal/igneum-prove-host); --patched-home: the patched server's HOME +while [ $# -gt 0 ]; do case "$1" in --candidates) CANDS="$2"; shift 2;; --backend) BACKEND="$2"; shift 2;; --fixture) FIXTURE="$2"; shift 2;; --shard) SHARD="$2"; shift 2;; --label) LABEL="$2"; shift 2;; --sp1-host) SP1_HOST="$2"; shift 2;; --r0-host) R0_HOST="$2"; shift 2;; --out) OUT="$2"; shift 2;; --patched-home) PATCHED_HOME="$2"; shift 2;; *) echo "unknown $1"; exit 2;; esac; done +MACHINE="$(hostname -s)"; DATE="$(date -u +%Y%m%d-%H%M)"; mkdir -p "$OUT" +if [ -z "$BACKEND" ]; then if command -v nvidia-smi >/dev/null 2>&1; then BACKEND=cuda; elif [ "$(uname)" = Darwin ]; then BACKEND=metal; else BACKEND=cpu; fi; fi +# the time tool: the Mac's /usr/bin/time -l prints "maximum resident set size" in bytes, GNU time -v in KB +if [ "$(uname)" = Darwin ]; then TIME="/usr/bin/time -l"; else TIME="/usr/bin/time -v"; fi +peak_mib() { if [ "$(uname)" = Darwin ]; then awk '/maximum resident set size/ {printf "%d", $1/1048576}' "$1"; else awk '/Maximum resident set size/ {printf "%d", $NF/1024}' "$1"; fi; } row() { # candidate backend host mode-env verify-mode local cand="$1" backend="$2" host="$3" env="$4"; local name="$cand" local dir; dir="$(mktemp -d)"; local res="$dir/results.json"; local tlog="$dir/time.log"; local plog="$dir/prove.log" if [ ! -x "$host" ]; then printf '{"candidate":"%s","machine":"%s","backend":"%s","fixture":"%s","shard":%s,"na":"host not built: %s","date":"%s"}\n' "$cand" "$MACHINE" "$backend" "$(basename "$FIXTURE")" "$SHARD" "$host" "$DATE" > "$OUT/$DATE-$MACHINE-$cand.json"; echo "row $cand: n/a (no host)"; return; fi local smi=""; if [ "$backend" = cuda ]; then nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -l 1 > "$dir/smi.csv" 2>/dev/null & smi=$!; fi local t0; t0=$(date +%s.%N) - env $env /usr/bin/time -l "$host" "$FIXTURE" --mode compressed --shard "$SHARD" --prover 0xCAfc6e74000000000000000000000000000000c2 --out "$res" > "$plog" 2> "$tlog" + env $env $TIME "$host" "$FIXTURE" --mode compressed --shard "$SHARD" --prover 0xCAfc6e74000000000000000000000000000000c2 --out "$res" > "$plog" 2> "$tlog" local rc=$?; local wall; wall=$(python3 -c "import time; print(round(time.time() - $t0, 2))") [ -n "$smi" ] && { kill $smi 2>/dev/null; sleep 1; } local peak; if [ "$backend" = cuda ] && [ -s "$dir/smi.csv" ]; then peak=$(sort -n "$dir/smi.csv" | tail -1); else peak=$(peak_mib "$tlog"); fi @@ -26,9 +30,9 @@ row() { # candidate backend host mode-env verify-mode # the verify on one core, three runs local stmt pf; stmt=$(python3 -c "import json; print(json.load(open('$res'))['statement'])"); pf=$(python3 -c "import json; print(json.load(open('$res'))['proof_file'])") local vs=""; for i in 1 2 3; do local v0; v0=$(date +%s.%N); env $env "$host" --mode verify --proof "$pf" --statement "$stmt" > "$dir/verify-$i.log" 2>&1; local vrc=$?; vs="$vs$(python3 -c "import time; print(round(time.time() - $v0, 3))"),"; [ $vrc -ne 0 ] && { echo "row $cand: VERIFY FAILED"; break; }; done - python3 - "$res" "$cand" "$MACHINE" "$backend" "$FIXTURE" "$SHARD" "$wall" "$peak" "${vs%,}" "$vrc" "$DATE" "$env" "$OUT" <<'PY' + python3 - "$res" "$cand" "$MACHINE" "$backend" "$FIXTURE" "$SHARD" "$wall" "$peak" "${vs%,}" "$vrc" "$DATE" "$env" "$OUT" "$LABEL" <<'PY' import json, sys -res, cand, machine, backend, fixture, shard, wall, peak, vs, vrc, date, env, out = sys.argv[1:] +res, cand, machine, backend, fixture, shard, wall, peak, vs, vrc, date, env, out, label = sys.argv[1:] r = json.load(open(res)) vl = [float(x) for x in vs.split(',') if x] row = { @@ -36,9 +40,10 @@ row = { 'cycles': r.get('cycles'), 'wall_s': float(wall), 'prove_s': r.get('compressed_prove_seconds'), 'peak_memory_mib': int(peak) if str(peak).isdigit() else None, 'proof_bytes': r.get('compressed_proof_bytes'), 'verify_s': min(vl) if vl else None, 'verify_runs': vl, 'verified': vrc == '0' and bool(vl), 'recursion': cand in ('sp1', 'sp1-patched', 'r0'), 'audit': cand in ('sp1', 'r0'), 'proof_system': r.get('proof_system', 1), 'image_or_program_id': r.get('image_id') or r.get('program_id'), - 'command': f"{env} {sys.argv[0]} {fixture} --mode compressed --shard {shard}", 'source': 'bench/proof-systems/run.sh', + 'command': f"{env} {fixture} --mode compressed --shard {shard}", 'source': 'bench/proof-systems/run.sh', 'label': label, } -json.dump(row, open(f"{out}/{date}-{machine}-{cand}.json", 'w'), indent=1) +tag = ('-' + label.replace(' ', '-')) if label else '' +json.dump(row, open(f"{out}/{date}-{machine}-{cand}{tag}.json", 'w'), indent=1) print(f"row {cand}: prove {row['prove_s']} s, wall {row['wall_s']} s, peak {row['peak_memory_mib']} MiB, proof {row['proof_bytes']} B, verify {row['verify_s']} s, verified {row['verified']}") PY rm -rf "$dir" @@ -46,7 +51,7 @@ PY for c in ${CANDS//,/ }; do case "$c" in sp1) if [ "$BACKEND" = cuda ]; then row sp1 cuda "$SP1_HOST" "SP1_PROVER=cuda RUST_LOG=off"; else row sp1 cpu "$SP1_HOST" "SP1_PROVER=cpu RUST_LOG=off"; fi ;; - sp1-patched) if [ "$BACKEND" = cuda ] && [ -d /opt/igneum-floor/home ]; then row sp1-patched cuda "$SP1_HOST" "HOME=/opt/igneum-floor/home SP1_PROVER=cuda RUST_LOG=off"; else printf '{"candidate":"sp1-patched","machine":"%s","backend":"%s","na":"the patched server exists only on PC 2 (/opt/igneum-floor/home)","date":"%s"}\n' "$MACHINE" "$BACKEND" "$DATE" > "$OUT/$DATE-$MACHINE-sp1-patched.json"; echo "row sp1-patched: n/a"; fi ;; + sp1-patched) if [ "$BACKEND" = cuda ] && [ -d "$PATCHED_HOME" ]; then row sp1-patched cuda "$SP1_HOST" "HOME=$PATCHED_HOME SP1_PROVER=cuda RUST_LOG=off"; else printf '{"candidate":"sp1-patched","machine":"%s","backend":"%s","na":"the patched server exists only on PC 2 (/opt/igneum-floor/home)","date":"%s"}\n' "$MACHINE" "$BACKEND" "$DATE" > "$OUT/$DATE-$MACHINE-sp1-patched.json"; echo "row sp1-patched: n/a"; fi ;; r0) if [ "$BACKEND" = cuda ]; then row r0 cuda "$R0_HOST" "RISC0_PROVER=local RUST_LOG=off"; elif [ "$BACKEND" = metal ]; then row r0 metal "$R0_HOST" "RISC0_PROVER=local RUST_LOG=off"; else row r0 cpu "$R0_HOST" "RISC0_PROVER=local RUST_LOG=off"; fi ;; jolt) printf '{"candidate":"jolt","machine":"%s","backend":"%s","na":"no guest for our shard statement and no shipped recursion (proving-methods.md route F)","date":"%s"}\n' "$MACHINE" "$BACKEND" "$DATE" > "$OUT/$DATE-$MACHINE-jolt.json"; echo "row jolt: n/a" ;; *) echo "unknown candidate $c" ;; diff --git a/bench/proof-systems/table.mjs b/bench/proof-systems/table.mjs index 965a3d8aa..1d5a5bfea 100644 --- a/bench/proof-systems/table.mjs +++ b/bench/proof-systems/table.mjs @@ -3,11 +3,11 @@ import { readdirSync, readFileSync } from 'node:fs'; const dir = new URL('./rows/', import.meta.url).pathname; const rows = readdirSync(dir).filter(f => f.endsWith('.json')).map(f => ({ file: f, ...JSON.parse(readFileSync(dir + f, 'utf8')) })).sort((a, b) => (b.date || '').localeCompare(a.date || '')); const latest = new Map(); -for (const r of rows) { const k = `${r.candidate}@${r.machine}@${r.backend}`; if (!latest.has(k)) latest.set(k, r); } +for (const r of rows) { const k = `${r.candidate}@${r.machine}@${r.backend}@${r.label || ''}@${r.fixture || ''}`; if (!latest.has(k)) latest.set(k, r); } const fmt = (v, d = 1) => v === null || v === undefined ? 'n/a' : typeof v === 'number' ? v.toFixed(d) : String(v); -console.log('| Candidate | Machine, backend | Fixture | Cycles | Prove s | Wall s | Peak memory MiB | Proof bytes | Verify s (one core) | Recursion | Audit | Verified | Date |'); +console.log('| Candidate | Machine, backend, note | Fixture | Cycles | Prove s | Wall s | Peak memory MiB | Proof bytes | Verify s (one core) | Recursion | Audit | Verified | Date |'); console.log('|---|---|---|---|---|---|---|---|---|---|---|---|---|'); for (const r of latest.values()) { if (r.na) { console.log(`| ${r.candidate} | ${r.machine}, ${r.backend || 'n/a'} | n/a: ${r.na} | | | | | | | | | | ${r.date} |`); continue; } - console.log(`| ${r.candidate} | ${r.machine}, ${r.backend} | ${r.fixture} shard ${r.shard} | ${fmt(r.cycles, 0)} | ${fmt(r.prove_s)} | ${fmt(r.wall_s)} | ${fmt(r.peak_memory_mib, 0)} | ${fmt(r.proof_bytes, 0)} | ${fmt(r.verify_s, 3)} | ${r.recursion ? 'yes' : 'no'} | ${r.audit ? 'yes' : 'no'} | ${r.verified ? 'yes' : 'NO'} | ${r.date} |`); + console.log(`| ${r.candidate} | ${r.machine}, ${r.backend}${r.label ? ', ' + r.label : ''} | ${r.fixture} shard ${r.shard} | ${fmt(r.cycles, 0)} | ${fmt(r.prove_s)} | ${fmt(r.wall_s)} | ${fmt(r.peak_memory_mib, 0)} | ${fmt(r.proof_bytes, 0)} | ${fmt(r.verify_s, 3)} | ${r.recursion ? 'yes' : 'no'} | ${r.audit ? 'yes' : 'no'} | ${r.verified ? 'yes' : 'NO'} | ${r.date} |`); }