From 7f17d9b9827f1b2f7041af88765ca41d0925db27 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:45:27 +0000 Subject: [PATCH] Packaging row for 0.3.13: the project's GPU prover server built on CI and the PCs from one recipe, signed on the Mac with the OTA key, carried by the Windows payload, verified and installed by the app (proverserver.rs), the per-card tiers from the measured rows (12 GB at 2^26 mines and proves, 16 GB at 2^27, 24/32 GB stock sizes, under 10 GB off), Settings override, /api/state server fields, the stock fallback; tests, plan section, analysis Co-Authored-By: Claude Fable 5.1 --- .github/workflows/prover-server.yml | 88 +++++++++ .github/workflows/windows.yml | 16 ++ app/igneum-app/src/bin/ota-sign.rs | 22 ++- app/igneum-app/src/config.rs | 6 +- app/igneum-app/src/engine.rs | 6 +- app/igneum-app/src/main.rs | 1 + app/igneum-app/src/provedefault.rs | 124 +++++++++++- app/igneum-app/src/prover.rs | 129 ++++++++++++- app/igneum-app/src/proverserver.rs | 285 ++++++++++++++++++++++++++++ app/igneum-app/src/state.rs | 9 + docs/analysis/prover-floor.md | 12 ++ docs/plans/proving-v1.md | 39 +++- packaging/prover/build-server.sh | 64 +++++++ packaging/prover/fetch-server.sh | 26 +++ packaging/prover/push-server.sh | 63 ++++++ packaging/windows/make-payload.sh | 14 ++ 16 files changed, 891 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/prover-server.yml create mode 100644 app/igneum-app/src/proverserver.rs create mode 100755 packaging/prover/build-server.sh create mode 100755 packaging/prover/fetch-server.sh create mode 100755 packaging/prover/push-server.sh diff --git a/.github/workflows/prover-server.yml b/.github/workflows/prover-server.yml new file mode 100644 index 000000000..3b13bfa29 --- /dev/null +++ b/.github/workflows/prover-server.yml @@ -0,0 +1,88 @@ +# The project's GPU prover server (prover floor, 6 October 2026): SP1's sp1-gpu-server 6.8.1 rebuilt from source +# with proving/prover-floor/sp1-gpu-6.8.1-floor.patch, which sizes the server's buffers to the shard instead of +# to a 24 GB card (bench-log "prover floor": the RTX 4070 12 GB mines and proves at threshold 2^26). One recipe, +# packaging/prover/build-server.sh, shared with the PCs' WSL2 path (tools/prover-floor/pc2-build-server.ps1). +# +# What comes out: the artifact sp1-gpu-server-floor (the binary, its sha256, build.json). Nothing is signed here: +# packaging/prover/push-server.sh on the Mac downloads the artifact by run id, writes prover-server.json (the SP1 +# version and commit, the patch's sha256, the CUDA targets, the binary's sha256 and size, this run's id), signs it +# with the OTA key the apps trust and publishes the three files to the downloads host; the Windows build +# (windows.yml) fetches and verifies them for the payload; the app verifies them again before use. +# +# Not byte-reproducible across machines (SP1's prover-types build script stamps the build time into the binary), so +# the signed manifest names THIS build; the PC build is a behavioural cross-check (the same fixtures, verified). +# The CUDA toolkit comes from Jimver/cuda-toolkit (nvcc, nvtx, cudart only), as SP1's own release workflow does. +name: prover-server +on: + workflow_dispatch: + inputs: + cuda_archs: + description: "CUDA_ARCHS (default 80,86,89,120: 3060/3090 sm_86, 4060 to 4090 sm_89, 5080/5090 sm_120, A100 sm_80)" + default: "80,86,89,120" + required: false + push: + branches: [master] + paths: + - 'proving/prover-floor/sp1-gpu-6.8.1-floor.patch' + - 'packaging/prover/build-server.sh' + - '.github/workflows/prover-server.yml' +permissions: + contents: read +jobs: + build: + name: sp1-gpu-server 6.8.1 + floor patch (linux x86_64) + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - uses: actions/checkout@v4 + - name: free disk (the CUDA toolkit and the SP1 tree need about 20 GB) + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL || true + df -h / | tail -1 + - name: CUDA 12.8.1 (nvcc, nvtx, cudart) + uses: Jimver/cuda-toolkit@v0.2.23 + with: + cuda: '12.8.1' + method: 'network' + sub-packages: '["nvcc", "nvtx", "cudart"]' + use-github-cache: false + use-local-cache: false + - name: Go 1.27 (the server's native-gnark feature) + uses: actions/setup-go@v5 + with: + go-version: '1.27.1' + - name: protoc, cmake, clang + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq protobuf-compiler cmake clang pkg-config libssl-dev + nvcc --version | tail -1; protoc --version; cmake --version | head -1; go version + - name: Rust stable + run: rustup toolchain install stable --profile minimal && rustup default stable + - name: cargo cache + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + /tmp/igneum-sp1-6.8.1/target + key: prover-server-${{ runner.os }}-${{ hashFiles('proving/prover-floor/sp1-gpu-6.8.1-floor.patch') }}-${{ github.event.inputs.cuda_archs || '80,86,89,120' }} + restore-keys: prover-server-${{ runner.os }}- + - name: build (packaging/prover/build-server.sh) + env: + CUDA_ARCHS: ${{ github.event.inputs.cuda_archs || '80,86,89,120' }} + CARGO_JOBS: 4 + run: | + set -euo pipefail + bash packaging/prover/build-server.sh build/prover-server /tmp/igneum-sp1-6.8.1 + cat build/prover-server/build.json + cat build/prover-server/sp1-gpu-server.sha256 + - name: the binary answers --version + run: | + v="$(build/prover-server/sp1-gpu-server --version)" + echo "version: $v"; [ "$v" = "6.8.1" ] || { echo "::error::the server reports $v, not 6.8.1 (the SDK refuses it)"; exit 1; } + - uses: actions/upload-artifact@v4 + with: + name: sp1-gpu-server-floor + path: build/prover-server/ + if-no-files-found: error + retention-days: 90 diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 377b452e9..c3e008f34 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -152,6 +152,19 @@ jobs: echo "files: $(ls "$HOME/.config/igneum" | tr '\n' ' ')" bash packaging/mac/packaged-config.sh --test + - name: prover server (sp1-gpu-server from the downloads host, its signed manifest verified; absent = stock server) + shell: bash + env: + DL_TOKEN: ${{ secrets.DL_TOKEN }} + run: | + set -euo pipefail + signer="app/igneum-app/target/release/igneum-ota-sign.exe" + [ -x "$signer" ] || { echo "::error::$signer was not built by the engine step"; exit 1; } + # prover floor (packaging/prover/fetch-server.sh, the same checks): a host without the manifest ships no + # server (the app runs SP1's stock one, 24 GB cards); a manifest that does not verify fails the build + DL_TOKEN="$DL_TOKEN" bash packaging/prover/fetch-server.sh build/prover-server --signer "$signer" + ls -la build/prover-server || true + - name: payload inputs (payload-inputs.zip from the downloads host, signature, hashes and node commit verified) shell: bash env: @@ -198,6 +211,9 @@ jobs: export IGNEUM_WIN_RELEASE="$PWD/build/inputs" export IGNEUM_WORKERS_DIR="$PWD/build/inputs" export IGNEUM_APP_EXE="$PWD/app/igneum-app/target/release/igneum-app.exe" + # prover floor: the verified server folder from the step above (empty = the stock server) and the signer + export IGNEUM_PROVER_SERVER="$PWD/build/prover-server" + export IGNEUM_OTA_SIGN="$PWD/app/igneum-app/target/release/igneum-ota-sign.exe" packaging/windows/make-payload.sh "$PWD/packaging/windows/dist/igneum-windows-app.zip" test -f "packaging/windows/igneum-windows-app/Igneum Miner.exe" || { echo "::error::the window host did not land in the payload"; exit 1; } diff --git a/app/igneum-app/src/bin/ota-sign.rs b/app/igneum-app/src/bin/ota-sign.rs index 32c26df46..ef444e1ec 100644 --- a/app/igneum-app/src/bin/ota-sign.rs +++ b/app/igneum-app/src/bin/ota-sign.rs @@ -24,6 +24,9 @@ mod manifest; mod jobs; #[path = "../inputs.rs"] mod inputs; +#[path = "../proverserver.rs"] +#[allow(dead_code)] +mod proverserver; use ed25519_dalek::{Signer, SigningKey}; use std::path::Path; @@ -83,6 +86,23 @@ fn main() { Err(e) => die(&e), } } + // the project's GPU prover server (src/proverserver.rs): `verify-server + // [--binary ]` checks the signature, parses the manifest and, with + // --binary, the binary's sha256 and size; prints the server's version, sha256 and targets + Some("verify-server") if args.len() == 4 || args.len() == 6 => { + let pk = if args[1] == "embedded" { manifest::OTA_PUBLIC_KEY_HEX.to_string() } else { read_key_arg(&args[1]) }; + let bytes = std::fs::read(&args[2]).unwrap_or_else(|e| die(&format!("{}: {e}", args[2]))); + let sig = std::fs::read_to_string(&args[3]).unwrap_or_else(|e| die(&format!("{}: {e}", args[3]))); + manifest::verify_signature(&bytes, sig.trim(), &pk).unwrap_or_else(|e| die(&e)); + let m = proverserver::parse(&String::from_utf8_lossy(&bytes)).unwrap_or_else(|e| die(&e)); + if args.len() == 6 { + if args[4] != "--binary" { + die("verify-server [--binary ]"); + } + proverserver::check_binary(&m, Path::new(&args[5])).unwrap_or_else(|e| die(&e)); + } + println!("ok: sp1-gpu-server {} ({} bytes, sha256 {}), SP1 {} {}, patch {}, targets {}, built {} by {}", m.sha256, m.bytes, &m.sha256[..16], m.sp1_version, m.sp1_commit, &m.patch_sha256[..16.min(m.patch_sha256.len())], m.cuda_archs, m.built_at, m.built_by); + } Some("embedded") if args.len() == 1 => { println!("{}", manifest::OTA_PUBLIC_KEY_HEX); println!("fingerprint sha256:{}", manifest::fingerprint(manifest::OTA_PUBLIC_KEY_HEX)); @@ -189,7 +209,7 @@ fn main() { println!("ok: inputs built {} from node commit {} ({}); checked: {}", m.built_at, m.node_source_commit, m.node_source_branch, checked.join(", ")); } _ => { - eprintln!("usage: igneum-ota-sign keygen | sign | verify | embedded | fingerprint | sha256 | sign-jobs | verify-jobs | envelope-jobs | verify-signed-jobs | sign-inputs | verify-inputs [--zip z] [--dir d] [--node-commit c]"); + eprintln!("usage: igneum-ota-sign keygen | sign | verify | embedded | fingerprint | sha256 | sign-jobs | verify-jobs | envelope-jobs | verify-signed-jobs | sign-inputs | verify-inputs [--zip z] [--dir d] [--node-commit c] | verify-server [--binary f]"); std::process::exit(2); } } diff --git a/app/igneum-app/src/config.rs b/app/igneum-app/src/config.rs index fe287df57..d66f949f3 100644 --- a/app/igneum-app/src/config.rs +++ b/app/igneum-app/src/config.rs @@ -108,6 +108,10 @@ pub struct Settings { /// on when the machine can prove, never switching an explicit on back off). Older installs apply it at their next start. #[serde(default)] pub prove_default_applied: bool, + /// The prover profile on the patched GPU server (src/provedefault.rs `profile`): "auto" (the measured tier from + /// the card's VRAM), "2^25", "2^26", "2^27" or "stock". Empty = auto. + #[serde(default)] + pub prove_profile: String, } fn one() -> u32 { @@ -119,7 +123,7 @@ fn yes() -> bool { impl Default for Settings { fn default() -> Settings { - Settings { setup_done: false, address: String::new(), address_source: String::new(), key_saved: false, identities: 1, cards: HashMap::new(), display_name: String::new(), vote: true, paused: false, accepted_total: 0, auto_update: true, remote_jobs: true, prove: false, sweep: true, power_control: false, installed_at: 0, dev_fee: true, fee_total: 0, proof_verify_trust: false, prove_default_applied: false } + Settings { setup_done: false, address: String::new(), address_source: String::new(), key_saved: false, identities: 1, cards: HashMap::new(), display_name: String::new(), vote: true, paused: false, accepted_total: 0, auto_update: true, remote_jobs: true, prove: false, sweep: true, power_control: false, installed_at: 0, dev_fee: true, fee_total: 0, proof_verify_trust: false, prove_default_applied: false, prove_profile: String::new() } } } diff --git a/app/igneum-app/src/engine.rs b/app/igneum-app/src/engine.rs index 767c5cbe2..8818c1574 100644 --- a/app/igneum-app/src/engine.rs +++ b/app/igneum-app/src/engine.rs @@ -283,7 +283,11 @@ impl Shared { } let cards = self.state.lock().unwrap().mining.cards.clone(); let wsl = if cfg!(windows) { Some(crate::wslhost::distro_answers()) } else { None }; - let d = crate::provedefault::decide(&cards, std::env::consts::OS, wsl, crate::detect::total_ram_mb()); + // the project's GPU server next to the engine (src/proverserver.rs): verified here once, so the gate it moves + // (24 GB to 12 GB) applies to the install-time default + let bin_dir = std::env::current_exe().ok().and_then(|p| p.parent().map(|d| d.to_path_buf())).unwrap_or_default(); + let patched = crate::proverserver::shipped(&bin_dir).map(|s| s.is_some()).unwrap_or(false); + let d = crate::provedefault::decide_with_server(&cards, std::env::consts::OS, wsl, crate::detect::total_ram_mb(), patched); let on = d.on || already_on; { let mut s = self.settings.lock().unwrap(); diff --git a/app/igneum-app/src/main.rs b/app/igneum-app/src/main.rs index 13a40f1bc..6c7e8cfc7 100644 --- a/app/igneum-app/src/main.rs +++ b/app/igneum-app/src/main.rs @@ -30,6 +30,7 @@ mod jobrun; mod jobbuild; mod prover; mod provedefault; +mod proverserver; mod segments; mod verifier; mod wslhost; diff --git a/app/igneum-app/src/provedefault.rs b/app/igneum-app/src/provedefault.rs index 6aebd90db..c6290b168 100644 --- a/app/igneum-app/src/provedefault.rs +++ b/app/igneum-app/src/provedefault.rs @@ -14,6 +14,19 @@ //! //! Decided 5 October 2026 (delegated by the project lead: "deploy what is absolute best"), docs/plans/proving-v1.md. The default //! never switches an explicit on back off, and Settings always wins afterwards. +//! +//! With the project's own build of the GPU server shipped and verified (src/proverserver.rs, 6 October 2026, the +//! bench-log entry "prover floor": the patched server sizes its buffers to the shard instead of to a 24 GB card), +//! the tiers are measured on the cards themselves and the gate moves to 12 GB: +//! +//! | Card (the per-card VRAM read, `nvidia-smi` MiB) | Profile (`SP1_GPU_ELEMENT_THRESHOLD` to the host) | Why | +//! |---|---|---| +//! | 20 GB and more (24 GB, 32 GB) | the server's own sizes (no override) | 16,851 MiB on the v1 shard at upstream's threshold on the 5090, 3.7 GB under the stock server | +//! | 14 to 20 GB (16 GB) | 2^27 = 134,217,728 | 12,915 MiB alone, 10.95 GB own plus the miner's 1.7 GB beside it, 17.4 s a v1 shard (the 5090's allocation) | +//! | 10 to 14 GB (12 GB) | 2^26 = 67,108,864, mining and proving | the RTX 4070 itself: 9,034 MiB of 12,282 beside its own miner, 24.1 s a v1 shard; 7,553 MiB alone in 7.7 s | +//! | under 10 GB (8 GB) | off, with the reason | the compressed v1 shard alone is 7,553 MiB on the 4070; nothing is left for a miner or a bigger shard | +//! +//! `Settings` overrides the profile (`prove_profile`: auto, 2^25, 2^26, 2^27, stock). use crate::state::CardState; @@ -37,6 +50,56 @@ fn gb(mb: u64) -> u64 { (mb + 512) / 1024 } +/// The patched server's gate: a 12 GB card (`nvidia-smi` 12,282 for the RTX 4070, 12,288 for a 3060) proves; a +/// 10 GB 3080 (10,240) does not. +pub const MIN_VRAM_MB_PATCHED: u64 = 11_000; + +/// The per-card profile on the patched server: the environment the host gets, and one line for the tile. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Profile { + /// "12gb", "16gb", "24gb", "32gb" or "off". + pub tier: &'static str, + /// `SP1_GPU_ELEMENT_THRESHOLD` for the host (the server reads it); None = the server's own sizes. + pub threshold: Option, + /// Whether the card mines and proves at once on this profile (measured), or proves only. + pub mine_and_prove: bool, + pub line: String, +} + +pub const THRESHOLD_2_25: u64 = 1 << 25; +pub const THRESHOLD_2_26: u64 = 1 << 26; +pub const THRESHOLD_2_27: u64 = 1 << 27; + +/// The profile from the card's VRAM, as measured (the module's table). `override_name` is Settings' `prove_profile` +/// ("auto" or "" = the table; "2^25", "2^26", "2^27" or "stock" force a threshold, the tier line says so). +pub fn profile(vram_mb: u64, override_name: &str) -> Profile { + let forced = match override_name.trim() { + "2^25" | "2**25" | "33554432" => Some(Some(THRESHOLD_2_25)), + "2^26" | "2**26" | "67108864" => Some(Some(THRESHOLD_2_26)), + "2^27" | "2**27" | "134217728" => Some(Some(THRESHOLD_2_27)), + "stock" | "full" => Some(None), + _ => None, + }; + let auto = if vram_mb >= 20_000 { + Profile { tier: if vram_mb >= VRAM_MB_PROTOTYPE_SHARD { "32gb" } else { "24gb" }, threshold: None, mine_and_prove: true, line: format!("{} GB card: the server's own sizes (16.9 GB measured on the v1 shard at upstream's threshold)", gb(vram_mb)) } + } else if vram_mb >= 14_000 { + Profile { tier: "16gb", threshold: Some(THRESHOLD_2_27), mine_and_prove: true, line: format!("{} GB card: threshold 2^27 (12.9 GB alone, 10.95 GB plus the miner beside it, measured on the 5090's allocation)", gb(vram_mb)) } + } else if vram_mb >= MIN_VRAM_MB_PATCHED { + Profile { tier: "12gb", threshold: Some(THRESHOLD_2_26), mine_and_prove: true, line: format!("{} GB card: threshold 2^26, mines and proves (the RTX 4070 measured 9,034 MiB of 12,282 beside its own miner, 24.1 s a shard)", gb(vram_mb)) } + } else { + Profile { tier: "off", threshold: None, mine_and_prove: false, line: format!("{} GB card: proving off, the compressed shard alone needs 7.6 GB (measured on the RTX 4070) and leaves nothing for a miner on this card", gb(vram_mb)) } + }; + match forced { + Some(t) if auto.tier != "off" || t.is_some() => Profile { + tier: auto.tier, + threshold: t, + mine_and_prove: auto.mine_and_prove, + line: format!("{} GB card: Settings forces {} (the measured profile would be {})", gb(vram_mb), match t { Some(THRESHOLD_2_25) => "threshold 2^25", Some(THRESHOLD_2_26) => "threshold 2^26", Some(THRESHOLD_2_27) => "threshold 2^27", _ => "the server's own sizes" }, match auto.threshold { Some(THRESHOLD_2_26) => "2^26", Some(THRESHOLD_2_27) => "2^27", _ => "the server's own sizes" }), + }, + _ => auto, + } +} + /// Windows machines under this much RAM stay off until measured (consequences review C4, 5 October 2026): PC 2 at /// 63 GB had 25.6 GB in use with the WSL2 VM's working set at 7.9 GB while proving; a 16 GB PC would swap. pub const MIN_RAM_MB_WINDOWS: u64 = 31_000; @@ -50,9 +113,17 @@ pub fn aggregation_card(cards: &[CardState]) -> Option<&CardState> { /// `os` is `std::env::consts::OS` ("windows", "linux", "macos"); `wsl_answers` is read on Windows only; `ram_mb` is the /// machine's RAM when the platform reports it (None = unknown, no gate). pub fn decide(cards: &[CardState], os: &str, wsl_answers: Option, ram_mb: Option) -> Decision { + decide_with_server(cards, os, wsl_answers, ram_mb, false) +} + +/// `patched_server`: the project's verified GPU server is in the payload (src/proverserver.rs), so the gate is the +/// measured 12 GB one and the tile names the card's profile; without it, the stock server's 24 GB gate. +pub fn decide_with_server(cards: &[CardState], os: &str, wsl_answers: Option, ram_mb: Option, patched_server: bool) -> Decision { let nvidia: Vec<&CardState> = cards.iter().filter(|c| c.vendor == "nvidia").collect(); - // a mining card needs 20 GB (the measured mine-and-prove peak of 16.8 GB), a card that only proves 16 GB - let able: Vec<&CardState> = nvidia.iter().copied().filter(|c| c.vram_mb >= if c.enabled { MIN_VRAM_MB_MINING } else { MIN_VRAM_MB_PROVE_ONLY }).collect(); + // the stock server: a card needs 24 GB (its floor is 13.9 GB for an empty shard, 20.4 for a full one); the + // patched server: 12 GB (the RTX 4070's own measurement) + let gate = |c: &CardState| if patched_server { MIN_VRAM_MB_PATCHED } else if c.enabled { MIN_VRAM_MB_MINING } else { MIN_VRAM_MB_PROVE_ONLY }; + let able: Vec<&CardState> = nvidia.iter().copied().filter(|c| c.vram_mb >= gate(c)).collect(); let off = |line: String| Decision { on: false, line }; if os == "macos" { return off("proving stays off on Apple silicon: the M5 Max CPU took 41 to 55 s for an empty shard and minutes for a full one; Settings switches it on (CPU, slow)".into()); @@ -63,7 +134,9 @@ pub fn decide(cards: &[CardState], os: &str, wsl_answers: Option, ram_mb: } else { nvidia.iter().map(|c| format!("{} {} GB{}", c.name, gb(c.vram_mb), if c.enabled { ", mining" } else { "" })).collect::>().join(", ") }; - let why = if nvidia.iter().any(|c| c.vram_mb >= 15_872) { + let why = if patched_server && !nvidia.is_empty() { + "the smallest card that proves is 12 GB (the RTX 4070 measured 7.6 GB for a shard alone); this card is under that" + } else if nvidia.iter().any(|c| c.vram_mb >= 15_872) { "a full shard needs a 24 GB card (measured 20.4 GB on the adopted shard size, 13.9 GB for an empty one); this card is under that, so Settings would switch proving on at your own risk" } else if nvidia.is_empty() { "this machine mines and does not prove: no zkVM proves on an AMD GPU today, and the CPU prover costs about 5 minutes a shard at a 30 GB RSS (bench-log, the SP1 CPU prover on PC 1); proving needs an NVIDIA card with 24 GB or more" @@ -80,7 +153,9 @@ pub fn decide(cards: &[CardState], os: &str, wsl_answers: Option, ram_mb: } } } - let size_note = if best.vram_mb >= VRAM_MB_PROTOTYPE_SHARD { "" } else { "; until the devnet's fee switch its shards are the prototype size, which needs 32 GB, so this card proves from the switch on" }; + let size_note = if patched_server { + format!("; {}", profile(best.vram_mb, "").line) + } else if best.vram_mb >= VRAM_MB_PROTOTYPE_SHARD { String::new() } else { "; until the devnet's fee switch its shards are the prototype size, which needs 32 GB, so this card proves from the switch on".to_string() }; match os { "windows" => match wsl_answers { Some(true) => Decision { on: true, line: format!("proving on by default: {card} with WSL2 (Ubuntu-24.04 answers){size_note}; Settings switches it off") }, @@ -95,6 +170,47 @@ pub fn decide(cards: &[CardState], os: &str, wsl_answers: Option, ram_mb: mod tests { use super::*; + #[test] + fn the_profile_follows_the_measured_tiers_and_settings_can_force_one() { + let p = profile(12_282, ""); + assert_eq!((p.tier, p.threshold, p.mine_and_prove), ("12gb", Some(THRESHOLD_2_26), true)); + assert!(p.line.contains("RTX 4070") && p.line.contains("9,034"), "{}", p.line); + assert_eq!(profile(12_288, "auto").threshold, Some(THRESHOLD_2_26), "a 3060 12 GB"); + let p = profile(16_303, ""); + assert_eq!((p.tier, p.threshold), ("16gb", Some(THRESHOLD_2_27))); + let p = profile(24_564, ""); + assert_eq!((p.tier, p.threshold), ("24gb", None)); + assert_eq!(profile(32_607, "").tier, "32gb"); + let p = profile(10_240, ""); + assert_eq!((p.tier, p.threshold, p.mine_and_prove), ("off", None, false)); + assert!(p.line.contains("proving off") && p.line.contains("7.6 GB"), "{}", p.line); + assert_eq!(profile(8_192, "").tier, "off"); + // Settings: a forced threshold keeps the tier and says so; a forced stock profile on a 12 GB card is allowed too + let p = profile(12_282, "2^25"); + assert_eq!((p.tier, p.threshold), ("12gb", Some(THRESHOLD_2_25))); + assert!(p.line.contains("Settings forces threshold 2^25") && p.line.contains("would be 2^26"), "{}", p.line); + assert_eq!(profile(24_564, "2^27").threshold, Some(THRESHOLD_2_27)); + assert_eq!(profile(16_303, "stock").threshold, None); + // an 8 GB card forced to a threshold proves at the owner's risk; forced to stock it stays off + assert_eq!(profile(8_192, "2^25").threshold, Some(THRESHOLD_2_25)); + assert_eq!(profile(8_192, "stock").tier, "off"); + assert_eq!(profile(12_282, "nonsense"), profile(12_282, "")); + } + + #[test] + fn with_the_patched_server_a_12_gb_card_is_on_and_a_10_gb_one_off() { + let d = decide_with_server(&[card("nvidia", "NVIDIA GeForce RTX 4070", 12_282)], "windows", Some(true), Some(95_902), true); + assert!(d.on, "{}", d.line); + assert!(d.line.contains("threshold 2^26") && d.line.contains("mines and proves"), "{}", d.line); + let d = decide_with_server(&[card("nvidia", "NVIDIA GeForce RTX 3080", 10_240)], "linux", None, None, true); + assert!(!d.on && d.line.contains("smallest card that proves is 12 GB"), "{}", d.line); + // the same cards without the patched server: the old 24 GB gate + assert!(!decide_with_server(&[card("nvidia", "NVIDIA GeForce RTX 4070", 12_282)], "linux", None, None, false).on); + assert!(decide_with_server(&[card("nvidia", "NVIDIA GeForce RTX 5080", 16_303)], "linux", None, None, true).on); + let d = decide_with_server(&[card("nvidia", "NVIDIA GeForce RTX 4090", 24_564)], "linux", None, None, true); + assert!(d.on && d.line.contains("the server's own sizes"), "{}", d.line); + } + fn card(vendor: &str, name: &str, vram_mb: u64) -> CardState { CardState { vendor: vendor.into(), name: name.into(), vram_mb, enabled: true, ..Default::default() } } diff --git a/app/igneum-app/src/prover.rs b/app/igneum-app/src/prover.rs index c1053eee2..51c71fa71 100644 --- a/app/igneum-app/src/prover.rs +++ b/app/igneum-app/src/prover.rs @@ -130,6 +130,20 @@ struct Tools { wsl: bool, setup_script: Option, cuda: bool, + /// The project's patched GPU server shipped in the payload and verified (src/proverserver.rs): its manifest and + /// its path as Ubuntu sees it. None: the stock server (the SDK's download) and the 24 GB gate. + server: Option<(crate::proverserver::ServerManifest, String)>, + /// Why the shipped server is not in use although it was shipped (verification failed, or the fallback fired). + server_note: String, +} + +/// The shipped server for `Tools`: verified, or the reason it is not used (the stock server then). +fn shipped_server(bin_dir: &Path) -> (Option<(crate::proverserver::ServerManifest, String)>, String) { + match crate::proverserver::shipped(bin_dir) { + Ok(Some((m, bin))) => (Some((m, crate::wslhost::wsl_path(&bin))), String::new()), + Ok(None) => (None, String::new()), + Err(e) => (None, format!("the shipped prover server is not used: {e}; the stock server (24 GB cards) runs instead")), + } } fn evm_rpc(shared: &Shared, method: &str, params: Value, timeout: Duration) -> Result { @@ -171,7 +185,8 @@ fn find_tools(bin_dir: &Path) -> Result { // a Linux path: never PathBuf::join here, which writes a backslash on Windows ("/opt/igneum\\igneum-prove-export" // broke every export on PC 2 under 0.3.7, 5 October 2026) let export = PathBuf::from(format!("{}/igneum-prove-export", host.to_string_lossy().rsplit_once('/').map(|(d, _)| d).unwrap_or(""))); - Ok(Tools { host, export, miner, wsl: true, setup_script: setup.exists().then_some(setup), cuda: text.contains("cuda") }) + let (server, server_note) = shipped_server(bin_dir); + Ok(Tools { host, export, miner, wsl: true, setup_script: setup.exists().then_some(setup), cuda: text.contains("cuda"), server, server_note }) } None => Err(probe_message(bin_dir, answered, setup.exists())), } @@ -181,7 +196,7 @@ fn find_tools(bin_dir: &Path) -> Result { if !host.exists() || !export.exists() { return Err(format!("igneum-prove-host and igneum-prove-export are not next to the engine ({})", bin_dir.display())); } - Ok(Tools { host, export, miner, wsl: false, setup_script: None, cuda: false }) + Ok(Tools { host, export, miner, wsl: false, setup_script: None, cuda: false, server: None, server_note: String::new() }) } } @@ -282,6 +297,78 @@ fn run_tool(shared: &Shared, t: &Tools, exe: &Path, args: &[String], env: &[(&st } } +/// The threshold the host gets (`SP1_GPU_ELEMENT_THRESHOLD`, as a string) on the patched server: the biggest +/// NVIDIA card's profile, with Settings' `prove_profile` override. None on the stock server or at the server's own sizes. +fn profile_threshold(shared: &Shared, t: &Tools) -> Option { + if t.server.is_none() { + return None; + } + let vram = shared.state.lock().unwrap().mining.cards.iter().filter(|c| c.vendor == "nvidia").map(|c| c.vram_mb).max().unwrap_or(0); + let override_name = shared.settings.lock().unwrap().prove_profile.clone(); + let p = crate::provedefault::profile(vram, &override_name); + set(shared, |st| st.server_profile = format!("{}: {}", p.tier, p.line)); + p.threshold.map(|v| v.to_string()) +} + +/// Windows: puts the shipped, verified server where the SDK looks (`~/.sp1/bin/sp1-gpu-server` in Ubuntu-24.04) when +/// the one there differs, or restores the stock one after the fallback; records the server in the state either way. +fn install_server(shared: &Shared, t: &mut Tools, fallback_to_stock: &mut bool) { + if !t.wsl { + return; + } + let Some((m, bin_wsl)) = t.server.clone() else { + let note = t.server_note.clone(); + set(shared, |p| { + p.server_version = String::new(); + p.server_sha256 = String::new(); + p.server_kind = "stock".into(); + p.server_note = note; + }); + return; + }; + let body = if *fallback_to_stock { crate::proverserver::restore_stock_script() } else { crate::proverserver::install_script(&bin_wsl, &m.sha256) }; + let line = match crate::wslhost::write_script("prove-server", &body) { + Ok(file) => crate::platform::quiet(&mut crate::wslhost::command(&crate::platform::tool("wsl"), crate::wslhost::DISTRO, None, &file.path, true, &[])).output().map(|o| String::from_utf8_lossy(&o.stdout).lines().find(|l| l.starts_with("RESULT server")).unwrap_or("").to_string()).unwrap_or_default(), + Err(e) => format!("RESULT server install FAILED: cannot write the script: {e}"), + }; + shared.log(&format!("prover: {}", if line.is_empty() { "the server install script printed nothing" } else { line.as_str() })); + if *fallback_to_stock { + t.server_note = "the patched GPU server did not come up on this machine; the stock server (24 GB cards) runs instead".into(); + t.server = None; + *fallback_to_stock = false; + let note = t.server_note.clone(); + set(shared, |p| { + p.server_version = String::new(); + p.server_sha256 = String::new(); + p.server_kind = "stock".into(); + p.server_note = note; + }); + return; + } + let installed = line.contains(" installed") || line.contains(" kept"); + if !installed { + t.server_note = format!("the shipped GPU server could not be installed into Ubuntu-24.04 ({}); the stock server runs instead", if line.is_empty() { "no answer" } else { line.as_str() }); + t.server = None; + } + let note = t.server_note.clone(); + set(shared, |p| { + if installed { + p.server_version = m.sp1_version.clone(); + p.server_sha256 = m.sha256.clone(); + p.server_kind = "patched".into(); + p.server_note = String::new(); + } else { + p.server_version = String::new(); + p.server_sha256 = String::new(); + p.server_kind = "stock".into(); + p.server_note = note; + } + }); +} + +/// The prefix of the closure's error when the GPU server itself did not come up (the fallback, not a proving error). +const SERVER_DOWN_MARK: &str = "server-down:"; + fn payout_address(shared: &Shared) -> String { shared.settings.lock().unwrap().address.clone() } @@ -356,6 +443,9 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { let mut held_segments: Vec = Vec::new(); let mut last_verifier_read = Instant::now() - Duration::from_secs(600); let mut asked_restart = false; + // the fallback to the stock server (src/proverserver.rs `fallback`): set after a proof failed because the + // patched server did not come up; the next probe restores the stock server and keeps the note + let mut fallback_to_stock = false; // macOS and Linux: the host sits next to the engine, so its pinned ids are read at once, proving on or off // (Windows runs the host inside WSL2, which is probed only once proving is on) if !cfg!(windows) { @@ -388,7 +478,7 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { if tools.is_none() && last_probe.elapsed() >= Duration::from_secs(60) { last_probe = Instant::now(); match find_tools(&bin_dir) { - Ok(t) => { + Ok(mut t) => { shared.log(&format!("prover: host {}{}{}", t.host.display(), if t.wsl { " (WSL2)" } else { "" }, if t.cuda { ", CUDA" } else { ", CPU (slow)" })); set(&shared, |p| { p.available = true; @@ -403,6 +493,7 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { shared.send(crate::engine::Cmd::RestartNode("the WSL2 prover is installed now; the node restarts to verify proof records".into())); } read_ids(&shared, &t); + install_server(&shared, &mut t, &mut fallback_to_stock); tools = Some(t); } Err(e) => { @@ -643,7 +734,21 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { } set(&shared, |p| p.message = if t.cuda { "proving on the GPU".into() } else { "CPU prover: about five minutes a shard, 30 GB of RAM, paid only when no card proves first".into() }); let prover_env = if t.cuda { "cuda" } else { "cpu" }; - let (ok, out) = run_tool(&shared, t, &t.host, &[fix_p, "--mode".into(), "compressed".into(), "--shard".into(), w.shard.to_string(), "--prover".into(), payout.clone(), "--out".into(), res_p], &[("SP1_PROVER", prover_env), ("RUST_LOG", "off")], Duration::from_secs(3 * 3600), &dir.join(format!("prove-{}-{}.log", w.number, w.shard))); + // the per-card profile on the patched server (src/provedefault.rs `profile`): the threshold the server + // sizes its buffers by, from the biggest NVIDIA card's VRAM and Settings' override; none on the stock server + let threshold = profile_threshold(&shared, t); + let mut env: Vec<(&str, &str)> = vec![("SP1_PROVER", prover_env), ("RUST_LOG", "off")]; + if let Some(th) = threshold.as_deref() { + env.push(("SP1_GPU_ELEMENT_THRESHOLD", th)); + } + let (ok, out) = run_tool(&shared, t, &t.host, &[fix_p, "--mode".into(), "compressed".into(), "--shard".into(), w.shard.to_string(), "--prover".into(), payout.clone(), "--out".into(), res_p], &env, Duration::from_secs(3 * 3600), &dir.join(format!("prove-{}-{}.log", w.number, w.shard))); + if !ok && t.server.is_some() { + if let Some(next) = crate::proverserver::fallback(crate::proverserver::Kind::Patched, &out) { + // the closure cannot touch the loop's state: the marker in the error is read after it + let why = out.lines().find(|l| l.contains("sp1-gpu-server")).unwrap_or("").trim().to_string(); + return Err(format!("{SERVER_DOWN_MARK}{}: {why}", next.word())); + } + } if !ok || !results.exists() { let last = out.lines().rev().find(|l| l.contains("RESULT") || l.contains("rror")).unwrap_or("failed").to_string(); // the root-socket class (5 October 2026, PC 2 at 20:00Z and 21:25Z): a job that ran the host as root @@ -673,6 +778,22 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { } Ok(()) })(); + // the fallback (src/proverserver.rs): the patched server did not come up, so the stock server takes over at + // the next probe (install_server restores it) and the tile says why + if let Err(e) = &outcome { + if let Some(rest) = e.strip_prefix(SERVER_DOWN_MARK) { + fallback_to_stock = rest.starts_with("stock"); + shared.log(&format!("prover: the patched GPU server did not come up ({}); the stock server takes over at the next probe", rest.split_once(": ").map(|(_, w)| w).unwrap_or(""))); + tools = None; + last_probe = Instant::now() - Duration::from_secs(600); + set(&shared, |p| { + p.status = "setup".into(); + p.message = "the patched GPU server did not start; switching to SP1's stock server (24 GB cards)".into(); + p.current = String::new(); + }); + continue; + } + } match outcome { Ok(()) => { shared.event("proving", &format!("block {} shard {} proven and submitted in {:.0} s", w.number, w.shard, started.elapsed().as_secs_f64())); diff --git a/app/igneum-app/src/proverserver.rs b/app/igneum-app/src/proverserver.rs new file mode 100644 index 000000000..35519ca08 --- /dev/null +++ b/app/igneum-app/src/proverserver.rs @@ -0,0 +1,285 @@ +//! The project's own build of SP1's GPU prover server (prover floor, 6 October 2026: the stock `sp1-gpu-server` +//! 6.8.1 refuses every card under 24 GB before it allocates and sizes every buffer for a 24 GB card; the patched +//! one, `proving/prover-floor/sp1-gpu-6.8.1-floor.patch`, sizes them to the shard, so a 12 GB card mines and +//! proves: the RTX 4070 in PC 1 proved the v1 shard at 9,034 MiB beside its miner, bench-log "prover floor"). +//! +//! The payload ships it under `wsl2\bin\sp1-gpu-server` next to `wsl2\prover-server.json` and `.sig`: a manifest +//! in the format of `payload-inputs.json` (the SP1 version it patches, the patch's sha256, the CUDA targets, the +//! binary's sha256 and size) signed on the Mac with the OTA key the app already trusts (`manifest::OTA_PUBLIC_KEY_HEX`). +//! Before the server is used the app checks the signature, then the binary's sha256 and size against the manifest +//! (`shipped`), and installs it where the SP1 SDK looks (`$HOME/.sp1/bin/sp1-gpu-server` inside Ubuntu-24.04, +//! `sp1-cuda-6.8.1/src/server.rs`), replacing whatever is there when the sha256 differs (`install_script`). The +//! SDK itself only checks `--version`, which both the stock and the patched server answer with 6.8.1, so the sha256 +//! is the only thing that says which server a machine runs; `/api/state` carries it (`ProvingState::server_*`). +//! +//! Fallback: when a proof fails because the server could not start or be reached while the patched server was the +//! one installed, the next run goes back to the stock server (`fallback`), the SDK downloads it, and the tier rule +//! falls back to the 24 GB gate; the tile says so. Nothing on chain or in the proof format changes with the server. +use crate::manifest::{sha256_file, verify_signature, OTA_PUBLIC_KEY_HEX}; +use std::path::Path; + +/// The manifest's `format` field. +pub const FORMAT: &str = "igneum-prover-server/1"; +/// The stock `sp1-gpu-server` 6.8.1 the SDK downloads (`sp1_gpu_server_v6.8.1_x86_64.tar.gz`), measured on PC 2 +/// on 5 October 2026 (251,306,680 bytes). +pub const STOCK_SHA256_6_8_1: &str = "c2642ad1c42e85d8525159cf0c7cd5200d8766c9be1283f452a1f9bf9fea725c"; +/// The file names next to the engine (Windows: under `wsl2\`). +pub const BINARY: &str = "sp1-gpu-server"; +pub const MANIFEST: &str = "prover-server.json"; +pub const SIGNATURE: &str = "prover-server.json.sig"; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ServerManifest { + pub sp1_version: String, + pub sp1_commit: String, + pub patch_sha256: String, + pub cuda_archs: String, + pub built_at: String, + pub built_by: String, + pub sha256: String, + pub bytes: u64, +} + +/// Which server a sha256 names. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Kind { + Patched, + Stock, + Unknown, +} + +impl Kind { + pub fn word(self) -> &'static str { + match self { + Kind::Patched => "patched", + Kind::Stock => "stock", + Kind::Unknown => "unknown", + } + } +} + +pub fn kind(sha256: &str, shipped: Option<&ServerManifest>) -> Kind { + if shipped.map(|m| m.sha256 == sha256).unwrap_or(false) { + Kind::Patched + } else if sha256 == STOCK_SHA256_6_8_1 { + Kind::Stock + } else { + Kind::Unknown + } +} + +fn is_hex(s: &str, n: usize) -> bool { + s.len() == n && s.bytes().all(|b| b.is_ascii_hexdigit() && !b.is_ascii_uppercase()) +} + +/// Parses the manifest text (after its signature was checked). +pub fn parse(text: &str) -> Result { + let v: serde_json::Value = serde_json::from_str(text).map_err(|e| format!("prover-server.json is not JSON: {e}"))?; + let s = |k: &str| v.get(k).and_then(|x| x.as_str()).unwrap_or("").to_string(); + if s("format") != FORMAT { + return Err(format!("prover-server.json: format {:?} is not {FORMAT}", s("format"))); + } + let f = v.get("files").and_then(|f| f.get(BINARY)).ok_or_else(|| format!("prover-server.json names no {BINARY}"))?; + let sha256 = f.get("sha256").and_then(|x| x.as_str()).unwrap_or("").to_string(); + let bytes = f.get("bytes").and_then(|x| x.as_u64()).unwrap_or(0); + if !is_hex(&sha256, 64) { + return Err("prover-server.json: the server's sha256 is not 64 lowercase hex characters".into()); + } + if bytes == 0 { + return Err("prover-server.json: the server's size is missing".into()); + } + let m = ServerManifest { + sp1_version: s("sp1_version"), + sp1_commit: s("sp1_commit"), + patch_sha256: s("patch_sha256"), + cuda_archs: s("cuda_archs"), + built_at: s("built_at"), + built_by: s("built_by"), + sha256, + bytes, + }; + if m.sp1_version.is_empty() { + return Err("prover-server.json: sp1_version is missing".into()); + } + Ok(m) +} + +/// The signature with the OTA key, then the parse. +pub fn verify_and_parse(manifest_bytes: &[u8], sig_hex: &str) -> Result { + verify_signature(manifest_bytes, sig_hex.trim(), OTA_PUBLIC_KEY_HEX)?; + parse(&String::from_utf8_lossy(manifest_bytes)) +} + +/// The binary on disk must be the one the manifest names: the same sha256, the same size. A modified or truncated +/// binary is refused here, before anything runs it. +pub fn check_binary(m: &ServerManifest, path: &Path) -> Result<(), String> { + let size = std::fs::metadata(path).map_err(|e| format!("{}: {e}", path.display()))?.len(); + if size != m.bytes { + return Err(format!("{}: {size} bytes is not the manifest's {}", path.display(), m.bytes)); + } + let sum = sha256_file(path).map_err(|e| format!("{}: {e}", path.display()))?; + if sum != m.sha256 { + return Err(format!("{}: sha256 {sum} is not the manifest's {}", path.display(), m.sha256)); + } + Ok(()) +} + +/// The shipped server next to the engine: `/wsl2/{sp1-gpu-server, prover-server.json, prover-server.json.sig}` +/// on Windows (`/{...}` elsewhere), its manifest verified and the binary checked. `Err` names what is +/// wrong; a payload without the three files is `Ok(None)` (the stock server then). +pub fn shipped(bin_dir: &Path) -> Result, String> { + let dir = if cfg!(windows) { bin_dir.join("wsl2") } else { bin_dir.to_path_buf() }; + let bin = dir.join("bin").join(BINARY); + let man = dir.join(MANIFEST); + let sig = dir.join(SIGNATURE); + if !bin.exists() && !man.exists() { + return Ok(None); + } + if !bin.exists() || !man.exists() || !sig.exists() { + return Err(format!("the shipped prover server is incomplete: {} {} {}", present(&bin), present(&man), present(&sig))); + } + let bytes = std::fs::read(&man).map_err(|e| format!("{}: {e}", man.display()))?; + let sig_text = std::fs::read_to_string(&sig).map_err(|e| format!("{}: {e}", sig.display()))?; + let m = verify_and_parse(&bytes, &sig_text)?; + check_binary(&m, &bin)?; + Ok(Some((m, bin))) +} + +fn present(p: &Path) -> String { + format!("{}={}", p.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default(), if p.exists() { "present" } else { "MISSING" }) +} + +/// The bash the app runs inside Ubuntu-24.04 (as the app's user) before a proof: installs the shipped server at +/// `$HOME/.sp1/bin/sp1-gpu-server` when the one there has another sha256 (or is absent), and prints one line: +/// `RESULT server `. `bin_wsl` is the shipped binary's path as seen from Ubuntu +/// (`/mnt//.../wsl2/bin/sp1-gpu-server`). The copy goes into the Linux file system because the SDK execs +/// the path it finds and a server on the Windows drive starts slowly and may not carry the execute bit. +pub fn install_script(bin_wsl: &str, sha256: &str) -> String { + // the sha256 is validated lowercase hex (`parse`), so it goes into the script bare; the path is quoted + let want: String = sha256.chars().filter(|c| c.is_ascii_hexdigit()).collect(); + format!( + "set -u\nD=\"$HOME/.sp1/bin\"; T=\"$D/sp1-gpu-server\"\nmkdir -p \"$D\"\nhave=\"$(sha256sum \"$T\" 2>/dev/null | cut -c1-64)\"\nif [ \"$have\" = {want} ]; then echo \"RESULT server {want} kept\"; exit 0; fi\npkill -f sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock\ncp {src} \"$T.new\" && chmod +x \"$T.new\" && mv -f \"$T.new\" \"$T\" || {{ echo \"RESULT server {want} install FAILED\"; exit 1; }}\ngot=\"$(sha256sum \"$T\" | cut -c1-64)\"\nif [ \"$got\" = {want} ]; then echo \"RESULT server {want} installed\"; else echo \"RESULT server $got MISMATCH after the copy\"; exit 1; fi\n", + src = sq(bin_wsl) + ) +} + +/// A single-quoted bash word (the rule of src/wslhost.rs, repeated here so the signer binary can include this +/// module without the WSL code). +fn sq(s: &str) -> String { + format!("'{}'", s.replace('\'', "'\\''")) +} + +/// The bash that puts the stock server back (the fallback): removes the patched binary so the SDK downloads the +/// stock release again, and prints `RESULT server stock restored`. +pub fn restore_stock_script() -> String { + "set -u\nT=\"$HOME/.sp1/bin/sp1-gpu-server\"\npkill -f sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock\nrm -f \"$T\"\necho \"RESULT server stock restored\"\n".to_string() +} + +/// The lines in a failed proof's log that mean the server itself did not come up or answer, as the SDK's client +/// words them (`sp1-cuda-6.8.1/src/error.rs`, `client.rs`): not a proving error, not a verify failure. +const SERVER_DOWN: [&str; 5] = [ + "Could not start `sp1-gpu-server`", + "Could not connect to `sp1-gpu-server` socket", + "Could not check `sp1-gpu-server` version", + "Failed to bind to socket addr", + "Error running server:", +]; + +/// After a failed proof: the server kind to use next. The patched server gives way to the stock one when the log +/// says the server did not start or answer; a proving error (out of memory, a bad shard, a verify failure) keeps +/// the server as it is, and the stock server never changes. +pub fn fallback(current: Kind, failed_log: &str) -> Option { + if current == Kind::Patched && SERVER_DOWN.iter().any(|m| failed_log.contains(m)) { + Some(Kind::Stock) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const SHA_A: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + fn manifest_json(sha: &str, bytes: u64) -> String { + format!( + r#"{{"format":"{FORMAT}","sp1_version":"6.8.1","sp1_commit":"c84ada1ed5911f28c4d3c9d0ed2f9e6cd7edb824","patch_sha256":"e81cb0d03b291f9fd4bf0a109d6da2d7c897795c9ffd7f797c0ddce723eee2b1","cuda_archs":"80,86,89,120","built_at":"2026-10-06T12:00:00Z","built_by":"ci:1","files":{{"{BINARY}":{{"sha256":"{sha}","bytes":{bytes}}}}}}}"# + ) + } + + #[test] + fn the_manifest_parses_and_refuses_the_wrong_format_or_a_bad_hash() { + let m = parse(&manifest_json(SHA_A, 10)).unwrap(); + assert_eq!(m.sp1_version, "6.8.1"); + assert_eq!(m.bytes, 10); + assert_eq!(m.cuda_archs, "80,86,89,120"); + assert!(parse(&manifest_json("ABCD", 10)).unwrap_err().contains("64 lowercase hex")); + assert!(parse(&manifest_json(SHA_A, 0)).unwrap_err().contains("size is missing")); + assert!(parse(&manifest_json(SHA_A, 10).replace(FORMAT, "igneum-prover-server/2")).unwrap_err().contains("format")); + assert!(parse("{}").unwrap_err().contains("format")); + } + + #[test] + fn a_modified_binary_is_refused_and_the_right_one_accepted() { + let dir = std::env::temp_dir().join(format!("igneum-proverserver-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let bin = dir.join(BINARY); + std::fs::write(&bin, b"the server bytes").unwrap(); + let sha = sha256_file(&bin).unwrap(); + let m = parse(&manifest_json(&sha, 16)).unwrap(); + check_binary(&m, &bin).unwrap(); + // one byte changed, same size: refused by the sha256 + std::fs::write(&bin, b"the server byteS").unwrap(); + let e = check_binary(&m, &bin).unwrap_err(); + assert!(e.contains("sha256") && e.contains("is not the manifest's"), "{e}"); + // truncated: refused by the size before the hash is even read + std::fs::write(&bin, b"the server").unwrap(); + let e = check_binary(&m, &bin).unwrap_err(); + assert!(e.contains("10 bytes is not the manifest's 16"), "{e}"); + // missing: refused + std::fs::remove_file(&bin).unwrap(); + assert!(check_binary(&m, &bin).is_err()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn an_unsigned_or_tampered_manifest_does_not_verify() { + let text = manifest_json(SHA_A, 10); + // a 64-byte zero signature is not the OTA key's signature of this text + let zero_sig = "0".repeat(128); + assert!(verify_and_parse(text.as_bytes(), &zero_sig).is_err()); + assert!(verify_and_parse(text.as_bytes(), "not hex").is_err()); + } + + #[test] + fn the_kind_follows_the_sha256() { + let m = parse(&manifest_json(SHA_A, 10)).unwrap(); + assert_eq!(kind(SHA_A, Some(&m)), Kind::Patched); + assert_eq!(kind(STOCK_SHA256_6_8_1, Some(&m)), Kind::Stock); + assert_eq!(kind(STOCK_SHA256_6_8_1, None), Kind::Stock); + assert_eq!(kind("bbbb", Some(&m)), Kind::Unknown); + assert_eq!(Kind::Patched.word(), "patched"); + } + + #[test] + fn the_install_script_quotes_its_paths_and_compares_the_hash_first() { + let s = install_script("/mnt/c/Program Files/Igneum Miner/wsl2/bin/sp1-gpu-server", SHA_A); + assert!(s.contains("'/mnt/c/Program Files/Igneum Miner/wsl2/bin/sp1-gpu-server'"), "{s}"); + assert!(s.contains(&format!("if [ \"$have\" = {SHA_A} ]; then echo \"RESULT server {SHA_A} kept\"")), "{s}"); + assert!(s.contains("pkill -f sp1-gpu-server") && s.contains("rm -f /tmp/sp1-cuda-*.sock"), "the running server is stopped and its socket unlinked before the swap"); + assert!(s.contains("chmod +x")); + assert!(restore_stock_script().contains("rm -f \"$T\"")); + } + + #[test] + fn the_fallback_fires_only_for_a_server_that_did_not_come_up_and_only_from_patched() { + let down = "Error: CudaClientError: Connect(Could not start `sp1-gpu-server`: No such file)"; + assert_eq!(fallback(Kind::Patched, down), Some(Kind::Stock)); + assert_eq!(fallback(Kind::Patched, "Could not connect to `sp1-gpu-server` socket: Connection refused"), Some(Kind::Stock)); + assert_eq!(fallback(Kind::Stock, down), None, "the stock server has nothing to fall back to"); + assert_eq!(fallback(Kind::Unknown, down), None); + assert_eq!(fallback(Kind::Patched, "ProverError: CUDA_OUT_OF_MEMORY while proving shard 3"), None, "a proving error keeps the server"); + assert_eq!(fallback(Kind::Patched, "RESULT compressed shard 0: ... NOT VERIFIED"), None); + } +} diff --git a/app/igneum-app/src/state.rs b/app/igneum-app/src/state.rs index 7ec9a944c..596357877 100644 --- a/app/igneum-app/src/state.rs +++ b/app/igneum-app/src/state.rs @@ -209,6 +209,15 @@ pub struct ProvingState { pub verifier_reason: String, /// the sentence on the tile for the state above pub verifier_note: String, + /// The GPU prover server this machine runs (src/proverserver.rs): its SP1 version, its sha256, and whether it is + /// the project's patched build ("patched"), SP1's stock release ("stock") or something else ("unknown"); + /// `server_profile` is the per-card profile in force (the tier and the threshold the host gets); `server_note` + /// says why the stock server is in use when the patched one was shipped (the fallback). + pub server_version: String, + pub server_sha256: String, + pub server_kind: String, + pub server_profile: String, + pub server_note: String, /// the node's proof pool: records held, verified, rejected pub pool_entries: u64, pub pool_verified: u64, diff --git a/docs/analysis/prover-floor.md b/docs/analysis/prover-floor.md index e87b1c5c2..aa9c5212a 100644 --- a/docs/analysis/prover-floor.md +++ b/docs/analysis/prover-floor.md @@ -225,3 +225,15 @@ unpatched verifier. So a 12 GB card mines and proves compressed shards at 2^26 w core-only hand-off is the reserve, not the requirement. The 5090's allocation pattern overstated the card by about 0.65 GB. The full tables and the per-tier consequences are in the bench-log entry; the public line moves to "12 GB mines and proves" when the packaging row ships the server. + +## The packaging row (6 October 2026, afternoon) + +Branch prover-floor carries the whole path: `packaging/prover/build-server.sh` (the one recipe), the CI workflow +`prover-server.yml`, `push-server.sh` (the Mac signs the CI build's manifest with the OTA key and publishes), +`fetch-server.sh` (verifies and places it for the payload), the `make-payload.sh` step (`wsl2\bin\sp1-gpu-server`, +`wsl2\prover-server.json`, `.sig`), the Windows build's fetch step, and the app: `proverserver.rs` (the manifest, +the hash check, the install into `~/.sp1/bin`, the fallback), `provedefault::profile` (the tiers from the VRAM, +Settings' override), `/api/state`'s `server_*` fields. The tier table and the shipper's steps are in +`docs/plans/proving-v1.md`, "The packaged GPU server". Tests: 147 in the app, of which the new ones refuse a +modified or truncated binary and an unsigned manifest, choose the tier per VRAM, and fire the fallback only on a +server that did not come up. diff --git a/docs/plans/proving-v1.md b/docs/plans/proving-v1.md index ed84006b6..8ccb262c0 100644 --- a/docs/plans/proving-v1.md +++ b/docs/plans/proving-v1.md @@ -116,9 +116,44 @@ The resume path (5 October 2026, the 0.3.11 app): `POST /api/resume` on 0.3.9 re The prover-floor agent's first sweep (job `floor-sweep-1`, 22:34 to 22:38Z, PC 2's 5090, the miners stopped, this plan's per-point recipe, its patched `sp1-gpu-server` 5568108b built for sm_86, sm_89 and sm_120, every proof VERIFIED by the unpatched pv1 host): the control at upstream's sizes reproduces the curve above (empty shard 13,892 MiB and 2.2 s; the v1 shard 20,516 MiB and 4.2 s); with the core element threshold at 2^26 the v1 shard proves as four core shards in 5.3 s at **12,708 MiB** and the empty shard at 12,772 MiB; 2^25 gives 12,836 MiB at 8.5 s; 2^27 gives 15,396 MiB. The 12.7 GB left is the server's Setup (five recursion keys pre-built at a fixed 2^27 capacity plus the shrink and core keys: 9.7 GB before the first shard), which its patch v2 sizes to the need. Decided for the 12 GB profile: the split that lands under 11 GB wins (5.3 s a shard is inside the loop's own 25 to 30 s of carriage and 100x inside T); 2^27 is the second profile only if v2 leaves it under 11 GB with the miner's 1.8 GB beside it. The 12 GB row stays OPEN until the final pair (alone and beside the miner) lands and the on-order 3060 runs it. -### A self-built CUDA server (the 12 GB path), before 0.3.12 (consequences C26) +### The packaged GPU server (the 12 GB path): the packaging row for 0.3.13 (6 October 2026, branch prover-floor) -If the prover-floor agent's rebuilt `sp1-gpu-server` (the Setup sizes cut, built on PC 2 under WSL2) proves a shard under 11 GB, it becomes a shipped artefact and needs its own row of rules before 0.3.12: it is built from a pinned SP1 source tag with `CUDA_ARCHS` covering sm_86, sm_89 and sm_120 (the 12 and 16 GB tiers are Ampere and Ada, not only the 5090's Blackwell; one card family per measured row), by the packaging path that builds the Windows payload (PC 1's build job for the Linux binary, the Mac signs the manifest as it does the DMG), lands in the DMG and the WSL2 package beside the host as `wsl2/bin/sp1-gpu-server` with its sha256 in `payload-inputs.json`, is named in `evidence.md` beside the prover rows ("prover built from SP1 at "), is rebuilt and re-measured at every SP1 upgrade, and ships only after `--mode verify-segment` and `--mode verify` on proofs it made show the pinned verifying keys unchanged (the server changes allocation, not the circuit; the ids `0x2b1a81cb...` and `0x474678f3...` must still verify them). The 12 GB claim itself waits for the on-order RTX 3060 to run that server on the same fixtures and recipe as the curve; until then the public line stays at 24 GB. +The measurement that decides it is in the bench-log entry "prover floor": SP1 6.8.1's stock `sp1-gpu-server` refuses +every card under 24 GB before it allocates (`sp1-gpu/crates/prover_components/src/builder.rs` 35 to 39) and sizes +every buffer for a 24 GB card; the project's patch (`proving/prover-floor/sp1-gpu-6.8.1-floor.patch`, 4 files, 206 +lines on tag v6.8.1) sizes them to the shard, and the RTX 4070 12 GB in PC 1 proved the v1 shard at 9,034 MiB of +12,282 beside its own miner in 24.1 s (7,553 MiB alone in 7.7 s), every proof verified by the unpatched verifier. +Nothing on chain, in the proof format, the pinned guest ids or the verifying key changes with the server. + +| Piece | Where | What it does | +|---|---|---| +| The build | `packaging/prover/build-server.sh` (one recipe), run by CI `.github/workflows/prover-server.yml` (ubuntu-24.04, CUDA 12.8.1 from Jimver/cuda-toolkit, Go 1.27.1, `CUDA_ARCHS=80,86,89,120`) and by the PCs' WSL2 path `tools/prover-floor/pc2-build-server.ps1` | clones SP1 at v6.8.1 (c84ada1e), applies the patch, builds `sp1-gpu-server`, writes its sha256 and `build.json`. Not byte-reproducible across machines (SP1's `prover-types` build script stamps the build time in), so the signed manifest names ONE build, CI's; a PC build is a behavioural cross-check on the same fixtures | +| The signing | `packaging/prover/push-server.sh --run ` on the Mac | downloads the artifact, checks its recorded sha256, writes `prover-server.json` (format `igneum-prover-server/1`, `app/igneum-app/src/proverserver.rs`: the SP1 version and commit it patches, the patch's sha256, the CUDA targets, the binary's sha256 and size, the run id), signs it with the OTA key the apps already trust (`igneum-ota-sign sign`), publishes the binary, the manifest and `prover-server.json.sig` to the downloads host | +| The payload | `packaging/prover/fetch-server.sh` (CI's windows.yml and the Mac), `packaging/windows/make-payload.sh` | fetches the three files, verifies the signature with the embedded key and the binary's sha256 and size (`igneum-ota-sign verify-server embedded ... --binary`), and the payload carries them as `wsl2in\sp1-gpu-server`, `wsl2\prover-server.json` and `.sig`. The Windows installer grows by about 60 MB zipped (167 MB raw). A host without the manifest ships no server (the stock one then); a manifest that does not verify fails the build. The Mac DMG carries nothing of it (a Linux binary no Mac runs) and the Linux app proves on the CPU today, so the Hive package waits for the CUDA path on Linux | +| The app | `app/igneum-app/src/proverserver.rs`, `prover.rs`, `provedefault.rs`, `config.rs`, `state.rs` | at the prover's probe the three files are verified again (signature, then sha256 and size; a modified binary is refused before anything runs it, tested), the server is installed into Ubuntu-24.04 at `~/.sp1/bin/sp1-gpu-server` where the SDK looks (replacing whatever is there when the sha256 differs; the stock server, which answers `--version` 6.8.1 too, is told apart by its sha256 c2642ad1), and `/api/state` carries `server_version`, `server_sha256`, `server_kind` (patched, stock, unknown), `server_profile` and `server_note`. Each proof gets `SP1_GPU_ELEMENT_THRESHOLD` from the card's profile. If a proof fails because the server did not come up, the stock server is restored at the next probe and the tile says so (tested on the log lines the SDK writes) | +| The tiers (`provedefault::profile`, from the per-card VRAM; Settings `prove_profile` overrides: auto, 2^25, 2^26, 2^27, stock) | 12 GB (10 to 14 GB read): 2^26, mines and proves; 16 GB (14 to 20): 2^27; 24 and 32 GB: the server's own sizes; under 10 GB: off, with the reason (the compressed shard alone is 7.6 GB on the 4070). The install-time default (`decide_with_server`) moves its gate from 24 GB to 12 GB only when the verified server is in the payload | measured rows: the 4070 itself (12 GB); the 5090's allocation for 16, 24 and 32 GB | + +The tier table, measured (bench-log "prover floor"; alone / beside the card's own miner; the v1 shard, 4.7 M cycles): + +| Card | Server | Peak MiB alone (time) | Peak MiB beside the miner (time) | Row | +|---|---|---|---|---| +| RTX 4070 12 GB (12,282), the card itself | patched, 2^26 | 7,553 (7.7 s) | 9,034 (24.1 s) | mines and proves | +| RTX 4070 12 GB, the card itself | patched, 2^27 | 10,177 (5.5 s) | 11,754 (17.3 s) | fits with 0.5 GB spare: not the profile | +| RTX 4070 12 GB, the card itself | patched, 2^25 core-only | 5,761 | 7,242 | the reserve (a hand-off path, route 2) | +| 16 GB (the 5090's allocation) | patched, 2^27 | 12,915 (4.3 s) | 10,953 own + the miner (17.4 s) | mines and proves, the card itself not yet run | +| 24 and 32 GB (the 5090) | patched, upstream's threshold | 16,851 (4.0 s) | | unchanged, 3.7 GB more headroom than stock | +| any card under 24 GB | stock | refused before allocating | | the 24 GB gate until the server ships | + +What the next cut's shipper must do (0.3.13), in order: (1) run the `prover-server` workflow on master (or merge +prover-floor and let the push trigger it; about 40 minutes cold on the hosted runner, approximate); (2) on the Mac, +`packaging/prover/push-server.sh --run ` (the OTA key, the dl token and the Vercel login as for +push-inputs.sh), which publishes and deploys the three files; (3) the Windows build then picks them up on its own +(the new `prover server` step before `payload inputs`); on a Mac-made payload, `packaging/prover/fetch-server.sh` +first; (4) the verification run on PC 2 before the publish: the app on the cut proves one shard with +`/api/state` showing `server_kind: patched` and the card's `server_profile`, and `--mode id` unchanged; (5) the +public line moves to "12 GB mines and proves (2^26), 16 GB and up at 2^27" with the cut, not before. Every SP1 +upgrade repeats (1) to (4) with the patch rebased (the patch is against the tag; `git apply` fails loudly when it +no longer fits). The root-socket class on PC 2, the two times: 20:00:56Z (my chain job's root run; the live prover failed with Connect(PermissionDenied) until the socket was gone) and 21:25:24Z (the aggregation-cost job's root run; the prover stayed dark through the 0.3.10 restart at 21:49:41Z until `socketfix-pc2-pv1` removed the root-owned `/tmp/sp1-cuda-0.sock` at 22:01:16Z; the next shard, block 89011, was proven at 22:02:13Z and paid, and every shard since). The permanent fix in the 0.3.11 app tree: every committed playbook that runs a prove mode as root carries `pkill -f sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock` at its start and end, `tools/ci/prover-socket-check.sh` (in `ci.yml`) fails a playbook without them, and the app's prover names the cause in its log line when the host reports PermissionDenied. The app itself cannot remove a socket another user owns, so a job written outside the tree must still follow the rule. diff --git a/packaging/prover/build-server.sh b/packaging/prover/build-server.sh new file mode 100755 index 000000000..e7fd5ba0f --- /dev/null +++ b/packaging/prover/build-server.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Builds the project's sp1-gpu-server (prover floor, 6 October 2026): SP1 at the pinned tag with the floor patch, +# which sizes the server's buffers to the shard instead of to a 24 GB card (bench-log "prover floor"; the RTX 4070 +# mines and proves at 2^26). The one recipe CI (.github/workflows/prover-server.yml) and the PCs' WSL2 path +# (tools/prover-floor/pc2-build-server.ps1) share. +# +# packaging/prover/build-server.sh [source dir] +# +# Needs: nvcc (CUDA 12.8), cmake, gcc or clang, protoc, cargo, git, and Go (the server's native-gnark feature; +# GO_TARBALL_URL and GO_SHA256 below fetch go1.27.1 when `go` is missing). CUDA_ARCHS (default 80,86,89,120: the +# 12 GB tier is sm_86 (3060) and sm_89 (4070), 16 GB sm_89 and sm_120, the 24 GB 3090 sm_86, the 4090 sm_89, the 5090 +# sm_120; the stock server lists 80, 86, 89, 90, 100, 120; sm_75 (Turing) is not in SP1's kernels, CUDA_ARCHS=75 +# fails in the sys crate's build). Writes /sp1-gpu-server, /sp1-gpu-server.sha256 and /build.json +# (the descriptor the signed manifest is made from: SP1 version and commit, the patch's sha256, the archs, the +# toolchain, the time). Not byte-reproducible across machines: SP1's prover-types build script stamps the build time +# into the binary, so the signed manifest names ONE build (CI's); a PC build is a behavioural cross-check. +set -euo pipefail +OUT="${1:?out dir}"; SRC="${2:-${TMPDIR:-/tmp}/igneum-sp1-6.8.1}" +HERE="$(cd "$(dirname "$0")" && pwd)"; ROOT="$(cd "$HERE/../.." && pwd)" +PATCH="$ROOT/proving/prover-floor/sp1-gpu-6.8.1-floor.patch" +SP1_TAG="v6.8.1"; SP1_COMMIT_EXPECTED="c84ada1ed5911f28c4d3c9d0ed2f9e6cd7edb824" +CUDA_ARCHS="${CUDA_ARCHS:-80,86,89,120}" +GO_TARBALL_URL="https://go.dev/dl/go1.27.1.linux-amd64.tar.gz"; GO_SHA256="63d339f0da5ab53635a56f2490a7984dfe12dfcff22ad749f63edaf590168445" +JOBS="${CARGO_JOBS:-$(nproc)}" +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +mkdir -p "$OUT" +[ -f "$PATCH" ] || { echo "no patch at $PATCH" >&2; exit 2; } +PATCH_SHA="$(sha256sum "$PATCH" | cut -c1-64)" +CUDA_DIR="${CUDA_PATH:-$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)}" +[ -n "$CUDA_DIR" ] && export CUDA_PATH="$CUDA_DIR" CUDACXX="$CUDA_DIR/bin/nvcc" PATH="$CUDA_DIR/bin:$PATH" LD_LIBRARY_PATH="$CUDA_DIR/lib64:${LD_LIBRARY_PATH:-}" +command -v nvcc >/dev/null || { echo "no nvcc on the PATH (CUDA 12.8 toolkit)" >&2; exit 2; } +if ! command -v go >/dev/null; then + GODIR="${GO_DIR:-$OUT/go-toolchain}" + if [ ! -x "$GODIR/go/bin/go" ]; then + echo "fetching Go ($GO_TARBALL_URL)" + mkdir -p "$GODIR"; curl -sSL -o "$GODIR/go.tgz" "$GO_TARBALL_URL" + echo "$GO_SHA256 $GODIR/go.tgz" | sha256sum -c - >/dev/null || { echo "Go tarball sha256 mismatch" >&2; exit 2; } + tar -xzf "$GODIR/go.tgz" -C "$GODIR" && rm -f "$GODIR/go.tgz" + fi + export PATH="$GODIR/go/bin:$PATH" GOPATH="$GODIR/gopath" GOCACHE="$GODIR/gocache" GOFLAGS=-mod=mod +fi +echo "toolchain: nvcc $(nvcc --version | grep -o 'release [0-9.]*'), $(cargo --version), $(go version), protoc $(protoc --version 2>/dev/null || echo MISSING), cmake $(cmake --version | head -1 || echo MISSING), jobs $JOBS, CUDA_ARCHS $CUDA_ARCHS" +if [ ! -d "$SRC/.git" ]; then git clone -q --depth 1 --branch "$SP1_TAG" https://github.com/succinctlabs/sp1 "$SRC"; fi +cd "$SRC" +git checkout -q -- . && git clean -qfd sp1-gpu/crates >/dev/null 2>&1 || true +SP1_COMMIT="$(git rev-parse HEAD)" +[ "$SP1_COMMIT" = "$SP1_COMMIT_EXPECTED" ] || { echo "SP1 $SP1_TAG is $SP1_COMMIT, expected $SP1_COMMIT_EXPECTED" >&2; exit 2; } +git apply "$PATCH" +# the patched files are re-stamped (cargo rebuilds by mtime; the clone may be older than the target dir) +git diff --name-only | xargs touch +echo "patched: $(git diff --stat | tail -1) (sha256 $PATCH_SHA)" +export CUDA_ARCHS +T0=$(date +%s) +cargo build --release --bin sp1-gpu-server -j "$JOBS" +BIN="${CARGO_TARGET_DIR:-$SRC/target}/release/sp1-gpu-server" +cp "$BIN" "$OUT/sp1-gpu-server"; chmod +x "$OUT/sp1-gpu-server" +SHA="$(sha256sum "$OUT/sp1-gpu-server" | cut -c1-64)"; BYTES="$(stat -c %s "$OUT/sp1-gpu-server")" +echo "$SHA sp1-gpu-server" > "$OUT/sp1-gpu-server.sha256" +VERSION="$("$OUT/sp1-gpu-server" --version 2>/dev/null || echo unknown)" +ARCHS_IN="$(cuobjdump --list-elf "$OUT/sp1-gpu-server" 2>/dev/null | grep -o 'sm_[0-9]*' | sort -u | tr '\n' ' ' | sed 's/ $//')" +cat > "$OUT/build.json" <&2; exit 2; } +mkdir -p "$OUT" +if ! curl -fsSL --retry 3 -o "$OUT/prover-server.json" "$BASE/prover-server.json"; then + echo "no prover-server.json on the downloads host: the payload ships no GPU server (the app uses SP1's stock one, 24 GB cards)"; rm -f "$OUT/prover-server.json"; exit 0 +fi +curl -fsSL --retry 3 -o "$OUT/prover-server.json.sig" "$BASE/prover-server.json.sig" +curl -fsSL --retry 3 -o "$OUT/sp1-gpu-server" "$BASE/sp1-gpu-server" +"$SIGNER" verify-server embedded "$OUT/prover-server.json" "$OUT/prover-server.json.sig" --binary "$OUT/sp1-gpu-server" +chmod +x "$OUT/sp1-gpu-server" +echo "fetched into $OUT: $(ls -la "$OUT" | tail -n +2 | awk '{print $NF, $5}' | tr '\n' ' ')" diff --git a/packaging/prover/push-server.sh b/packaging/prover/push-server.sh new file mode 100755 index 000000000..9ab861969 --- /dev/null +++ b/packaging/prover/push-server.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Publishes the project's GPU prover server (prover floor, 6 October 2026): the binary CI built +# (.github/workflows/prover-server.yml, artifact `sp1-gpu-server-floor`) or a given build folder, its signed manifest +# prover-server.json (format app/igneum-app/src/proverserver.rs: the SP1 version and commit it patches, the patch's +# sha256, the CUDA targets, the binary's sha256 and size, the build) and prover-server.json.sig, the detached Ed25519 +# signature made on this Mac with the OTA key the apps already trust, into the downloads folder (dl//), and +# deploys it. The Windows build (.github/workflows/windows.yml) and packaging/prover/fetch-server.sh fetch the three +# files from there and verify them before the payload carries them (wsl2\bin\sp1-gpu-server, wsl2\prover-server.json +# and .sig); the app verifies them again before use. +# +# packaging/prover/push-server.sh --run the CI artifact (gh run download; gh must be logged in as igneum-labs) +# packaging/prover/push-server.sh --dir a folder with sp1-gpu-server, sp1-gpu-server.sha256 and build.json +# [--no-deploy] +# +# Same secrets layout as push-inputs.sh: ~/.config/igneum/{dl-token, dlsite-dir, vercel, ota-signing-key(.pub)}. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)"; ROOT="$(cd "$HERE/../.." && pwd)" +RUN=""; DIR=""; DEPLOY=1 +while [ $# -gt 0 ]; do case "$1" in --run) RUN="$2"; shift 2 ;; --dir) DIR="$2"; shift 2 ;; --no-deploy) DEPLOY=0; shift ;; *) echo "unknown argument $1" >&2; exit 2 ;; esac; done +[ -n "$RUN" ] || [ -n "$DIR" ] || { echo "usage: push-server.sh --run | --dir [--no-deploy]" >&2; exit 2; } +TOKEN="$(tr -d '[:space:]' < "$HOME/.config/igneum/dl-token")" +DLSITE="${IGNEUM_DLSITE:-}"; [ -n "$DLSITE" ] || DLSITE="$(tr -d '[:space:]' < "$HOME/.config/igneum/dlsite-dir")" +[ -d "$DLSITE/dl/$TOKEN" ] || { echo "no downloads folder at $DLSITE/dl/" >&2; exit 1; } +KEY="$HOME/.config/igneum/ota-signing-key"; PUB="$HOME/.config/igneum/ota-signing-key.pub" +SIGNER="$ROOT/app/igneum-app/target/release/igneum-ota-sign" +if [ ! -x "$SIGNER" ]; then echo "building igneum-ota-sign"; (cd "$ROOT/app/igneum-app" && nice -n 19 cargo build --release -j 4 --bin igneum-ota-sign --quiet); fi +if [ -n "$RUN" ]; then + DIR="$(mktemp -d)/server"; mkdir -p "$DIR" + gh auth status 2>/dev/null | grep -q "igneum-labs" || echo "warning: gh's active account is not igneum-labs (gh auth switch --user igneum-labs)" + gh run download "$RUN" --repo igneum-network/igneum --name sp1-gpu-server-floor --dir "$DIR" + BUILT_BY="ci:$RUN" +else + BUILT_BY="dir:$(hostname -s)" +fi +BIN="$DIR/sp1-gpu-server"; [ -f "$BIN" ] || { echo "no sp1-gpu-server in $DIR" >&2; exit 1; } +[ -f "$DIR/build.json" ] || { echo "no build.json in $DIR (build-server.sh writes it)" >&2; exit 1; } +# the hash the builder recorded must be the file's hash now (a corrupt download is refused here) +SHA="$(shasum -a 256 "$BIN" | cut -c1-64)"; BYTES="$(stat -f %z "$BIN")" +REC="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["sha256"])' "$DIR/build.json")" +[ "$SHA" = "$REC" ] || { echo "the binary's sha256 $SHA is not build.json's $REC" >&2; exit 1; } +[ -f "$DIR/sp1-gpu-server.sha256" ] && { grep -q "^$SHA " "$DIR/sp1-gpu-server.sha256" || { echo "sp1-gpu-server.sha256 disagrees" >&2; exit 1; }; } +DEST="$DLSITE/dl/$TOKEN" +python3 - "$DIR/build.json" "$DEST/prover-server.json" "$BUILT_BY" "$SHA" "$BYTES" <<'PY' +import json, sys, datetime +b = json.load(open(sys.argv[1])) +m = {"format": "igneum-prover-server/1", "sp1_version": b["sp1_version"], "sp1_commit": b["sp1_commit"], "patch_sha256": b["patch_sha256"], + "cuda_archs": b.get("elf_targets") or b["cuda_archs"], "built_at": b["built_at"], "built_by": sys.argv[3], + "published_at": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "files": {"sp1-gpu-server": {"sha256": sys.argv[4], "bytes": int(sys.argv[5])}}} +open(sys.argv[2], "w").write(json.dumps(m, sort_keys=True, separators=(",", ":")) + "\n") +PY +"$SIGNER" sign "$KEY" "$DEST/prover-server.json" > "$DEST/prover-server.json.sig" +cp "$BIN" "$DEST/sp1-gpu-server" +"$SIGNER" verify-server "$PUB" "$DEST/prover-server.json" "$DEST/prover-server.json.sig" --binary "$DEST/sp1-gpu-server" +"$SIGNER" verify-server embedded "$DEST/prover-server.json" "$DEST/prover-server.json.sig" >/dev/null || { echo "the embedded key does not verify this signature: the OTA key on this Mac is not the one the apps carry" >&2; exit 1; } +cat "$DEST/prover-server.json"; echo "signature: $(cut -c1-16 "$DEST/prover-server.json.sig")..." +if [ "$DEPLOY" = 1 ]; then + echo "deploying $DLSITE" + (cd "$DLSITE" && npx vercel@latest --global-config "$HOME/.config/igneum/vercel" deploy --prod --yes 2>&1 | grep -v "$TOKEN" || true) + echo "published: https://dl.igneum.network/dl//{sp1-gpu-server,prover-server.json,prover-server.json.sig}" +else + echo "not deployed (--no-deploy); run the Vercel deploy from $DLSITE when ready" +fi diff --git a/packaging/windows/make-payload.sh b/packaging/windows/make-payload.sh index 1d7ecce59..c446f9eab 100755 --- a/packaging/windows/make-payload.sh +++ b/packaging/windows/make-payload.sh @@ -97,6 +97,20 @@ if [ -f "$PROVE_LINUX/igneum-prove-host" ] && [ -f "$PROVE_LINUX/igneum-prove-ex cp "$PROVE_LINUX/igneum-prove-host" "$PROVE_LINUX/igneum-prove-export" "$STAGE/wsl2/bin/" echo "prover (WSL2): igneum-prove-host $(stat -f %z "$PROVE_LINUX/igneum-prove-host") bytes, igneum-prove-export $(stat -f %z "$PROVE_LINUX/igneum-prove-export") bytes" else echo "warning: no Linux igneum-prove-host/igneum-prove-export in $PROVE_LINUX (cargo zigbuild --target x86_64-unknown-linux-gnu.2.36 --features igneum-prove-host/cuda); the Proving tile will ask for the WSL2 setup, which builds them"; fi +# prover floor (6 October 2026, app/igneum-app/src/proverserver.rs): the project's build of SP1's GPU server, which +# lets 12 GB and 16 GB cards prove (the stock one refuses them). The three files come from packaging/prover/ +# fetch-server.sh (the downloads host, published by push-server.sh; IGNEUM_PROVER_SERVER overrides the folder) and +# are verified here with the key the apps carry before they go in; the app verifies them again before use. Without +# them the app runs SP1's stock server and the 24 GB gate. The binary is 167 MB (about 60 MB zipped). +SERVER_DIR="${IGNEUM_PROVER_SERVER:-$ROOT/proving/prover-floor/server}" +SIGNER_BIN="${IGNEUM_OTA_SIGN:-$ROOT/app/igneum-app/target/release/igneum-ota-sign}" +if [ -f "$SERVER_DIR/sp1-gpu-server" ] && [ -f "$SERVER_DIR/prover-server.json" ] && [ -f "$SERVER_DIR/prover-server.json.sig" ]; then + if [ -x "$SIGNER_BIN" ] || [ -x "$SIGNER_BIN.exe" ]; then + "$SIGNER_BIN" verify-server embedded "$SERVER_DIR/prover-server.json" "$SERVER_DIR/prover-server.json.sig" --binary "$SERVER_DIR/sp1-gpu-server" || { echo "the prover server in $SERVER_DIR does not verify; not shipping it"; exit 1; } + else echo "warning: no igneum-ota-sign at $SIGNER_BIN; the server goes in unverified here (the app verifies it before use)"; fi + cp "$SERVER_DIR/sp1-gpu-server" "$STAGE/wsl2/bin/" && cp "$SERVER_DIR/prover-server.json" "$SERVER_DIR/prover-server.json.sig" "$STAGE/wsl2/" + echo "prover server (WSL2): sp1-gpu-server $(stat -f %z "$SERVER_DIR/sp1-gpu-server" 2>/dev/null || stat -c %s "$SERVER_DIR/sp1-gpu-server") bytes, sha256 $(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["files"]["sp1-gpu-server"]["sha256"][:16])' "$SERVER_DIR/prover-server.json")..., signed manifest wsl2/prover-server.json" +else echo "warning: no verified prover server in $SERVER_DIR (packaging/prover/fetch-server.sh); the app will run SP1's stock server, which refuses cards under 24 GB"; fi cp "$ROOT"/proving/windows-wsl2/*.sh "$ROOT"/proving/windows-wsl2/*.ps1 "$ROOT"/proving/windows-wsl2/*.bat "$ROOT/proving/windows-wsl2/README.txt" "$STAGE/wsl2/" cp "$ROOT"/proving/fixtures/*.json "$STAGE/wsl2/fixtures/"