From 4da5edf24fa8defa8c956700b79484c41246e9df Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:25:08 +0000 Subject: [PATCH] MF-10: the prover reads the card's compute capability and the GPU server's sm_ words and refuses a mismatch with the reason (and reads a named-symbol proof failure as the same class); MF-8: the injector's kept-datadir step (a previous release's node writes the datadir, the new node must open it) Co-Authored-By: Claude Fable 5.1 --- app/igneum-app/src/provedefault.rs | 45 ++++++++++++++++++++++++++ app/igneum-app/src/prover.rs | 44 +++++++++++++++++++++++++- tools/reliability/app-run.mjs | 51 ++++++++++++++++++++++++++++-- 3 files changed, 137 insertions(+), 3 deletions(-) diff --git a/app/igneum-app/src/provedefault.rs b/app/igneum-app/src/provedefault.rs index 2516fdae..729eb7f3 100644 --- a/app/igneum-app/src/provedefault.rs +++ b/app/igneum-app/src/provedefault.rs @@ -203,3 +203,48 @@ mod tests { assert_eq!(PROVE_UNDER_12GB_LINE, "proving needs a 12 GB card; mining continues"); } } + +/// MF-10 (docs/plans/miner-faults.md, 7 October 2026): a `sp1-gpu-server` built for another card's architecture fails +/// every proof in 12 s with `CudaRustError: named symbol not found` and nothing notices. `card_cap` is the card's +/// compute capability as nvidia-smi prints it ("12.0", "8.9", "8.6"); `server_archs` are the `sm_NN` words found in +/// the server binary (empty = unknown, which passes: an SDK server may carry no arch string). Some(the sentence) when +/// the server names architectures and none is the card's. +pub fn server_mismatch(card_cap: &str, server_archs: &[String]) -> Option { + let cap = card_cap.trim(); + if cap.is_empty() || server_archs.is_empty() { + return None; + } + let want = format!("sm_{}", cap.replace('.', "")); + if server_archs.iter().any(|a| a.trim() == want) { + return None; + } + Some(format!("the proving server is built for {} and this card is {want} (compute capability {cap}); proving stays off here until a server for this card is installed", server_archs.join(", "))) +} + +/// A proof failure line that names the architecture class (MF-10): the server's kernels do not load on this card. +pub fn is_arch_failure(text: &str) -> bool { + text.contains("named symbol not found") || text.contains("no kernel image is available") +} + +#[cfg(test)] +mod arch_tests { + use super::*; + + /// The prover roll, 7 October 2026: sm_86 servers on a 4070 (8.9) and a 5090 (12.0) failed every proof; the + /// 3080 and 3090 (8.6) proved. + #[test] + fn a_server_for_another_card_is_refused() { + let sm86 = vec!["sm_86".to_string()]; + assert!(server_mismatch("8.9", &sm86).unwrap().contains("built for sm_86 and this card is sm_89")); + assert!(server_mismatch("12.0", &sm86).unwrap().contains("sm_120")); + assert_eq!(server_mismatch("8.6", &sm86), None); + // a fat binary names every card + let fat = vec!["sm_86".to_string(), "sm_89".to_string(), "sm_120".to_string()]; + assert_eq!(server_mismatch("8.9", &fat), None); + // unknown on either side passes (the proof failure line is the second guard) + assert_eq!(server_mismatch("", &sm86), None); + assert_eq!(server_mismatch("8.9", &[]), None); + assert!(is_arch_failure("CudaRustError: named symbol not found")); + assert!(!is_arch_failure("PermissionDenied")); + } +} diff --git a/app/igneum-app/src/prover.rs b/app/igneum-app/src/prover.rs index 724fc12d..3f5ae8f2 100644 --- a/app/igneum-app/src/prover.rs +++ b/app/igneum-app/src/prover.rs @@ -384,6 +384,8 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { read_ids(&shared, &t); } } + // MF-10: the server architecture check, once per tools probe (None = not read yet; Some(None) = matches) + let mut arch_check: Option> = None; loop { std::thread::sleep(Duration::from_secs(10)); let enabled = shared.settings.lock().unwrap().prove; @@ -468,6 +470,23 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { continue; } } + // MF-10 (7 October 2026, the prover roll): a GPU server built for another card's architecture fails every + // proof; the card's compute capability and the server's sm_ words are read once per tools probe, a mismatch + // refuses the GPU path with the reason on the tile + if t.cuda { + if arch_check.is_none() { + arch_check = Some(server_arch_check(&shared, t)); + } + if let Some(Some(line)) = arch_check.as_ref() { + set(&shared, |p| { + p.enabled = true; + p.available = false; + p.status = "off".into(); + p.message = line.clone(); + }); + continue; + } + } set(&shared, |p| { p.enabled = true; p.available = true; @@ -702,7 +721,10 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { let last = out.lines().rev().find(|l| l.contains("RESULT") || l.contains("rror")).unwrap_or("failed").to_string(); // the root-socket class (5 October 2026, PC 2 at 20:00Z and 21:25Z): a job that ran the host as root // inside WSL2 left /tmp/sp1-cuda-0.sock owned by root, and this user's client cannot open it - let hint = if last.contains("PermissionDenied") { " (a GPU-server socket /tmp/sp1-cuda-*.sock owned by another user, left by a job that ran the prover as root: remove it as that user, or run the socket-fix job)" } else { "" }; + let hint = if crate::provedefault::is_arch_failure(&last) { " (MF-10: the proving server is built for another card's architecture; proving stays off here until a server for this card is installed)" } else if last.contains("PermissionDenied") { " (a GPU-server socket /tmp/sp1-cuda-*.sock owned by another user, left by a job that ran the prover as root: remove it as that user, or run the socket-fix job)" } else { "" }; + if crate::provedefault::is_arch_failure(&last) { + shared.log(&format!("FAULT class=prover-arch card=\"gpu\" app={} reason=\"{}\"", crate::engine::VERSION, last.replace('"', "'"))); + } return Err(format!("prover: {last}{hint}")); } let res: Value = serde_json::from_str(&std::fs::read_to_string(&results).map_err(|e| e.to_string())?).map_err(|e| e.to_string())?; @@ -1175,3 +1197,23 @@ mod tests { assert!(probe_message(Path::new("C:\\Igneum"), false, true).contains("Ubuntu-24.04 did not answer")); } } + +/// MF-10: the card's compute capability and the GPU server's architecture words, read through the same shell the +/// tools live in (WSL2 on Windows). None = no mismatch (or nothing readable); Some(line) = refuse with this reason. +fn server_arch_check(shared: &Shared, t: &Tools) -> Option { + let script = "cap=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1); srv=\"$HOME/.sp1/bin/sp1-gpu-server\"; archs=$( [ -f \"$srv\" ] && strings \"$srv\" 2>/dev/null | grep -o 'sm_[0-9]*' | sort -u | tr '\\n' ' '); echo \"CAP=$cap\"; echo \"ARCHS=$archs\""; + let out = if t.wsl { + let file = crate::wslhost::write_script("prove-arch", script).ok()?; + crate::platform::quiet(&mut crate::wslhost::command(&crate::platform::tool("wsl"), crate::wslhost::DISTRO, None, &file.path, true, &[])).output().ok().map(|o| String::from_utf8_lossy(&o.stdout).to_string())? + } else { + crate::detect::run_timeout(std::process::Command::new("bash").args(["-c", script]), None, Duration::from_secs(20))? + }; + let cap = out.lines().find_map(|l| l.strip_prefix("CAP=")).unwrap_or("").trim().to_string(); + let archs: Vec = out.lines().find_map(|l| l.strip_prefix("ARCHS=")).unwrap_or("").split_whitespace().map(|s| s.to_string()).collect(); + shared.log(&format!("prover: card compute capability {}, server architectures {}", if cap.is_empty() { "unknown" } else { &cap }, if archs.is_empty() { "unknown".to_string() } else { archs.join(" ") })); + let line = crate::provedefault::server_mismatch(&cap, &archs); + if let Some(l) = &line { + shared.log(&format!("FAULT class=prover-arch card=\"gpu\" app={} reason=\"{}\"", crate::engine::VERSION, l.replace('"', "'"))); + } + line +} diff --git a/tools/reliability/app-run.mjs b/tools/reliability/app-run.mjs index 539cdee4..e35f2769 100755 --- a/tools/reliability/app-run.mjs +++ b/tools/reliability/app-run.mjs @@ -21,6 +21,8 @@ // no tap; it leaves: its row is marked removed; it comes back: mining again (Linux and Windows only) // node-silent the node is stopped with SIGSTOP: no sign of life for 120 s, the app restarts the node in-process // and the miner comes back once it is ready +// kept-datadir MF-8 (with --node-old): the previous release's node ran this datadir to a few hundred blocks and was +// stopped; the engine's new node must open it and sync (a node that dies inside 10 s is the class) // orphan-miner MF-7: a stray igneum-miner on this engine's node, not started by it, is killed by the minute sweep; // the engine's own miner is left alone // one-card-fails MF-4: two more cards appear, one failing its self-test for ever: the healthy two mine, the failing @@ -38,8 +40,12 @@ const opt = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : const APP = opt('--app'); const MINER = opt('--miner'); const NODE = opt('--node'); +// MF-8: a previous release's igneumd; the kept-datadir pre-phase runs it first on the app's node dir, stops it, and the +// engine's own (new) node must come up on that datadir +const NODE_OLD = opt('--node-old'); const ONLY = opt('--only', '').split(',').filter(Boolean); const SCRATCH = process.env.SCRATCH || `/tmp/igneum-reliability-app-${process.pid}`; +const CHAIN_ID_ENV = {}; const RPC = 29960, P2P = 29961, SUFFIX = 9960; const MAC = process.platform === 'darwin'; for (const [k, v] of Object.entries({ APP, MINER, NODE })) if (!v || !existsSync(v)) { console.error(`missing ${k} (${v})`); process.exit(2); } @@ -75,6 +81,31 @@ const env = { // easy genesis bits so the CPU block producer of the catch-up step makes blocks on two threads IGNEUM_DEVNET_GENESIS_BITS: '0x1f100000', }; +// MF-8 pre-phase: the previous release writes the datadir first +let keptPre = null; +if (NODE_OLD) { + if (!existsSync(NODE_OLD)) { console.error(`missing --node-old ${NODE_OLD}`); process.exit(2); } + const ndir = join(data, `devnet-${SUFFIX}`); mkdirSync(ndir, { recursive: true }); + const nodeArgs = ['--devnet', `--devnet-suffix=${SUFFIX}`, '--nodnsseed', '--disable-upnp', '--nologfiles', '--enable-unsynced-mining', '--outpeers=0', + `--appdir=${ndir}`, `--rpclisten=127.0.0.1:${RPC}`, `--evm-rpclisten=127.0.0.1:${RPC + 180}`, `--listen=127.0.0.1:${P2P}`, '--loglevel=warn', '--yes']; + const oldLog = join(SCRATCH, 'node-old.log'); + const oldNode = spawn(NODE_OLD, nodeArgs, { stdio: ['ignore', 'pipe', 'pipe'], env: { ...env } }); + oldNode.stdout.on('data', d => appendFileSync(oldLog, d)); oldNode.stderr.on('data', d => appendFileSync(oldLog, d)); + log(`kept-datadir: the previous release's node pid ${oldNode.pid} on ${ndir}`); + await sleep(5000); + const producer = spawn(MINER, ['mine', `grpc://127.0.0.1:${RPC}`, '2', '600', 'cpu0', '--engine', 'igneum-pow', '--no-vote', '--payout-label', 'cpu0'], { stdio: ['ignore', 'pipe', 'ignore'] }); + let blocks = 0; producer.stdout.on('data', d => { blocks += (d.toString().match(/ACCEPTED block/g) || []).length; }); + const t0b = Date.now(); + while (blocks < 150 && Date.now() - t0b < 400000) await sleep(2000); + try { producer.kill('SIGTERM'); } catch {} + await sleep(3000); + const tStop = Date.now(); + oldNode.kill('SIGTERM'); + let gone = false; for (let i = 0; i < 60 && !gone; i++) { await sleep(500); try { process.kill(oldNode.pid, 0); } catch { gone = true; } } + if (!gone) { oldNode.kill('SIGKILL'); await sleep(1000); } + keptPre = { blocks, stop_s: (Date.now() - tStop) / 1000, gone }; + log(`kept-datadir: ${blocks} blocks written by the previous release, its node stopped in ${keptPre.stop_s.toFixed(1)} s`); +} const app = spawn(APP, ['--no-open'], { stdio: ['pipe', 'pipe', 'pipe'], env }); const appOut = join(SCRATCH, 'app.out'); app.stdout.on('data', d => appendFileSync(appOut, d)); app.stderr.on('data', d => appendFileSync(appOut, d)); @@ -363,8 +394,24 @@ S['orphan-miner'] = async () => { ], { inject_to_kill: killedAt ? s(killedAt - tInject) : 'n/a' }); }; -const order = ['catch-up', 'card-appears', 'own-restart', 'zero-ladder', 'no-status', 'node-silent', 'one-card-fails', 'orphan-miner']; -const names = ONLY.length ? order.filter(n => ONLY.includes(n)) : order; +S['kept-datadir'] = async () => { + if (!NODE_OLD) { verdict('kept-datadir', [{ ok: false, what: 'not run: no --node-old given (a previous release\'s igneumd)' }], {}); return; } + const tStart = Date.now(); + const synced = await until((st) => st.node.synced && st.node.starts >= 1, 240000, 'the new node to come up on the kept datadir'); + const tSynced = Date.now(); + const st1 = await poll(); + const died = engineLogLines(/igneumd exited at once|igneumd exited with code/); + const rewrite = engineLogLines(/virtual state|rewrit|v1 row/i); + verdict('kept-datadir', [ + { ok: keptPre && keptPre.blocks >= 100, what: `the previous release wrote the datadir (${keptPre ? keptPre.blocks : 0} blocks) and stopped in ${keptPre ? keptPre.stop_s.toFixed(1) : '?'} s` }, + { ok: !!synced, what: `the new node opened the kept datadir and read synced (${s(tSynced - tStart)} after the engine started)` }, + { ok: died.length === 0, what: `no node exit at start (${died.length} exit lines)` }, + { ok: st1 && st1.node.starts === 1, what: `one node start, no restart loop (starts ${st1 ? st1.node.starts : '?'})` }, + ], { blocks_by_old_node: keptPre ? keptPre.blocks : 0, old_node_stop_s: keptPre ? keptPre.stop_s.toFixed(1) : 'n/a', new_node_synced_after: s(tSynced - tStart), rewrite_lines: rewrite.length }); +}; + +const order = ['kept-datadir', 'catch-up', 'card-appears', 'own-restart', 'zero-ladder', 'no-status', 'node-silent', 'one-card-fails', 'orphan-miner']; +const names = (ONLY.length ? order.filter(n => ONLY.includes(n)) : order).filter(n => n !== 'kept-datadir' || NODE_OLD); for (const n of names) { log(`=== ${n}`); try { await S[n](); } catch (e) { log(`step ${n} threw: ${e.stack || e}`); results.push({ name: n, pass: false, checks: [{ ok: false, what: String(e) }], metrics: {} }); }