igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

MF-10: the prover reads the card's compute capability and the GPU server's sm_ words and refuses a mismatch with the reason (and reads a named-symbol proof failure as the same class); MF-8: the injector's kept-datadir step (a previous release's node writes the datadir, the new node must open it)

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-07 13:25:08 +00:00
parent 240ecf122f
commit 4da5edf24f
3 changed files with 137 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
app/igneum-app/src/provedefault.rs
View file

@ -203,3 +203,48 @@ mod tests {
assert_eq!(PROVE_UNDER_12GB_LINE, "proving needs a 12 GB card; mining continues");
}
}
/// MF-10 (docs/plans/miner-faults.md, 7 October 2026): a `sp1-gpu-server` built for another card's architecture fails
/// every proof in 12 s with `CudaRustError: named symbol not found` and nothing notices. `card_cap` is the card's
/// compute capability as nvidia-smi prints it ("12.0", "8.9", "8.6"); `server_archs` are the `sm_NN` words found in
/// the server binary (empty = unknown, which passes: an SDK server may carry no arch string). Some(the sentence) when
/// the server names architectures and none is the card's.
pub fn server_mismatch(card_cap: &str, server_archs: &[String]) -> Option<String> {
let cap = card_cap.trim();
if cap.is_empty() || server_archs.is_empty() {
return None;
}
let want = format!("sm_{}", cap.replace('.', ""));
if server_archs.iter().any(|a| a.trim() == want) {
return None;
}
Some(format!("the proving server is built for {} and this card is {want} (compute capability {cap}); proving stays off here until a server for this card is installed", server_archs.join(", ")))
}
/// A proof failure line that names the architecture class (MF-10): the server's kernels do not load on this card.
pub fn is_arch_failure(text: &str) -> bool {
text.contains("named symbol not found") || text.contains("no kernel image is available")
}
#[cfg(test)]
mod arch_tests {
use super::*;
/// The prover roll, 7 October 2026: sm_86 servers on a 4070 (8.9) and a 5090 (12.0) failed every proof; the
/// 3080 and 3090 (8.6) proved.
#[test]
fn a_server_for_another_card_is_refused() {
let sm86 = vec!["sm_86".to_string()];
assert!(server_mismatch("8.9", &sm86).unwrap().contains("built for sm_86 and this card is sm_89"));
assert!(server_mismatch("12.0", &sm86).unwrap().contains("sm_120"));
assert_eq!(server_mismatch("8.6", &sm86), None);
// a fat binary names every card
let fat = vec!["sm_86".to_string(), "sm_89".to_string(), "sm_120".to_string()];
assert_eq!(server_mismatch("8.9", &fat), None);
// unknown on either side passes (the proof failure line is the second guard)
assert_eq!(server_mismatch("", &sm86), None);
assert_eq!(server_mismatch("8.9", &[]), None);
assert!(is_arch_failure("CudaRustError: named symbol not found"));
assert!(!is_arch_failure("PermissionDenied"));
}
}

44
app/igneum-app/src/prover.rs
View file

@ -384,6 +384,8 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
read_ids(&shared, &t);
}
}
// MF-10: the server architecture check, once per tools probe (None = not read yet; Some(None) = matches)
let mut arch_check: Option<Option<String>> = None;
loop {
std::thread::sleep(Duration::from_secs(10));
let enabled = shared.settings.lock().unwrap().prove;
@ -468,6 +470,23 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
continue;
}
}
// MF-10 (7 October 2026, the prover roll): a GPU server built for another card's architecture fails every
// proof; the card's compute capability and the server's sm_ words are read once per tools probe, a mismatch
// refuses the GPU path with the reason on the tile
if t.cuda {
if arch_check.is_none() {
arch_check = Some(server_arch_check(&shared, t));
}
if let Some(Some(line)) = arch_check.as_ref() {
set(&shared, |p| {
p.enabled = true;
p.available = false;
p.status = "off".into();
p.message = line.clone();
});
continue;
}
}
set(&shared, |p| {
p.enabled = true;
p.available = true;
@ -702,7 +721,10 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
let last = out.lines().rev().find(|l| l.contains("RESULT") || l.contains("rror")).unwrap_or("failed").to_string();
// the root-socket class (5 October 2026, PC 2 at 20:00Z and 21:25Z): a job that ran the host as root
// inside WSL2 left /tmp/sp1-cuda-0.sock owned by root, and this user's client cannot open it
let hint = if last.contains("PermissionDenied") { " (a GPU-server socket /tmp/sp1-cuda-*.sock owned by another user, left by a job that ran the prover as root: remove it as that user, or run the socket-fix job)" } else { "" };
let hint = if crate::provedefault::is_arch_failure(&last) { " (MF-10: the proving server is built for another card's architecture; proving stays off here until a server for this card is installed)" } else if last.contains("PermissionDenied") { " (a GPU-server socket /tmp/sp1-cuda-*.sock owned by another user, left by a job that ran the prover as root: remove it as that user, or run the socket-fix job)" } else { "" };
if crate::provedefault::is_arch_failure(&last) {
shared.log(&format!("FAULT class=prover-arch card=\"gpu\" app={} reason=\"{}\"", crate::engine::VERSION, last.replace('"', "'")));
}
return Err(format!("prover: {last}{hint}"));
}
let res: Value = serde_json::from_str(&std::fs::read_to_string(&results).map_err(|e| e.to_string())?).map_err(|e| e.to_string())?;
@ -1175,3 +1197,23 @@ mod tests {
assert!(probe_message(Path::new("C:\\Igneum"), false, true).contains("Ubuntu-24.04 did not answer"));
}
}
/// MF-10: the card's compute capability and the GPU server's architecture words, read through the same shell the
/// tools live in (WSL2 on Windows). None = no mismatch (or nothing readable); Some(line) = refuse with this reason.
fn server_arch_check(shared: &Shared, t: &Tools) -> Option<String> {
let script = "cap=$(nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>/dev/null | head -1); srv=\"$HOME/.sp1/bin/sp1-gpu-server\"; archs=$( [ -f \"$srv\" ] && strings \"$srv\" 2>/dev/null | grep -o 'sm_[0-9]*' | sort -u | tr '\\n' ' '); echo \"CAP=$cap\"; echo \"ARCHS=$archs\"";
let out = if t.wsl {
let file = crate::wslhost::write_script("prove-arch", script).ok()?;
crate::platform::quiet(&mut crate::wslhost::command(&crate::platform::tool("wsl"), crate::wslhost::DISTRO, None, &file.path, true, &[])).output().ok().map(|o| String::from_utf8_lossy(&o.stdout).to_string())?
} else {
crate::detect::run_timeout(std::process::Command::new("bash").args(["-c", script]), None, Duration::from_secs(20))?
};
let cap = out.lines().find_map(|l| l.strip_prefix("CAP=")).unwrap_or("").trim().to_string();
let archs: Vec<String> = out.lines().find_map(|l| l.strip_prefix("ARCHS=")).unwrap_or("").split_whitespace().map(|s| s.to_string()).collect();
shared.log(&format!("prover: card compute capability {}, server architectures {}", if cap.is_empty() { "unknown" } else { &cap }, if archs.is_empty() { "unknown".to_string() } else { archs.join(" ") }));
let line = crate::provedefault::server_mismatch(&cap, &archs);
if let Some(l) = &line {
shared.log(&format!("FAULT class=prover-arch card=\"gpu\" app={} reason=\"{}\"", crate::engine::VERSION, l.replace('"', "'")));
}
line
}

51
tools/reliability/app-run.mjs
View file

@ -21,6 +21,8 @@
// no tap; it leaves: its row is marked removed; it comes back: mining again (Linux and Windows only)
// node-silent the node is stopped with SIGSTOP: no sign of life for 120 s, the app restarts the node in-process
// and the miner comes back once it is ready
// kept-datadir MF-8 (with --node-old): the previous release's node ran this datadir to a few hundred blocks and was
// stopped; the engine's new node must open it and sync (a node that dies inside 10 s is the class)
// orphan-miner MF-7: a stray igneum-miner on this engine's node, not started by it, is killed by the minute sweep;
// the engine's own miner is left alone
// one-card-fails MF-4: two more cards appear, one failing its self-test for ever: the healthy two mine, the failing
@ -38,8 +40,12 @@ const opt = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] :
const APP = opt('--app');
const MINER = opt('--miner');
const NODE = opt('--node');
// MF-8: a previous release's igneumd; the kept-datadir pre-phase runs it first on the app's node dir, stops it, and the
// engine's own (new) node must come up on that datadir
const NODE_OLD = opt('--node-old');
const ONLY = opt('--only', '').split(',').filter(Boolean);
const SCRATCH = process.env.SCRATCH || `/tmp/igneum-reliability-app-${process.pid}`;
const CHAIN_ID_ENV = {};
const RPC = 29960, P2P = 29961, SUFFIX = 9960;
const MAC = process.platform === 'darwin';
for (const [k, v] of Object.entries({ APP, MINER, NODE })) if (!v || !existsSync(v)) { console.error(`missing ${k} (${v})`); process.exit(2); }
@ -75,6 +81,31 @@ const env = {
// easy genesis bits so the CPU block producer of the catch-up step makes blocks on two threads
IGNEUM_DEVNET_GENESIS_BITS: '0x1f100000',
};
// MF-8 pre-phase: the previous release writes the datadir first
let keptPre = null;
if (NODE_OLD) {
if (!existsSync(NODE_OLD)) { console.error(`missing --node-old ${NODE_OLD}`); process.exit(2); }
const ndir = join(data, `devnet-${SUFFIX}`); mkdirSync(ndir, { recursive: true });
const nodeArgs = ['--devnet', `--devnet-suffix=${SUFFIX}`, '--nodnsseed', '--disable-upnp', '--nologfiles', '--enable-unsynced-mining', '--outpeers=0',
`--appdir=${ndir}`, `--rpclisten=127.0.0.1:${RPC}`, `--evm-rpclisten=127.0.0.1:${RPC + 180}`, `--listen=127.0.0.1:${P2P}`, '--loglevel=warn', '--yes'];
const oldLog = join(SCRATCH, 'node-old.log');
const oldNode = spawn(NODE_OLD, nodeArgs, { stdio: ['ignore', 'pipe', 'pipe'], env: { ...env } });
oldNode.stdout.on('data', d => appendFileSync(oldLog, d)); oldNode.stderr.on('data', d => appendFileSync(oldLog, d));
log(`kept-datadir: the previous release's node pid ${oldNode.pid} on ${ndir}`);
await sleep(5000);
const producer = spawn(MINER, ['mine', `grpc://127.0.0.1:${RPC}`, '2', '600', 'cpu0', '--engine', 'igneum-pow', '--no-vote', '--payout-label', 'cpu0'], { stdio: ['ignore', 'pipe', 'ignore'] });
let blocks = 0; producer.stdout.on('data', d => { blocks += (d.toString().match(/ACCEPTED block/g) || []).length; });
const t0b = Date.now();
while (blocks < 150 && Date.now() - t0b < 400000) await sleep(2000);
try { producer.kill('SIGTERM'); } catch {}
await sleep(3000);
const tStop = Date.now();
oldNode.kill('SIGTERM');
let gone = false; for (let i = 0; i < 60 && !gone; i++) { await sleep(500); try { process.kill(oldNode.pid, 0); } catch { gone = true; } }
if (!gone) { oldNode.kill('SIGKILL'); await sleep(1000); }
keptPre = { blocks, stop_s: (Date.now() - tStop) / 1000, gone };
log(`kept-datadir: ${blocks} blocks written by the previous release, its node stopped in ${keptPre.stop_s.toFixed(1)} s`);
}
const app = spawn(APP, ['--no-open'], { stdio: ['pipe', 'pipe', 'pipe'], env });
const appOut = join(SCRATCH, 'app.out');
app.stdout.on('data', d => appendFileSync(appOut, d)); app.stderr.on('data', d => appendFileSync(appOut, d));
@ -363,8 +394,24 @@ S['orphan-miner'] = async () => {
], { inject_to_kill: killedAt ? s(killedAt - tInject) : 'n/a' });
};
const order = ['catch-up', 'card-appears', 'own-restart', 'zero-ladder', 'no-status', 'node-silent', 'one-card-fails', 'orphan-miner'];
const names = ONLY.length ? order.filter(n => ONLY.includes(n)) : order;
S['kept-datadir'] = async () => {
if (!NODE_OLD) { verdict('kept-datadir', [{ ok: false, what: 'not run: no --node-old given (a previous release\'s igneumd)' }], {}); return; }
const tStart = Date.now();
const synced = await until((st) => st.node.synced && st.node.starts >= 1, 240000, 'the new node to come up on the kept datadir');
const tSynced = Date.now();
const st1 = await poll();
const died = engineLogLines(/igneumd exited at once|igneumd exited with code/);
const rewrite = engineLogLines(/virtual state|rewrit|v1 row/i);
verdict('kept-datadir', [
{ ok: keptPre && keptPre.blocks >= 100, what: `the previous release wrote the datadir (${keptPre ? keptPre.blocks : 0} blocks) and stopped in ${keptPre ? keptPre.stop_s.toFixed(1) : '?'} s` },
{ ok: !!synced, what: `the new node opened the kept datadir and read synced (${s(tSynced - tStart)} after the engine started)` },
{ ok: died.length === 0, what: `no node exit at start (${died.length} exit lines)` },
{ ok: st1 && st1.node.starts === 1, what: `one node start, no restart loop (starts ${st1 ? st1.node.starts : '?'})` },
], { blocks_by_old_node: keptPre ? keptPre.blocks : 0, old_node_stop_s: keptPre ? keptPre.stop_s.toFixed(1) : 'n/a', new_node_synced_after: s(tSynced - tStart), rewrite_lines: rewrite.length });
};
const order = ['kept-datadir', 'catch-up', 'card-appears', 'own-restart', 'zero-ladder', 'no-status', 'node-silent', 'one-card-fails', 'orphan-miner'];
const names = (ONLY.length ? order.filter(n => ONLY.includes(n)) : order).filter(n => n !== 'kept-datadir' || NODE_OLD);
for (const n of names) {
log(`=== ${n}`);
try { await S[n](); } catch (e) { log(`step ${n} threw: ${e.stack || e}`); results.push({ name: n, pass: false, checks: [{ ok: false, what: String(e) }], metrics: {} }); }