diff --git a/app/igneum-app/src/prover.rs b/app/igneum-app/src/prover.rs index a1ecc6e6f..1d2140bbc 100644 --- a/app/igneum-app/src/prover.rs +++ b/app/igneum-app/src/prover.rs @@ -135,6 +135,22 @@ struct Tools { /// The node's re-derivation boundary (0.3.14, 6 October 2026): the chain block its EVM restarted at after the /// bodies below the pruning point were gone (`igneum_getExecStatus.restartNumber`; on a 0.3.13 node the /// `startedFrom` text "restart at chain block N"), else 0. The prover never claims work below it (no bodies, no +/// The exporter's failure line. When the node's export carries the account dump (0.3.14) and the exporter's output +/// never mentions it, the exporter at `path` predates 0.3.14: it replays the dump's own segment from the restart +/// state and fails there with "port state root differs" (PC 1, 6 October 2026 18:16Z, block 140,662: the installer +/// had not replaced igneum-prove-export); the line names the stale binary instead of the misleading root error. +fn exporter_failure(export_has_dump: bool, out: &str, path: &Path, block: Option) -> String { + let last = out.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("failed"); + let what = match block { + Some(b) => format!("exporter, block {b}"), + None => "exporter".to_string(), + }; + if export_has_dump && !out.contains("account dump") { + return format!("{what}: stale igneum-prove-export at {}: the node's export carries the account dump and this exporter does not read it (it predates 0.3.14); install this release's exporter there ({last})", path.display()); + } + format!("{what}: {last}") +} + /// state: unprovable on every node). fn exec_boundary(shared: &Shared) -> u64 { let st = match evm_rpc(shared, "igneum_getExecStatus", json!([]), Duration::from_secs(5)) { @@ -665,7 +681,7 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { let (seq_p, fix_p, res_p) = if t.wsl { (wsl_path(&seq), wsl_path(&fixture), wsl_path(&results)) } else { (seq.display().to_string(), fixture.display().to_string(), results.display().to_string()) }; let (ok, out) = run_tool(&shared, t, &t.export, &[seq_p, w.number.to_string(), fix_p.clone()], &[], Duration::from_secs(600), &dir.join(format!("export-{}.log", w.number))); if !ok || !fixture.exists() { - return Err(format!("exporter: {}", out.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("failed"))); + return Err(exporter_failure(export["preState"].is_array(), &out, &t.export, None)); } set(&shared, |p| p.message = if t.cuda { "proving on the GPU".into() } else { "CPU prover: about five minutes a shard, 30 GB of RAM, paid only when no card proves first".into() }); let prover_env = if t.cuda { "cuda" } else { "cpu" }; @@ -854,7 +870,7 @@ fn prove_segment(shared: &Shared, t: &Tools, seg: &crate::segments::SegmentWork, let fixture = dir.join(format!("block-{b}.json")); let (ok, out) = run_tool(shared, t, &t.export, &[as_host_path(&seq), b.to_string(), as_host_path(&fixture)], &[], Duration::from_secs(600), &dir.join(format!("export-{b}.log"))); if !ok || !fixture.exists() { - return Err(format!("exporter, block {b}: {}", out.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("failed"))); + return Err(exporter_failure(export["preState"].is_array(), &out, &t.export, Some(b))); } fixtures.push(as_host_path(&fixture)); } @@ -1092,6 +1108,21 @@ pub fn setup(shared: &Shared) -> Result { mod tests { use super::*; + #[test] + fn a_stale_exporter_is_named_when_the_export_carries_the_dump() { + let p = Path::new("C:/x/igneum-prove-export.exe"); + let old_out = "Error: segment 140661: port state root 0xe45c differs from the node's 0xddd7; the port is not the node's executor, stop\n"; + let line = exporter_failure(true, old_out, p, Some(140662)); + assert!(line.contains("stale igneum-prove-export at C:/x/igneum-prove-export.exe"), "{line}"); + assert!(line.starts_with("exporter, block 140662:")); + // the 0.3.14 exporter read the dump and failed later: the real line, not the stale one + let new_out = "account dump: 79 accounts after chain block 140661, state root 0x1 (equals the node's)\nError: segment 140662: port state root 0x2 differs from the node's 0x3\n"; + let line = exporter_failure(true, new_out, p, None); + assert_eq!(line, "exporter: Error: segment 140662: port state root 0x2 differs from the node's 0x3"); + // an export without a dump (an older node): never the stale line + assert_eq!(exporter_failure(false, old_out, p, None), format!("exporter: {}", old_out.trim())); + } + #[test] fn pinned_ids_come_out_of_the_hosts_id_line() { let out = "RESULT id: pinned guests: shard program id 0x2b1a81cb413236cf063077b46ed3111628f6c41036bcf6e23ee4cbbf5679ef7a (2832504 bytes, sha256 0x150f4c05a2951fc5) aggregator id 0x474678f35f7545db28055d5e5bbc308231d84a5a072202087a2a8d5b09123896 (319744 bytes), pinned 2026-10-05T16:20:38Z on Darwin, SP1 5.0 circuit v5\n"; diff --git a/docs/bench-log.md b/docs/bench-log.md index de7874b45..8b81238a2 100644 --- a/docs/bench-log.md +++ b/docs/bench-log.md @@ -2544,3 +2544,23 @@ task now runs ...Programs\Igneum Miner\igneum-app.exe --power-helper"), then the task with no prompt ("-pl 460: set to 460.00 W from 575.00 W", "-pl 160: set to 160.00 W from 100.00 W"). Task Running, Highest, user Admin. Cards: 5090 221 W at 1845 MHz, 4070 75.8 W at 1860 MHz, 9070 XT 202 W, all mining through the installed app. Ember closed 16:53Z. +### 6 October 2026, 17:55Z to 17:57Z, the first segment above the restart cut from a snapshot-restored node on the new export (Mac) + +A copy of node 1's data dir started on the shipper's 0.3.14 Mac binary (target-0314, release-0.3.14-node 4c6b129d) with +`--igneum-exec-snapshot=node1-copy-snapshot.bin,0xac101f13...` (the hands' recovery file, tip 130,272) and node 1 as its +peer: IBD of 8,049 blocks in 29 s, the executor resumed from the file and reached the tip 140,023 with no restart replay. +`igneum_exportSegments [139951, 139959]` answered in under 0.1 s with 9 segments, `preState` 79 accounts, `execRestart` +27,276. The branch exporter (exec-app-0314 bfebff1) cut all 8 blocks of segment 139,952..139,959 in under 0.1 s each: +"replayed 8 segments from the account dump; every state root equals the node's". The first block's fixture carries +pre-state root 0x64304441... and post-state root 0x60d60bba..., equal to node 1's eth_getBlockByNumber roots at 139,951 +and 139,952 (an independent node). Consequence: a prover on a snapshot-restored node (every hand, nine fleet boxes) cuts a +provable fixture again with the 0.3.14 app and exporter; what is left for a paid record is the proof itself (the PCs or +the fleet, not this Mac). + +### 6 October 2026, 18:11Z to 18:13Z, the PC shape over p2p (Mac, `tools/lock/with-lock.sh run`) + +Fork 1255c0f9. `node tools/exec-sync/net.mjs`: 18/18 in 117.5 s. Case 7: a node whose data dir holds the tip-0 pair the 0.3.13 +genesis replay left behind, with `--igneum-exec-snapshot=peer`, sets the pair aside (.bin.tip0), asks A over p2p and loads +A's state with no hand action; its state root at 60 equals A's. `reorg.mjs` 18/18 in 262.1 s on the same binary; the exec +suite 20. Consequence: a PC or a box left with the tip-0 pair recovers on its own once a hand serves (the 5-minute +no-progress rule fires the ask even when the error text is new); until 0.3.14.1 the recovery file by hand is the way. diff --git a/infra/fast-time/override-60x.json b/infra/fast-time/override-60x.json index f232111df..cd6cd7994 100644 --- a/infra/fast-time/override-60x.json +++ b/infra/fast-time/override-60x.json @@ -61,6 +61,7 @@ "exec_restart_number": 18446744073709551615, "exec_restart_hash": "", "exec_restart_trust_daa": 18446744073709551615, + "exec_restart_state_root": "", "pow_genesis_dataset_log2": 28, "proving_v1_activation_daa": 18446744073709551615, "proving_v1_segment_blocks": 8, diff --git a/tools/exec-sync/net.mjs b/tools/exec-sync/net.mjs index 70592d6c4..25d09601a 100644 --- a/tools/exec-sync/net.mjs +++ b/tools/exec-sync/net.mjs @@ -13,10 +13,11 @@ // "unknown to consensus" (the file's tip is from another chain: a second simnet) // 5. known-failed: a node E whose flag names a file that cannot be read blocks loudly (0.3.13.1), never genesis // 6. the account dump (0.3.14): export [tip-1, tip] from B carries preState; the exporter seeds from it and cuts +// 7. the PC shape (0.3.14.1): G with the tip-0 pair and --igneum-exec-snapshot=peer recovers from A over p2p // Usage: node tools/exec-sync/net.mjs [--blocks 60] // IGNEUM_EXEC_BIN= (default vendor/igneum-node/target-exec-sync/release) import { spawn, spawnSync } from 'node:child_process'; -import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, openSync } from 'node:fs'; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, openSync, copyFileSync } from 'node:fs'; import { createHash } from 'node:crypto'; const ROOT = new URL('../../', import.meta.url).pathname; @@ -81,10 +82,14 @@ function sha256(path) { return '0x' + createHash('sha256').update(readFileSync(p async function main() { const a = await new Node(0).start(); log(`node A up on ${a.evm}`); + const tip0File = `${TMP}/tip0.bin`; const miner = mine(a, 'exec-sync-a'); await waitTip(a, BLOCKS, 900); const tipA = hexn(await a.eth('eth_blockNumber')); const st0 = await a.eth('igneum_getExecStatus'); + // a tip-0 snapshot (the genesis replay's leftover shape) for case 7: the state after chain block 0, from A's ring + const t0x = await a.eth('igneum_exportExecSnapshot', [tip0File, '0x0']); + if (!existsSync(tip0File) || hexn(t0x.tip) !== 0) throw new Error('no tip-0 export'); check('A executes from genesis', st0.startedFrom === 'genesis' && st0.blocked === null && hexn(st0.executedTip) >= BLOCKS, st0); // case 1: the export, the stop, the resume const file = `${TMP}/snapshot.bin`; @@ -151,6 +156,26 @@ async function main() { const out = (r.stdout || '') + (r.stderr || ''); check('the exporter seeds from the dump and cuts the block (state root equals the node\'s)', r.status === 0 && existsSync(fix) && /account dump: \d+ accounts after chain block/.test(out) && !/differs from the node/.test(out), { status: r.status, line: (out.split('\n').find(l => /account dump/.test(l)) || out.slice(-200)).slice(0, 200) }); } + // 7. the PC shape (0.3.14.1): a node G whose data dir holds the tip-0 pair the 0.3.13 genesis replay left behind, + // with --igneum-exec-snapshot=peer: G refuses the pair (set aside as .tip0), blocks, asks A over p2p and loads + // A's state with no hand action; then it follows the chain. (A pruning point above the anchor is not reproducible + // on a simnet in minutes; the pruned-node branch of the same classifier is unit-tested in service.rs.) + { + const g = new Node(6, [`127.0.0.1:${a.p2pPort}`], ['--igneum-exec-snapshot=peer']); + const evmDir = `${g.dir}/igneum-devnet-${SUFFIX}/datadir/evm`; + mkdirSync(evmDir, { recursive: true }); + copyFileSync(tip0File, `${evmDir}/exec-snapshot.bin`); copyFileSync(tip0File, `${evmDir}/exec-snapshot.prev.bin`); + await g.start(); + let stG = null; + for (let k = 0; k < 240; k++) { stG = await g.eth('igneum_getExecStatus'); if (stG.startedFrom.startsWith('peer') && hexn(stG.executedTip) > 0) break; await sleep(500); } + const gline = g.logText().split('\n').find(l => /tip 0 \(a genesis state\) is never a resume/.test(l)); + check('G refused the tip-0 pair in its data dir and set it aside', !!gline && existsSync(`${evmDir}/exec-snapshot.bin.tip0`), { line: gline && gline.slice(-140) }); + check('G recovered over p2p from A with no hand action and follows the chain', stG.startedFrom.startsWith('peer') && hexn(stG.executedTip) > 0 && stG.blocked === null, { startedFrom: stG.startedFrom, tip: stG.executedTip }); + const h = hexn(stG.executedTip); + const ra = await a.eth('eth_getBlockByNumber', ['0x' + h.toString(16), false]), rg = await g.eth('eth_getBlockByNumber', ['0x' + h.toString(16), false]); + check('G\'s state root at its tip equals A\'s', ra.stateRoot === rg.stateRoot, { height: h }); + await g.stop(); + } // 5. known-failed: a node E whose flag names a file that does not exist (the seed, 6 October 2026: unreadable under // /root) blocks loudly, never runs as if the flag were unset, never executes genesis const e = await new Node(5, [`127.0.0.1:${a.p2pPort}`], ['--igneum-exec-snapshot=/nonexistent/exec-snapshot.bin,0x00']).start();