From 825bf970e08f570bd80fdb609b1983ddd7349212 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:54:52 +0000 Subject: [PATCH 1/4] infra/fast-time/override-60x.json: the four exec restart fields at their never values (the devnet-profile test wants every override field present; the 0.3.14 node added them) Measured: IGNEUM_FAST_TIME_FILE= cargo test -p kaspa-consensus-core --lib fast_time_60x: 1 passed (fork 1f59c5d0, 6 October 2026 17:54Z). Co-Authored-By: Claude Fable 5.1 --- infra/fast-time/override-60x.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/infra/fast-time/override-60x.json b/infra/fast-time/override-60x.json index 81651706b..e68716c23 100644 --- a/infra/fast-time/override-60x.json +++ b/infra/fast-time/override-60x.json @@ -62,5 +62,9 @@ "proving_v1_aggregator_share_bps": 1000, "fees_v1_activation_daa": 0, "proving_v1_fresh_rule_daa": 0, + "exec_restart_number": 18446744073709551615, + "exec_restart_hash": "", + "exec_restart_trust_daa": 18446744073709551615, + "exec_restart_state_root": "", "fees": {"pgas": {"version": 1, "cycles_per_pgas": 1000, "intrinsic_pgas_per_tx": 300, "modexp_base": 10, "modexp_per_byte_numer": 1, "modexp_per_byte_denom": 10}, "block_proving_gas_limit": 120000, "shard_proving_gas_budget": 30000, "min_execution_base_fee_wei": 100000000000, "min_proving_base_fee_wei": 10000000000000, "initial_execution_base_fee_wei": 100000000000, "initial_proving_base_fee_wei": 10000000000000, "base_fee_change_denominator": 8} } From d53b60b32b4d4dc939789ba0d51c8ac1db412504 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:57:54 +0000 Subject: [PATCH 2/4] bench log: the first segment above the restart cut from a snapshot-restored node on the new export (8 blocks, roots equal node 1) Co-Authored-By: Claude Fable 5.1 --- docs/bench-log.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/bench-log.md b/docs/bench-log.md index 129f7b8dc..3a5496f7b 100644 --- a/docs/bench-log.md +++ b/docs/bench-log.md @@ -2271,3 +2271,16 @@ a 50 blocks/s flood from one peer, honest template p50/p95/max 0.4/0.6/1.4 ms, r same once; a pool user nothing; a fleet operator gets a node that keeps its only peer, and a seed that stops amplifying old certificates to every peer (13,354 lines of work it did not need in seven minutes). Owed: the fleet agent's synced-node reading; a receiver-side limit on certificates per index per minute as a second belt once the seed is fixed; the formatter's reflow of `finality.rs` (taken out of the commit). + +### 6 October 2026, 17:55Z to 17:57Z, the first segment above the restart cut from a snapshot-restored node on the new export (Mac) + +A copy of node 1's data dir started on the shipper's 0.3.14 Mac binary (target-0314, release-0.3.14-node 4c6b129d) with +`--igneum-exec-snapshot=node1-copy-snapshot.bin,0xac101f13...` (the hands' recovery file, tip 130,272) and node 1 as its +peer: IBD of 8,049 blocks in 29 s, the executor resumed from the file and reached the tip 140,023 with no restart replay. +`igneum_exportSegments [139951, 139959]` answered in under 0.1 s with 9 segments, `preState` 79 accounts, `execRestart` +27,276. The branch exporter (exec-app-0314 bfebff1) cut all 8 blocks of segment 139,952..139,959 in under 0.1 s each: +"replayed 8 segments from the account dump; every state root equals the node's". The first block's fixture carries +pre-state root 0x64304441... and post-state root 0x60d60bba..., equal to node 1's eth_getBlockByNumber roots at 139,951 +and 139,952 (an independent node). Consequence: a prover on a snapshot-restored node (every hand, nine fleet boxes) cuts a +provable fixture again with the 0.3.14 app and exporter; what is left for a paid record is the proof itself (the PCs or +the fleet, not this Mac). From 0e3180340f32b2b918d01ecf053f01f6dec5636d Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:13:44 +0000 Subject: [PATCH 3/4] exec-sync harness case 7: the PC shape (tip-0 pair, peer flag) recovers over p2p; bench-log line Co-Authored-By: Claude Fable 5.1 --- docs/bench-log.md | 8 ++++++++ tools/exec-sync/net.mjs | 27 ++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/bench-log.md b/docs/bench-log.md index 3a5496f7b..630b3d2dd 100644 --- a/docs/bench-log.md +++ b/docs/bench-log.md @@ -2284,3 +2284,11 @@ pre-state root 0x64304441... and post-state root 0x60d60bba..., equal to node 1' and 139,952 (an independent node). Consequence: a prover on a snapshot-restored node (every hand, nine fleet boxes) cuts a provable fixture again with the 0.3.14 app and exporter; what is left for a paid record is the proof itself (the PCs or the fleet, not this Mac). + +### 6 October 2026, 18:11Z to 18:13Z, the PC shape over p2p (Mac, `tools/lock/with-lock.sh run`) + +Fork 1255c0f9. `node tools/exec-sync/net.mjs`: 18/18 in 117.5 s. Case 7: a node whose data dir holds the tip-0 pair the 0.3.13 +genesis replay left behind, with `--igneum-exec-snapshot=peer`, sets the pair aside (.bin.tip0), asks A over p2p and loads +A's state with no hand action; its state root at 60 equals A's. `reorg.mjs` 18/18 in 262.1 s on the same binary; the exec +suite 20. Consequence: a PC or a box left with the tip-0 pair recovers on its own once a hand serves (the 5-minute +no-progress rule fires the ask even when the error text is new); until 0.3.14.1 the recovery file by hand is the way. diff --git a/tools/exec-sync/net.mjs b/tools/exec-sync/net.mjs index 70592d6c4..25d09601a 100644 --- a/tools/exec-sync/net.mjs +++ b/tools/exec-sync/net.mjs @@ -13,10 +13,11 @@ // "unknown to consensus" (the file's tip is from another chain: a second simnet) // 5. known-failed: a node E whose flag names a file that cannot be read blocks loudly (0.3.13.1), never genesis // 6. the account dump (0.3.14): export [tip-1, tip] from B carries preState; the exporter seeds from it and cuts +// 7. the PC shape (0.3.14.1): G with the tip-0 pair and --igneum-exec-snapshot=peer recovers from A over p2p // Usage: node tools/exec-sync/net.mjs [--blocks 60] // IGNEUM_EXEC_BIN= (default vendor/igneum-node/target-exec-sync/release) import { spawn, spawnSync } from 'node:child_process'; -import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, openSync } from 'node:fs'; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync, openSync, copyFileSync } from 'node:fs'; import { createHash } from 'node:crypto'; const ROOT = new URL('../../', import.meta.url).pathname; @@ -81,10 +82,14 @@ function sha256(path) { return '0x' + createHash('sha256').update(readFileSync(p async function main() { const a = await new Node(0).start(); log(`node A up on ${a.evm}`); + const tip0File = `${TMP}/tip0.bin`; const miner = mine(a, 'exec-sync-a'); await waitTip(a, BLOCKS, 900); const tipA = hexn(await a.eth('eth_blockNumber')); const st0 = await a.eth('igneum_getExecStatus'); + // a tip-0 snapshot (the genesis replay's leftover shape) for case 7: the state after chain block 0, from A's ring + const t0x = await a.eth('igneum_exportExecSnapshot', [tip0File, '0x0']); + if (!existsSync(tip0File) || hexn(t0x.tip) !== 0) throw new Error('no tip-0 export'); check('A executes from genesis', st0.startedFrom === 'genesis' && st0.blocked === null && hexn(st0.executedTip) >= BLOCKS, st0); // case 1: the export, the stop, the resume const file = `${TMP}/snapshot.bin`; @@ -151,6 +156,26 @@ async function main() { const out = (r.stdout || '') + (r.stderr || ''); check('the exporter seeds from the dump and cuts the block (state root equals the node\'s)', r.status === 0 && existsSync(fix) && /account dump: \d+ accounts after chain block/.test(out) && !/differs from the node/.test(out), { status: r.status, line: (out.split('\n').find(l => /account dump/.test(l)) || out.slice(-200)).slice(0, 200) }); } + // 7. the PC shape (0.3.14.1): a node G whose data dir holds the tip-0 pair the 0.3.13 genesis replay left behind, + // with --igneum-exec-snapshot=peer: G refuses the pair (set aside as .tip0), blocks, asks A over p2p and loads + // A's state with no hand action; then it follows the chain. (A pruning point above the anchor is not reproducible + // on a simnet in minutes; the pruned-node branch of the same classifier is unit-tested in service.rs.) + { + const g = new Node(6, [`127.0.0.1:${a.p2pPort}`], ['--igneum-exec-snapshot=peer']); + const evmDir = `${g.dir}/igneum-devnet-${SUFFIX}/datadir/evm`; + mkdirSync(evmDir, { recursive: true }); + copyFileSync(tip0File, `${evmDir}/exec-snapshot.bin`); copyFileSync(tip0File, `${evmDir}/exec-snapshot.prev.bin`); + await g.start(); + let stG = null; + for (let k = 0; k < 240; k++) { stG = await g.eth('igneum_getExecStatus'); if (stG.startedFrom.startsWith('peer') && hexn(stG.executedTip) > 0) break; await sleep(500); } + const gline = g.logText().split('\n').find(l => /tip 0 \(a genesis state\) is never a resume/.test(l)); + check('G refused the tip-0 pair in its data dir and set it aside', !!gline && existsSync(`${evmDir}/exec-snapshot.bin.tip0`), { line: gline && gline.slice(-140) }); + check('G recovered over p2p from A with no hand action and follows the chain', stG.startedFrom.startsWith('peer') && hexn(stG.executedTip) > 0 && stG.blocked === null, { startedFrom: stG.startedFrom, tip: stG.executedTip }); + const h = hexn(stG.executedTip); + const ra = await a.eth('eth_getBlockByNumber', ['0x' + h.toString(16), false]), rg = await g.eth('eth_getBlockByNumber', ['0x' + h.toString(16), false]); + check('G\'s state root at its tip equals A\'s', ra.stateRoot === rg.stateRoot, { height: h }); + await g.stop(); + } // 5. known-failed: a node E whose flag names a file that does not exist (the seed, 6 October 2026: unreadable under // /root) blocks loudly, never runs as if the flag were unset, never executes genesis const e = await new Node(5, [`127.0.0.1:${a.p2pPort}`], ['--igneum-exec-snapshot=/nonexistent/exec-snapshot.bin,0x00']).start(); From 54049463b191277e31effb825de63e59ff87b21c Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:24:40 +0000 Subject: [PATCH 4/4] prover: a stale igneum-prove-export is named when the node's export carries the account dump and the exporter never reads it PC 1 on 0.3.14 (6 October 2026, 18:16Z, block 140,662): "segment 140661: port state root 0xe45c... differs from the node's 0xddd7...". The export was [140661, 140662] with the dump; the 0.3.14 exporter seeds from segment 140,661 and never replays it, so the only exporter that replays 140,661 (from the restart state, hence the mismatch) is the one from before 0.3.14: the installed binary was not replaced, the same class as the stale verifier host. The prover's failure line now names the stale exporter by path when the export carries preState and the output has no "account dump" line (exporter_failure, unit-tested with the PC 1 text); the real line stays when the exporter did read the dump. Co-Authored-By: Claude Fable 5.1 --- app/igneum-app/src/prover.rs | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/app/igneum-app/src/prover.rs b/app/igneum-app/src/prover.rs index a1ecc6e6f..1d2140bbc 100644 --- a/app/igneum-app/src/prover.rs +++ b/app/igneum-app/src/prover.rs @@ -135,6 +135,22 @@ struct Tools { /// The node's re-derivation boundary (0.3.14, 6 October 2026): the chain block its EVM restarted at after the /// bodies below the pruning point were gone (`igneum_getExecStatus.restartNumber`; on a 0.3.13 node the /// `startedFrom` text "restart at chain block N"), else 0. The prover never claims work below it (no bodies, no +/// The exporter's failure line. When the node's export carries the account dump (0.3.14) and the exporter's output +/// never mentions it, the exporter at `path` predates 0.3.14: it replays the dump's own segment from the restart +/// state and fails there with "port state root differs" (PC 1, 6 October 2026 18:16Z, block 140,662: the installer +/// had not replaced igneum-prove-export); the line names the stale binary instead of the misleading root error. +fn exporter_failure(export_has_dump: bool, out: &str, path: &Path, block: Option) -> String { + let last = out.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("failed"); + let what = match block { + Some(b) => format!("exporter, block {b}"), + None => "exporter".to_string(), + }; + if export_has_dump && !out.contains("account dump") { + return format!("{what}: stale igneum-prove-export at {}: the node's export carries the account dump and this exporter does not read it (it predates 0.3.14); install this release's exporter there ({last})", path.display()); + } + format!("{what}: {last}") +} + /// state: unprovable on every node). fn exec_boundary(shared: &Shared) -> u64 { let st = match evm_rpc(shared, "igneum_getExecStatus", json!([]), Duration::from_secs(5)) { @@ -665,7 +681,7 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { let (seq_p, fix_p, res_p) = if t.wsl { (wsl_path(&seq), wsl_path(&fixture), wsl_path(&results)) } else { (seq.display().to_string(), fixture.display().to_string(), results.display().to_string()) }; let (ok, out) = run_tool(&shared, t, &t.export, &[seq_p, w.number.to_string(), fix_p.clone()], &[], Duration::from_secs(600), &dir.join(format!("export-{}.log", w.number))); if !ok || !fixture.exists() { - return Err(format!("exporter: {}", out.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("failed"))); + return Err(exporter_failure(export["preState"].is_array(), &out, &t.export, None)); } set(&shared, |p| p.message = if t.cuda { "proving on the GPU".into() } else { "CPU prover: about five minutes a shard, 30 GB of RAM, paid only when no card proves first".into() }); let prover_env = if t.cuda { "cuda" } else { "cpu" }; @@ -854,7 +870,7 @@ fn prove_segment(shared: &Shared, t: &Tools, seg: &crate::segments::SegmentWork, let fixture = dir.join(format!("block-{b}.json")); let (ok, out) = run_tool(shared, t, &t.export, &[as_host_path(&seq), b.to_string(), as_host_path(&fixture)], &[], Duration::from_secs(600), &dir.join(format!("export-{b}.log"))); if !ok || !fixture.exists() { - return Err(format!("exporter, block {b}: {}", out.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("failed"))); + return Err(exporter_failure(export["preState"].is_array(), &out, &t.export, Some(b))); } fixtures.push(as_host_path(&fixture)); } @@ -1092,6 +1108,21 @@ pub fn setup(shared: &Shared) -> Result { mod tests { use super::*; + #[test] + fn a_stale_exporter_is_named_when_the_export_carries_the_dump() { + let p = Path::new("C:/x/igneum-prove-export.exe"); + let old_out = "Error: segment 140661: port state root 0xe45c differs from the node's 0xddd7; the port is not the node's executor, stop\n"; + let line = exporter_failure(true, old_out, p, Some(140662)); + assert!(line.contains("stale igneum-prove-export at C:/x/igneum-prove-export.exe"), "{line}"); + assert!(line.starts_with("exporter, block 140662:")); + // the 0.3.14 exporter read the dump and failed later: the real line, not the stale one + let new_out = "account dump: 79 accounts after chain block 140661, state root 0x1 (equals the node's)\nError: segment 140662: port state root 0x2 differs from the node's 0x3\n"; + let line = exporter_failure(true, new_out, p, None); + assert_eq!(line, "exporter: Error: segment 140662: port state root 0x2 differs from the node's 0x3"); + // an export without a dump (an older node): never the stale line + assert_eq!(exporter_failure(false, old_out, p, None), format!("exporter: {}", old_out.trim())); + } + #[test] fn pinned_ids_come_out_of_the_hosts_id_line() { let out = "RESULT id: pinned guests: shard program id 0x2b1a81cb413236cf063077b46ed3111628f6c41036bcf6e23ee4cbbf5679ef7a (2832504 bytes, sha256 0x150f4c05a2951fc5) aggregator id 0x474678f35f7545db28055d5e5bbc308231d84a5a072202087a2a8d5b09123896 (319744 bytes), pinned 2026-10-05T16:20:38Z on Darwin, SP1 5.0 circuit v5\n";