diff --git a/proving/igneum-prove/Cargo.lock b/proving/igneum-prove/Cargo.lock index 5d35d64f1..ad4982a84 100644 --- a/proving/igneum-prove/Cargo.lock +++ b/proving/igneum-prove/Cargo.lock @@ -2762,6 +2762,16 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "igneum-prove-aggregator" +version = "0.1.0" +dependencies = [ + "bincode", + "igneum-prove-core", + "sha2 0.10.9", + "sp1-zkvm", +] + [[package]] name = "igneum-prove-core" version = "0.1.0" @@ -2782,6 +2792,7 @@ version = "0.1.0" dependencies = [ "alloy-primitives", "anyhow", + "bincode", "hex", "igneum-prove-core", "serde_json", @@ -2792,13 +2803,17 @@ name = "igneum-prove-host" version = "0.1.0" dependencies = [ "alloy-primitives", + "alloy-rlp", + "alloy-trie", "anyhow", "bincode", "hex", + "igneum-evm-types", "igneum-prove-core", "serde_json", "sp1-build", "sp1-sdk", + "tokio", ] [[package]] @@ -6274,6 +6289,7 @@ dependencies = [ "libm", "rand 0.8.8", "sha2 0.10.9", + "slop-algebra", "sp1-lib", "sp1-primitives", ] diff --git a/proving/igneum-prove/Cargo.toml b/proving/igneum-prove/Cargo.toml index 261704b2b..71b460efa 100644 --- a/proving/igneum-prove/Cargo.toml +++ b/proving/igneum-prove/Cargo.toml @@ -1,9 +1,9 @@ -# Igneum proving v0: an SP1 program that re-executes one Igneum chain block and a host that proves it. +# Igneum proving, devnet v4: SP1 programs that prove one shard of an Igneum chain block and aggregate the shards, and a host that runs them. # Crate versions are pinned to what the execution layer (vendor/igneum-node-exec, branch execution-layer, -# commit fb33069) and SP1 v6.8.1 (24 Sep 2026) use. Change one, re-run the export and the native check. +# commit b7fca5a0, merged into devnet-v4) and SP1 v6.8.1 (24 Sep 2026) use. Change one, re-run the export and the native check. [workspace] resolver = "2" -members = ["core", "program", "host", "export"] +members = ["core", "program", "aggregator", "host", "export"] [workspace.package] version = "0.1.0" diff --git a/proving/igneum-prove/core/src/agg.rs b/proving/igneum-prove/core/src/agg.rs new file mode 100644 index 000000000..8f8277880 --- /dev/null +++ b/proving/igneum-prove/core/src/agg.rs @@ -0,0 +1,191 @@ +//! The block (segment) statement by recursion (design 5.3): every shard proof of the segment verified in +//! order, chained on roots, links and transaction commitments, with the provers' payout addresses carried out +//! (ledger P12); optionally the previous segment's block proof verified too, so one proof attests the chain. + +use crate::executor::Carry; +use crate::shard::ShardOutput; +use alloy_primitives::{keccak256, B256}; +use serde::{Deserialize, Serialize}; + +/// A verifying key hash as the guest receives it (SP1's `hash_u32`) and as the public values carry it. +pub fn vk_bytes(vk: &[u32; 8]) -> B256 { + let mut b = [0u8; 32]; + for (i, w) in vk.iter().enumerate() { + b[i * 4..i * 4 + 4].copy_from_slice(&w.to_be_bytes()); + } + B256::from(b) +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PrevLink { + pub agg_vk: [u32; 8], + pub public_values: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct AggInput { + pub shard_vk: [u32; 8], + /// The shard proofs' public values, shard order. + pub shards: Vec>, + pub parent_hash: B256, + pub prev: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BlockOutput { + pub chain_id: u64, + pub number: u64, + pub block_hash: B256, + pub parent_hash: B256, + pub shard_count: u32, + pub tx_commitment: B256, + pub pre_root: B256, + pub post_root: B256, + /// keccak over the shards' receipts roots in order. + pub receipts: B256, + pub gas_used: u64, + pub pgas_used: u64, + pub executed: u32, + pub skipped: u32, + /// keccak over the shards' prover payout addresses in order (the record's `provers` list must match). + pub provers: B256, + pub shard_vk: B256, + /// The aggregator's own key when a previous block proof was verified, else zero. + pub agg_vk: B256, + /// Blocks attested by this proof: 1, or the previous proof's count plus one. + pub chain_len: u64, +} + +impl BlockOutput { + pub const LEN: usize = 8 + 8 + 32 + 32 + 4 + 32 * 4 + 8 + 8 + 4 + 4 + 32 * 3 + 8; + + pub fn to_bytes(&self) -> Vec { + let mut v = Vec::with_capacity(Self::LEN); + v.extend_from_slice(&self.chain_id.to_be_bytes()); + v.extend_from_slice(&self.number.to_be_bytes()); + v.extend_from_slice(self.block_hash.as_slice()); + v.extend_from_slice(self.parent_hash.as_slice()); + v.extend_from_slice(&self.shard_count.to_be_bytes()); + for w in [&self.tx_commitment, &self.pre_root, &self.post_root, &self.receipts] { + v.extend_from_slice(w.as_slice()); + } + v.extend_from_slice(&self.gas_used.to_be_bytes()); + v.extend_from_slice(&self.pgas_used.to_be_bytes()); + v.extend_from_slice(&self.executed.to_be_bytes()); + v.extend_from_slice(&self.skipped.to_be_bytes()); + for w in [&self.provers, &self.shard_vk, &self.agg_vk] { + v.extend_from_slice(w.as_slice()); + } + v.extend_from_slice(&self.chain_len.to_be_bytes()); + debug_assert_eq!(v.len(), Self::LEN); + v + } + + pub fn from_bytes(b: &[u8]) -> Option { + if b.len() != Self::LEN { + return None; + } + let u64_at = |i: usize| u64::from_be_bytes(b[i..i + 8].try_into().unwrap()); + let u32_at = |i: usize| u32::from_be_bytes(b[i..i + 4].try_into().unwrap()); + let b256_at = |i: usize| B256::from_slice(&b[i..i + 32]); + Some(Self { + chain_id: u64_at(0), + number: u64_at(8), + block_hash: b256_at(16), + parent_hash: b256_at(48), + shard_count: u32_at(80), + tx_commitment: b256_at(84), + pre_root: b256_at(116), + post_root: b256_at(148), + receipts: b256_at(180), + gas_used: u64_at(212), + pgas_used: u64_at(220), + executed: u32_at(228), + skipped: u32_at(232), + provers: b256_at(236), + shard_vk: b256_at(268), + agg_vk: b256_at(300), + chain_len: u64_at(332), + }) + } +} + +/// The aggregation statement. `verify(vk, public_values)` is the zkVM's deferred-proof verification in the +/// guest and a no-op or a real verification on the host; everything else is checked here and panics on any +/// inconsistency, which makes the proof impossible. +pub fn aggregate(input: &AggInput, verify: &mut dyn FnMut(&[u32; 8], &[u8])) -> BlockOutput { + assert!(!input.shards.is_empty(), "a block has at least one shard"); + let mut receipts = Vec::with_capacity(32 * input.shards.len()); + let mut provers = Vec::with_capacity(20 * input.shards.len()); + let mut first: Option = None; + let mut last: Option = None; + let (mut gas, mut pgas, mut executed, mut skipped) = (0u64, 0u64, 0u32, 0u32); + for (i, pv) in input.shards.iter().enumerate() { + verify(&input.shard_vk, pv); + let s = ShardOutput::from_bytes(pv).expect("shard public values decode"); + assert_eq!(s.shard_index as usize, i, "shard index"); + match &last { + None => { + assert_eq!(s.tx_start, 0, "the first shard starts at the first transaction"); + assert_eq!(s.link_in, Carry::default().link(), "the first shard starts from the empty carry"); + assert_eq!(s.tx_acc_in, B256::ZERO); + } + Some(p) => { + assert_eq!(s.chain_id, p.chain_id); + assert_eq!(s.number, p.number); + assert_eq!(s.block_hash, p.block_hash, "every shard is of the same segment"); + assert_eq!(s.tx_start, p.tx_start + p.tx_count, "shards are contiguous"); + assert_eq!(s.link_in, p.link_out, "shard {i} does not continue shard {}", i - 1); + assert_eq!(s.tx_acc_in, p.tx_acc_out); + assert_eq!(s.pre_root, p.post_root, "shard {i} does not start from shard {}'s post-root", i - 1); + } + } + receipts.extend_from_slice(s.receipts_root.as_slice()); + provers.extend_from_slice(s.prover.as_slice()); + gas += s.gas_used; + pgas += s.pgas_used; + executed += s.executed; + skipped += s.skipped; + if first.is_none() { + first = Some(s.clone()); + } + last = Some(s); + } + let first = first.unwrap(); + let last = last.unwrap(); + let shard_vk = vk_bytes(&input.shard_vk); + let (agg_vk, chain_len) = match &input.prev { + None => (B256::ZERO, 1u64), + Some(prev) => { + verify(&prev.agg_vk, &prev.public_values); + let p = BlockOutput::from_bytes(&prev.public_values).expect("previous block public values decode"); + let agg_vk = vk_bytes(&prev.agg_vk); + assert_eq!(p.chain_id, first.chain_id); + assert_eq!(p.number + 1, first.number, "the previous proof is of the parent segment"); + assert_eq!(p.block_hash, input.parent_hash, "the previous proof is of the parent block"); + assert_eq!(p.post_root, first.pre_root, "the chain of state continues"); + assert_eq!(p.shard_vk, shard_vk, "the same shard program"); + assert!(p.agg_vk == agg_vk || (p.chain_len == 1 && p.agg_vk == B256::ZERO), "the same aggregator program"); + (agg_vk, p.chain_len + 1) + } + }; + BlockOutput { + chain_id: first.chain_id, + number: first.number, + block_hash: first.block_hash, + parent_hash: input.parent_hash, + shard_count: input.shards.len() as u32, + tx_commitment: last.tx_acc_out, + pre_root: first.pre_root, + post_root: last.post_root, + receipts: keccak256(receipts), + gas_used: gas, + pgas_used: pgas, + executed, + skipped, + provers: keccak256(provers), + shard_vk, + agg_vk, + chain_len, + } +} diff --git a/proving/igneum-prove/core/src/config.rs b/proving/igneum-prove/core/src/config.rs index 1a4669ff2..74cd54603 100644 --- a/proving/igneum-prove/core/src/config.rs +++ b/proving/igneum-prove/core/src/config.rs @@ -1,9 +1,16 @@ -//! Consensus constants of the execution layer, devnet v3 values (mirror of igneum-exec `config.rs` at fb33069). +//! Consensus constants of the execution layer, devnet v4 values (mirror of igneum-exec `config.rs` at b7fca5a0), +//! plus the shard budget of design 5.1. use alloy_primitives::{address, Address}; pub const BLOCK_EXECUTION_GAS_LIMIT: u64 = 30_000_000; pub const BLOCK_PROVING_GAS_LIMIT: u64 = 30_000_000; +/// `S_p`, the most proving gas one shard carries (design 5.1, spec 7.2). The specification gives no number yet +/// (Target: about 20 s on a 12 GB card; the provisional value is "the project's own first run", benchmark +/// standard 9). Provisional, 4 October 2026: a quarter of `B_p`, so a block at its proving budget is exactly +/// four shards and the devnet v4 fixtures are one, two and four shards. Set from the RTX 5090 and 3060-class +/// measurements, never from the fixtures (benchmark standard 2.3). +pub const SHARD_PROVING_GAS_BUDGET: u64 = BLOCK_PROVING_GAS_LIMIT / 4; pub const DEVELOPER_REGISTRY_ADDRESS: Address = address!("0000000000000000000000000000000000000210"); pub const PROVING_POOL_ADDRESS: Address = address!("0000000000000000000000000000000000000220"); pub const BLOCKHASH_WINDOW: u64 = 256; diff --git a/proving/igneum-prove/core/src/executor.rs b/proving/igneum-prove/core/src/executor.rs index c86f45924..fa85334d1 100644 --- a/proving/igneum-prove/core/src/executor.rs +++ b/proving/igneum-prove/core/src/executor.rs @@ -1,15 +1,18 @@ -//! Segment execution, ported from igneum-exec `executor.rs` at fb33069 (design sections 1 to 4): rewards by -//! rule, every block's transactions in sequence order under the Igneum environment, two-dimensional gas, fee -//! flows and the nonce-rule skip; the state root and the receipts root as outputs. +//! Segment execution, ported from igneum-exec `executor.rs` at b7fca5a0 (design sections 1 to 4, spec 7.5): +//! rewards by rule, every block's transactions in sequence order under the Igneum environment, two-dimensional +//! gas with the incremental cap and the abort, fee flows and the nonce-rule skip. Devnet v4 addition: the +//! executor runs any contiguous range of a segment's transactions from a carried-in position (design 5.1), and +//! emits one boundary per transaction with the cumulative gas and pgas, the carry, and (natively) the state +//! root, which is what the shard planner cuts on. use crate::config::*; -use crate::fixture::{BlockFixture, FixtureEnv}; +use crate::fixture::FixtureEnv; use crate::pgas::{developer_shares, IgneumInspector}; use crate::registry::{address_from_word, payee_slot}; use crate::state::{DbRef, IgneumDb}; use alloy_consensus::{Receipt, ReceiptEnvelope, ReceiptWithBloom, Transaction, TxEnvelope}; use alloy_eips::eip2718::Encodable2718; -use alloy_primitives::{Address, Bloom, Log, B256, U256}; +use alloy_primitives::{keccak256, Address, Bloom, Bytes, Log, B256, U256}; use igneum_evm_types::DecodedTx; use revm::context::result::{EVMError, ExecutionResult, InvalidTransaction}; use revm::context::{BlockEnv, CfgEnv, Context, TxEnv}; @@ -18,6 +21,7 @@ use revm::inspector::InspectEvm; use revm::primitives::hardfork::SpecId; use revm::MainContext; use revm::{DatabaseCommit, ExecuteEvm}; +use serde::{Deserialize, Serialize}; fn cfg(chain_id: u64) -> CfgEnv { let mut cfg = CfgEnv::new_with_spec(SpecId::CANCUN); @@ -108,62 +112,138 @@ pub struct ExecutedReceipt { pub logs_bloom: Bloom, pub tx_type: u8, pub over_budget: bool, + pub pgas_aborted: bool, } -pub struct BlockOutcome { +/// One transaction of a segment in sequence order with its including block (index in the segment's block +/// list, its miner, whether it is blue). +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ShardTx { + pub block_index: u32, + pub miner: Address, + pub blue: bool, + pub raw: Bytes, +} + +/// The position the executor is at between two transactions of a segment: everything the next transaction's +/// outcome depends on besides the state. Shard i ends with the carry shard i+1 starts from; `link()` is the hash +/// the shard statements chain on. +#[derive(Clone, Copy, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct Carry { + /// Transactions of the segment consumed so far. + pub tx_index: u32, + pub block_index: u32, + pub block_gas: u64, + pub block_pgas: u64, + /// Cumulative execution gas of the executed transactions (the receipts' running total). + pub cumulative_gas: u64, + /// Running commitment to the transactions consumed: keccak(acc, miner, blue, len, raw) per transaction. + pub tx_acc: B256, +} + +impl Carry { + pub fn link(&self) -> B256 { + let mut b = Vec::with_capacity(96); + b.extend_from_slice(b"igneum-shard-link/v1"); + b.extend_from_slice(&self.tx_index.to_be_bytes()); + b.extend_from_slice(&self.block_index.to_be_bytes()); + b.extend_from_slice(&self.block_gas.to_be_bytes()); + b.extend_from_slice(&self.block_pgas.to_be_bytes()); + b.extend_from_slice(&self.cumulative_gas.to_be_bytes()); + b.extend_from_slice(self.tx_acc.as_slice()); + keccak256(b) + } + + fn absorb(&mut self, tx: &ShardTx) { + let mut b = Vec::with_capacity(60 + tx.raw.len()); + b.extend_from_slice(self.tx_acc.as_slice()); + b.extend_from_slice(tx.miner.as_slice()); + b.push(tx.blue as u8); + b.extend_from_slice(&(tx.raw.len() as u32).to_be_bytes()); + b.extend_from_slice(&tx.raw); + self.tx_acc = keccak256(b); + } +} + +/// The state of the run after one more transaction (design 5.1 trace). +#[derive(Clone, Debug)] +pub struct Boundary { + pub carry: Carry, + /// Cumulative over the range: executed gas and segment pgas (intrinsic pgas of skipped copies included). + pub gas: u64, + pub pgas: u64, + /// The state root after this transaction (native runs only; the guest never computes it per transaction). + pub state_root: Option, +} + +pub struct RangeOutcome { pub executed: Vec, pub skipped: Vec<(B256, Skip)>, pub gas_used: u64, pub pgas_used: u64, - pub state_root: B256, + pub carry_out: Carry, + pub boundaries: Vec, pub receipts_root: B256, } -/// Executes one chain block's segment on `db` (design 1.2): the rewards of the fixture, then every block's -/// transactions in sequence order, then the roots as outputs. `db` is the pre-state on entry and the post-state -/// on return. -pub fn execute_block(db: &mut IgneumDb, f: &BlockFixture) -> BlockOutcome { - let env = &f.env; +/// Executes a contiguous range of a segment's transactions on `db` from `carry_in` (design 1.2 and 5.1): the +/// rewards first when given (the segment's first shard), then every transaction in order. `db` is the state +/// before the range on entry and after it on return. With `roots`, the state root is computed at every boundary. +pub fn execute_range(db: &mut IgneumDb, chain_id: u64, env: &FixtureEnv, rewards: Option<(&[(Address, U256)], U256)>, txs: &[ShardTx], carry_in: Carry, roots: bool) -> RangeOutcome { let base_fee_exec = env.base_fee_exec as u128; let base_fee_proving = env.base_fee_proving as u128; - for (miner, wei) in &f.rewards { - db.add_balance(*miner, *wei); + if let Some((rewards, pool)) = rewards { + for (miner, wei) in rewards { + db.add_balance(*miner, *wei); + } + db.add_balance(PROVING_POOL_ADDRESS, pool); } - db.add_balance(PROVING_POOL_ADDRESS, f.proving_pool_credit); let mut executed: Vec = Vec::new(); let mut skipped: Vec<(B256, Skip)> = Vec::new(); - let mut cumulative_gas = 0u64; - let mut segment_pgas = 0u64; + let mut boundaries: Vec = Vec::with_capacity(txs.len()); + let mut carry = carry_in; + let mut range_gas = 0u64; + let mut range_pgas = 0u64; - for b in &f.blocks { - let mut block_pgas = 0u64; - let mut block_gas = 0u64; - for raw in &b.txs { - let tx = match igneum_evm_types::decode_and_check(raw, f.chain_id) { + for t in txs { + if t.block_index != carry.block_index { + carry.block_index = t.block_index; + carry.block_gas = 0; + carry.block_pgas = 0; + } + carry.absorb(t); + carry.tx_index += 1; + let raw = &t.raw; + 'tx: { + let tx = match igneum_evm_types::decode_and_check(raw, chain_id) { Ok(tx) => tx, Err(e) => { - skipped.push((alloy_primitives::keccak256(raw), Skip::Undecodable(e.to_string()))); - continue; + skipped.push((keccak256(raw), Skip::Undecodable(e.to_string()))); + break 'tx; } }; - block_pgas = block_pgas.saturating_add(INTRINSIC_PGAS_PER_TX); - segment_pgas = segment_pgas.saturating_add(INTRINSIC_PGAS_PER_TX); - if block_pgas > BLOCK_PROVING_GAS_LIMIT { + carry.block_pgas = carry.block_pgas.saturating_add(INTRINSIC_PGAS_PER_TX); + range_pgas = range_pgas.saturating_add(INTRINSIC_PGAS_PER_TX); + if carry.block_pgas > BLOCK_PROVING_GAS_LIMIT { + // Only the intrinsic pgas of the included copies can get here: one comparison, no execution. skipped.push((tx.hash, Skip::BlockProvingBudget)); - continue; + break 'tx; } - if block_gas.saturating_add(tx.gas_limit()) > BLOCK_EXECUTION_GAS_LIMIT { + if carry.block_gas.saturating_add(tx.gas_limit()) > BLOCK_EXECUTION_GAS_LIMIT { skipped.push((tx.hash, Skip::BlockExecutionBudget)); - continue; + break 'tx; } let price = effective_gas_price(&tx, base_fee_exec); let budget = tx.wei_budget(); - let inspector = IgneumInspector::new(tx.intrinsic_gas, price, base_fee_proving, budget); + // Spec 7.5 item 1: the transaction may meter at most the block's remaining proving budget; the + // inspector halts it the moment one more opcode or precompile would cross that. + let pgas_cap = BLOCK_PROVING_GAS_LIMIT - carry.block_pgas; + let inspector = IgneumInspector::new(tx.intrinsic_gas, price, base_fee_proving, budget, pgas_cap); let (result, state, insp) = { - let ctx = Context::mainnet().with_db(DbRef(db)).with_cfg(cfg(f.chain_id)).with_block(block_env(env, b.miner)); + let ctx = Context::mainnet().with_db(DbRef(db)).with_cfg(cfg(chain_id)).with_block(block_env(env, t.miner)); let mut evm = ctx.build_mainnet_with_inspector(inspector); let result = evm.inspect_one_tx(tx_env(&tx)); let state = evm.finalize(); @@ -174,17 +254,14 @@ pub fn execute_block(db: &mut IgneumDb, f: &BlockFixture) -> BlockOutcome { Ok(r) => r, Err(e) => { skipped.push((tx.hash, skip_reason_from(&e))); - continue; + break 'tx; } }; let pgas_used = insp.pgas.saturating_add(INTRINSIC_PGAS_PER_TX); - if block_pgas.saturating_add(insp.pgas) > BLOCK_PROVING_GAS_LIMIT { - skipped.push((tx.hash, Skip::BlockProvingBudget)); - continue; - } - block_pgas = block_pgas.saturating_add(insp.pgas); - segment_pgas = segment_pgas.saturating_add(insp.pgas); + debug_assert!(carry.block_pgas.saturating_add(insp.pgas) <= BLOCK_PROVING_GAS_LIMIT, "the inspector cap bounds the block's pgas"); + carry.block_pgas = carry.block_pgas.saturating_add(insp.pgas); + range_pgas = range_pgas.saturating_add(insp.pgas); let (status, gas_used, logs) = match &result { ExecutionResult::Success { gas, logs, .. } => (true, gas.tx_gas_used(), logs.clone()), @@ -192,7 +269,7 @@ pub fn execute_block(db: &mut IgneumDb, f: &BlockFixture) -> BlockOutcome { ExecutionResult::Halt { gas, logs, .. } => (false, gas.tx_gas_used(), logs.clone()), }; - // Fee flows (design 4.1 and 4.4), exactly as the node applies them. + // Fee flows (design 4.1 and 4.4, spec 7.5 item 2), exactly as the node applies them. let (status, gas_used, logs) = if insp.over_budget { let gas_used = tx.gas_limit(); let burned_exec = (gas_used as u128) * base_fee_exec; @@ -202,7 +279,7 @@ pub fn execute_block(db: &mut IgneumDb, f: &BlockFixture) -> BlockOutcome { db.sub_balance(tx.sender, U256::from(budget)); let shares = developer_shares(tip_total, &insp.attributions, |code| registered_payee(db, code)); let dev_total: u128 = shares.iter().map(|(_, w)| *w).sum(); - db.add_balance(b.miner, U256::from(tip_total - dev_total)); + db.add_balance(t.miner, U256::from(tip_total - dev_total)); for (payee, wei) in &shares { if let Some(p) = payee { db.add_balance(*p, U256::from(*wei)); @@ -211,12 +288,25 @@ pub fn execute_block(db: &mut IgneumDb, f: &BlockFixture) -> BlockOutcome { (false, gas_used, Vec::new()) } else { db.commit(state); - let burned_proving = (pgas_used as u128) * base_fee_proving; + let (status, gas_used, logs) = if insp.pgas_aborted { + // Spec 7.5 item 2: charged like an out-of-gas transaction for what it consumed up to the abort; + // the gas beyond the abort point goes back to the sender and its tip part comes back from the + // miner. State changes reverted, nonce advanced (committed above), status 0, no logs. + let consumed = insp.gas_at_abort.unwrap_or(gas_used).clamp(tx.intrinsic_gas, gas_used); + let unconsumed = (gas_used - consumed) as u128; + db.add_balance(tx.sender, U256::from(unconsumed * price)); + db.sub_balance(t.miner, U256::from(unconsumed * (price - base_fee_exec))); + (false, consumed, Vec::new()) + } else { + (status, gas_used, logs) + }; + // The proving charge never takes the sender past the signed budget (design 4.1). + let burned_proving = ((pgas_used as u128) * base_fee_proving).min(budget.saturating_sub((gas_used as u128) * price)); db.sub_balance(tx.sender, U256::from(burned_proving)); let tip_total = (gas_used as u128) * (price - base_fee_exec); let shares = developer_shares(tip_total, &insp.attributions, |code| registered_payee(db, code)); let dev_total: u128 = shares.iter().map(|(_, w)| *w).sum(); - db.sub_balance(b.miner, U256::from(dev_total)); + db.sub_balance(t.miner, U256::from(dev_total)); for (payee, wei) in &shares { if let Some(p) = payee { db.add_balance(*p, U256::from(*wei)); @@ -225,8 +315,9 @@ pub fn execute_block(db: &mut IgneumDb, f: &BlockFixture) -> BlockOutcome { (status, gas_used, logs) }; - block_gas = block_gas.saturating_add(gas_used); - cumulative_gas = cumulative_gas.saturating_add(gas_used); + carry.block_gas = carry.block_gas.saturating_add(gas_used); + carry.cumulative_gas = carry.cumulative_gas.saturating_add(gas_used); + range_gas = range_gas.saturating_add(gas_used); let mut bloom = Bloom::default(); bloom.accrue_logs(&logs); executed.push(ExecutedReceipt { @@ -234,18 +325,19 @@ pub fn execute_block(db: &mut IgneumDb, f: &BlockFixture) -> BlockOutcome { status, gas_used, pgas_used, - cumulative_gas_used: cumulative_gas, + cumulative_gas_used: carry.cumulative_gas, logs, logs_bloom: bloom, tx_type: tx.tx_type(), over_budget: insp.over_budget, + pgas_aborted: insp.pgas_aborted, }); } + boundaries.push(Boundary { carry, gas: range_gas, pgas: range_pgas, state_root: roots.then(|| db.state_root()) }); } - let state_root = db.state_root(); let receipts_root = receipts_root(&executed); - BlockOutcome { executed, skipped, gas_used: cumulative_gas, pgas_used: segment_pgas, state_root, receipts_root } + RangeOutcome { executed, skipped, gas_used: range_gas, pgas_used: range_pgas, carry_out: carry, boundaries, receipts_root } } pub fn registered_payee(db: &IgneumDb, code_address: Address) -> Option
{ @@ -257,6 +349,8 @@ pub fn registered_payee(db: &IgneumDb, code_address: Address) -> Option
} } +/// Ordered trie root over receipts (per block natively, per shard in the statement; the cumulative gas runs +/// across the whole segment either way). pub fn receipts_root(executed: &[ExecutedReceipt]) -> B256 { alloy_trie::root::ordered_trie_root_with_encoder(executed, |r, buf| { let receipt = Receipt { status: r.status.into(), cumulative_gas_used: r.cumulative_gas_used, logs: r.logs.clone() }; @@ -270,8 +364,40 @@ pub fn receipts_root(executed: &[ExecutedReceipt]) -> B256 { }) } +pub struct BlockOutcome { + pub executed: Vec, + pub skipped: Vec<(B256, Skip)>, + pub gas_used: u64, + pub pgas_used: u64, + pub state_root: B256, + pub receipts_root: B256, + pub tx_commitment: B256, + pub boundaries: Vec, + pub carry_out: Carry, +} + +/// Executes one whole chain block's segment on `db` (the node's statement, used by the exporter's check and by +/// the planner). `db` is the pre-state on entry and the post-state on return; a boundary with its state root is +/// recorded after every transaction. +pub fn execute_block(db: &mut IgneumDb, f: &crate::fixture::BlockFixture) -> BlockOutcome { + let txs = f.flatten(); + let out = execute_range(db, f.chain_id, &f.env, Some((&f.rewards, f.proving_pool_credit)), &txs, Carry::default(), true); + let state_root = db.state_root(); + BlockOutcome { + executed: out.executed, + skipped: out.skipped, + gas_used: out.gas_used, + pgas_used: out.pgas_used, + state_root, + receipts_root: out.receipts_root, + tx_commitment: out.carry_out.tx_acc, + boundaries: out.boundaries, + carry_out: out.carry_out, + } +} + /// Loads the fixture's pre-state and block hashes into a fresh database. -pub fn load_pre_state(f: &BlockFixture) -> IgneumDb { +pub fn load_pre_state(f: &crate::fixture::BlockFixture) -> IgneumDb { let mut db = IgneumDb::new(); for a in &f.pre_state { db.insert_account(a.address, a.nonce, a.balance, &a.code, &a.storage); @@ -281,23 +407,3 @@ pub fn load_pre_state(f: &BlockFixture) -> IgneumDb { } db } - -/// The whole statement: pre-state root, execution, outputs. Used by the guest and by the host's native check. -pub fn prove_statement(f: &BlockFixture) -> crate::output::ProveOutput { - let mut db = load_pre_state(f); - let pre_state_root = db.state_root(); - let out = execute_block(&mut db, f); - crate::output::ProveOutput { - chain_id: f.chain_id, - number: f.env.number, - block_hash: f.env.hash, - tx_commitment: f.tx_commitment(), - pre_state_root, - post_state_root: out.state_root, - receipts_root: out.receipts_root, - gas_used: out.gas_used, - pgas_used: out.pgas_used, - executed: out.executed.len() as u32, - skipped: out.skipped.len() as u32, - } -} diff --git a/proving/igneum-prove/core/src/fixture.rs b/proving/igneum-prove/core/src/fixture.rs index 8d46bd07d..9666c1f95 100644 --- a/proving/igneum-prove/core/src/fixture.rs +++ b/proving/igneum-prove/core/src/fixture.rs @@ -1,11 +1,11 @@ -//! The fixture: one real chain block as the execution layer exported it, with the pre-state it ran on. -//! JSON on disk (`proving/fixtures/*.json`, human readable, hex strings); the same structs go to the guest as -//! bincode through `ProveInput`. +//! The fixture: one real chain block as the execution layer exported it, with the pre-state it ran on, its +//! shard plan and what every shard must reproduce. JSON on disk (`proving/fixtures/*.json`). +use crate::executor::ShardTx; use alloy_primitives::{Address, Bytes, B256, U256}; use serde::{Deserialize, Serialize}; -pub const FORMAT: &str = "igneum-prove-fixture-v0"; +pub const FORMAT: &str = "igneum-prove-fixture-v1"; /// The chain block's environment (design section 3), as the node computed it. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -54,12 +54,54 @@ pub struct BlockFixture { pub pre_state: Vec, } -/// What the native run says the proof must reproduce. +impl BlockFixture { + /// The segment's transactions in sequence order with their including block. + pub fn flatten(&self) -> Vec { + let mut out = Vec::new(); + for (i, b) in self.blocks.iter().enumerate() { + for t in &b.txs { + out.push(ShardTx { block_index: i as u32, miner: b.miner, blue: b.blue, raw: t.clone() }); + } + } + out + } +} + +/// What the native run says a shard's proof must reproduce. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ShardExpected { + pub index: u32, + pub tx_start: u32, + pub tx_end: u32, + pub over_budget: bool, + pub pre_root: B256, + pub post_root: B256, + pub receipts_root: B256, + pub link_in: B256, + pub link_out: B256, + pub gas_used: u64, + pub pgas_used: u64, + pub executed: u32, + pub skipped: u32, + /// Witness size: accounts, slots, trie leaves, trie hashes, bincode bytes. + pub witness: (u32, u32, u32, u32, u64), +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Plan { + /// `S_p` the plan was cut with. `consensus` is false for a test cut below the consensus budget. + pub shard_budget: u64, + pub consensus: bool, + pub shards: Vec, +} + +/// What the whole block's native run gives. #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct Expected { pub pre_state_root: B256, pub post_state_root: B256, pub receipts_root: B256, + pub tx_commitment: B256, pub gas_used: u64, pub pgas_used: u64, pub executed: u32, @@ -73,28 +115,6 @@ pub struct Fixture { pub format: String, pub source: String, pub block: BlockFixture, + pub plan: Plan, pub expected: Expected, } - -/// What the guest reads: the block fixture alone. The expected values stay on the host. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ProveInput { - pub block: BlockFixture, -} - -impl BlockFixture { - /// keccak256 over the raw transactions in sequence order with their including miners: the commitment the - /// proof makes to its ordered input. - pub fn tx_commitment(&self) -> B256 { - let mut buf = Vec::new(); - for b in &self.blocks { - buf.extend_from_slice(b.miner.as_slice()); - buf.push(b.blue as u8); - for t in &b.txs { - buf.extend_from_slice(&(t.len() as u32).to_be_bytes()); - buf.extend_from_slice(t); - } - } - alloy_primitives::keccak256(buf) - } -} diff --git a/proving/igneum-prove/core/src/lib.rs b/proving/igneum-prove/core/src/lib.rs index 7146a2744..eb58a43bd 100644 --- a/proving/igneum-prove/core/src/lib.rs +++ b/proving/igneum-prove/core/src/lib.rs @@ -1,30 +1,37 @@ -//! Igneum proving v0: the statement one SP1 proof makes about one chain block. +//! Igneum proving, devnet v4: the shard statement one SP1 proof makes about part of a chain block, and the +//! block statement that aggregates the shards. //! -//! Statement. Given a chain block's environment, the ordered transaction list of its segment (per including -//! block, in sequence order), the rewards the consensus rules credit, and the pre-state (every account the -//! executor can touch, here the whole in-memory devnet state), running the Igneum executor yields the post-state -//! root and the receipts root. The guest commits both plus a commitment to the inputs (section `output`). +//! Shard statement (design 5.1). Given a chain block's environment, a contiguous range of its segment's +//! transactions from a carried-in position, the rewards (first shard only), and a witness of the pre-state the +//! range touches (accounts, slots and trie nodes, checked against the pre-root), running the Igneum executor +//! yields the post-root, the shard's receipts root, its gas and pgas and the carried-out position; the prover's +//! payout address is committed (ledger P12). Block statement (design 5.3): the shard proofs verified in order +//! and chained (`agg`). //! -//! This crate is a port of `vendor/igneum-node-exec/igneum/exec/src/{executor,state,pgas,registry,config}.rs` -//! at commit fb33069 (branch execution-layer, 3 October 2026) with the kaspa types removed so it compiles for -//! the zkVM target. The decoder is the execution layer's own crate (`igneum-evm-types`). Every rule is the -//! node's rule: rewards by rule, the nonce-rule skip, two-dimensional gas with the prototype pgas table, fee -//! flows (base fees burned, proving charge burned, 80/20 tip with the developer split per call frame), the -//! registry population on CREATE, the state root over every non-empty account (EIP-161) through alloy-trie. -//! The exporter (`export/`) checks the port against the node by replaying the simnet's export from genesis -//! and comparing every segment's state root; design 2.1 ("the executor is a library with no network -//! dependency, used by the node, the shard planner, the prover's witness generator") is where this duplicate -//! goes away. +//! The executor is a port of `vendor/igneum-node-exec/igneum/exec/src/{executor,state,pgas,registry,config}.rs` +//! at commit b7fca5a0 (branch execution-layer, merged into devnet-v4, 4 October 2026) with the kaspa types +//! removed so it compiles for the zkVM target. The decoder is the execution layer's own crate +//! (`igneum-evm-types`). Every rule is the node's rule: rewards by rule, the nonce-rule skip, two-dimensional +//! gas with the prototype pgas table and the spec 7.5 cap and abort, fee flows (base fees burned, proving +//! charge burned, 80/20 tip with the developer split per call frame), the registry population on CREATE, the +//! state root over every non-empty account (EIP-161) through alloy-trie. The exporter (`export/`) checks the +//! port against the node by replaying the simnet's export from genesis and comparing every segment's state +//! root; design 2.1 is where this duplicate goes away. +pub mod agg; pub mod config; pub mod executor; pub mod fixture; -pub mod output; pub mod pgas; +pub mod plan; pub mod registry; +pub mod shard; pub mod state; +pub mod trie; +pub mod witness; -pub use executor::{execute_block, BlockOutcome}; -pub use fixture::{AccountFixture, BlockFixture, Expected, Fixture, FixtureEnv, IncludingBlock, ProveInput}; -pub use output::ProveOutput; +pub use executor::{execute_block, execute_range, BlockOutcome, Carry, ShardTx}; +pub use fixture::{AccountFixture, BlockFixture, Expected, Fixture, FixtureEnv, IncludingBlock, Plan, ShardExpected}; +pub use shard::{ShardInput, ShardOutput}; pub use state::IgneumDb; +pub use witness::StateWitness; diff --git a/proving/igneum-prove/core/src/output.rs b/proving/igneum-prove/core/src/output.rs deleted file mode 100644 index cc1f9f720..000000000 --- a/proving/igneum-prove/core/src/output.rs +++ /dev/null @@ -1,60 +0,0 @@ -//! The public values of the proof, in one fixed byte layout so a verifier in any language can read them. - -use alloy_primitives::B256; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ProveOutput { - pub chain_id: u64, - pub number: u64, - pub block_hash: B256, - pub tx_commitment: B256, - pub pre_state_root: B256, - pub post_state_root: B256, - pub receipts_root: B256, - pub gas_used: u64, - pub pgas_used: u64, - pub executed: u32, - pub skipped: u32, -} - -impl ProveOutput { - pub const LEN: usize = 8 + 8 + 32 * 5 + 8 + 8 + 4 + 4; - - pub fn to_bytes(&self) -> Vec { - let mut v = Vec::with_capacity(Self::LEN); - v.extend_from_slice(&self.chain_id.to_be_bytes()); - v.extend_from_slice(&self.number.to_be_bytes()); - v.extend_from_slice(self.block_hash.as_slice()); - v.extend_from_slice(self.tx_commitment.as_slice()); - v.extend_from_slice(self.pre_state_root.as_slice()); - v.extend_from_slice(self.post_state_root.as_slice()); - v.extend_from_slice(self.receipts_root.as_slice()); - v.extend_from_slice(&self.gas_used.to_be_bytes()); - v.extend_from_slice(&self.pgas_used.to_be_bytes()); - v.extend_from_slice(&self.executed.to_be_bytes()); - v.extend_from_slice(&self.skipped.to_be_bytes()); - v - } - - pub fn from_bytes(b: &[u8]) -> Option { - if b.len() != Self::LEN { - return None; - } - let u64_at = |i: usize| u64::from_be_bytes(b[i..i + 8].try_into().unwrap()); - let u32_at = |i: usize| u32::from_be_bytes(b[i..i + 4].try_into().unwrap()); - let b256_at = |i: usize| B256::from_slice(&b[i..i + 32]); - Some(Self { - chain_id: u64_at(0), - number: u64_at(8), - block_hash: b256_at(16), - tx_commitment: b256_at(48), - pre_state_root: b256_at(80), - post_state_root: b256_at(112), - receipts_root: b256_at(144), - gas_used: u64_at(176), - pgas_used: u64_at(184), - executed: u32_at(192), - skipped: u32_at(196), - }) - } -} diff --git a/proving/igneum-prove/core/src/pgas.rs b/proving/igneum-prove/core/src/pgas.rs index 7dbae34e5..e99f40bb8 100644 --- a/proving/igneum-prove/core/src/pgas.rs +++ b/proving/igneum-prove/core/src/pgas.rs @@ -1,4 +1,5 @@ -//! Mirror of igneum-exec `pgas.rs` at fb33069 (the prototype pgas table and the per-frame developer attribution). +//! Mirror of igneum-exec `pgas.rs` at b7fca5a0 (the prototype pgas table, the per-frame developer attribution and +//! the spec 7.5 cap: metering halts the transaction before the opcode or precompile that would cross it). //! Proving gas (design 4.2) and the per-frame developer attribution (design 4.5), both measured by one revm //! inspector while the native execution runs. //! @@ -12,7 +13,7 @@ use alloy_primitives::{Address, U256}; use revm::context_interface::{ContextTr, JournalTr}; use revm::inspector::Inspector; use revm::interpreter::interpreter_types::Jumps; -use revm::interpreter::{CallInputs, CallOutcome, CallScheme, CreateInputs, CreateOutcome, Interpreter}; +use revm::interpreter::{CallInputs, CallOutcome, CallScheme, CreateInputs, CreateOutcome, Gas, InstructionResult, Interpreter, InterpreterResult}; /// Fixed pgas of one opcode plus a per-word (32-byte) component read from the stack where the shape needs it. pub fn opcode_pgas(op: u8, interp: &Interpreter) -> u64 { @@ -111,6 +112,15 @@ pub struct FrameAttribution { pub struct IgneumInspector { /// Proving gas metered so far (opcodes plus precompiles), without the intrinsic per-transaction pgas. pub pgas: u64, + /// The proving-gas cap of this transaction (spec 7.5): the including block's remaining `B_p`, without the + /// intrinsic pgas already counted. Metering is incremental and the transaction is halted the moment one more + /// opcode or precompile would cross it, so the native work of an over-budget transaction is bounded. + pub pgas_cap: u64, + /// Set when the cap was hit: the transaction is charged as out of gas for what it consumed (spec 7.5). + pub pgas_aborted: bool, + /// Execution gas the transaction had consumed when the cap was hit (intrinsic included), the amount the + /// sender pays for under spec 7.5 item 2. `None` until an abort. + pub gas_at_abort: Option, /// Execution gas spent before the first frame opened (the intrinsic gas), supplied by the executor. pub intrinsic_gas: u64, /// Price per execution gas unit the sender pays (`f_e + tip`), the proving base fee, and the signed budget. @@ -128,8 +138,31 @@ pub struct IgneumInspector { } impl IgneumInspector { - pub fn new(intrinsic_gas: u64, exec_price: u128, proving_base_fee: u128, budget_wei: u128) -> Self { - Self { intrinsic_gas, exec_price, proving_base_fee, budget_wei, ..Default::default() } + pub fn new(intrinsic_gas: u64, exec_price: u128, proving_base_fee: u128, budget_wei: u128, pgas_cap: u64) -> Self { + Self { intrinsic_gas, exec_price, proving_base_fee, budget_wei, pgas_cap, ..Default::default() } + } + + /// Meters `delta` pgas unless that would cross the cap. Returns false, and marks the abort, when it would; + /// the work that would have cost `delta` is then never done. + fn meter(&mut self, delta: u64) -> bool { + match self.pgas.checked_add(delta) { + Some(next) if next <= self.pgas_cap => { + self.pgas = next; + true + } + _ => { + self.pgas_aborted = true; + false + } + } + } + + /// Halts the running frame for the pgas abort and records the execution gas consumed so far, once. + fn halt_for_pgas(&mut self, interp: &mut Interpreter, spent: u64) { + if self.gas_at_abort.is_none() { + self.gas_at_abort = Some(self.running_gas(spent)); + } + interp.halt_oog(); } /// Execution gas spent by the whole transaction so far, every unit counted once (design 4.1 running charge). @@ -162,8 +195,6 @@ impl IgneumInspector { impl Inspector for IgneumInspector { fn step(&mut self, interp: &mut Interpreter, _ctx: &mut CTX) { - let op = interp.bytecode.opcode(); - self.pgas = self.pgas.saturating_add(opcode_pgas(op, interp)); let spent = interp.gas.total_gas_spent(); if let Some(frame) = self.stack.last_mut() { // Gas of the previous instruction of this frame lands here (spent moved since the last look). @@ -171,6 +202,17 @@ impl Inspector for IgneumInspector { frame.last_seen_spent = spent; } self.current_spent = spent; + // Spec 7.5: once the cap is hit (here, or in a precompile call below) every frame still open halts at its + // next instruction, so no contract can catch the failed call and keep going. + if self.pgas_aborted { + self.halt_for_pgas(interp, spent); + return; + } + let op = interp.bytecode.opcode(); + if !self.meter(opcode_pgas(op, interp)) { + self.halt_for_pgas(interp, spent); + return; + } if self.running_charge(spent) > self.budget_wei { self.over_budget = true; interp.halt_oog(); @@ -194,7 +236,15 @@ impl Inspector for IgneumInspector { // Precompile frames are charged here (they run no opcodes); their attribution share burns (design 4.5). let input_bytes: Vec = inputs.input.bytes(_ctx).to_vec(); if let Some(p) = precompile_pgas(inputs.bytecode_address, input_bytes.len(), &input_bytes) { - self.pgas = self.pgas.saturating_add(p); + if !self.meter(p) { + // Spec 7.5: the precompile is not run. The frame is answered with a revert that spends none of the + // forwarded gas, so the parent's consumed gas stays exact; the parent halts at its next step + // (`step` above) before it can act on the result. `call_end` still fires for an overridden call, + // so the frame is opened here to keep the attribution stack balanced. + self.open(code_address, inputs.gas_limit); + let result = InterpreterResult { result: InstructionResult::Revert, output: Default::default(), gas: Gas::new(inputs.gas_limit) }; + return Some(CallOutcome::new(result, inputs.return_memory_offset.clone())); + } } self.open(code_address, inputs.gas_limit); None diff --git a/proving/igneum-prove/core/src/plan.rs b/proving/igneum-prove/core/src/plan.rs new file mode 100644 index 000000000..b910b5373 --- /dev/null +++ b/proving/igneum-prove/core/src/plan.rs @@ -0,0 +1,121 @@ +//! The shard planner of design 5.1 and spec 7.2: cuts a segment at transaction boundaries so each shard carries +//! at most `S_p` pgas, from the native trace alone, so every node computes the same shard list. + +use crate::executor::{Boundary, Carry}; +use alloy_primitives::{keccak256, B256}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ShardSpec { + pub index: u32, + /// Transactions `tx_start..tx_end` of the segment (sequence order, including blocks flattened). + pub tx_start: u32, + pub tx_end: u32, + pub gas: u64, + pub pgas: u64, + /// A single transaction above `S_p` is a shard of its own, proven with the zkVM's own continuations. + pub over_budget: bool, + pub carry_in: Carry, + pub carry_out: Carry, + pub pre_root: B256, + pub post_root: B256, +} + +/// Shard id `(segment hash, shard index)` as one word. +pub fn shard_id(segment: &B256, index: u32) -> B256 { + let mut b = Vec::with_capacity(52); + b.extend_from_slice(b"igneum-shard-id/v1"); + b.extend_from_slice(segment.as_slice()); + b.extend_from_slice(&index.to_be_bytes()); + keccak256(b) +} + +/// Cuts the trace greedily: a shard grows while the next transaction keeps it at or under `budget`; a +/// transaction that alone exceeds the budget becomes its own shard. A segment with no transactions is one +/// empty shard (it still carries the rewards). `pre_root` and `pre_carry` describe the state before the first +/// transaction; every boundary must carry its state root. +pub fn plan(pre_root: B256, pre_carry: Carry, boundaries: &[Boundary], budget: u64) -> Vec { + let root_at = |i: usize| -> B256 { + if i == 0 { + pre_root + } else { + boundaries[i - 1].state_root.expect("the planner needs the state root at every boundary") + } + }; + let carry_at = |i: usize| -> Carry { if i == 0 { pre_carry } else { boundaries[i - 1].carry } }; + let pgas_at = |i: usize| -> u64 { if i == 0 { 0 } else { boundaries[i - 1].pgas } }; + let gas_at = |i: usize| -> u64 { if i == 0 { 0 } else { boundaries[i - 1].gas } }; + + let mut shards = Vec::new(); + let n = boundaries.len(); + let mut start = 0usize; + let mut index = 0u32; + loop { + let mut end = start; + let mut over = false; + while end < n { + let next_pgas = pgas_at(end + 1) - pgas_at(start); + if next_pgas > budget { + if end == start { + end += 1; // one transaction above the budget: its own shard + over = true; + } + break; + } + end += 1; + } + shards.push(ShardSpec { + index, + tx_start: start as u32, + tx_end: end as u32, + gas: gas_at(end) - gas_at(start), + pgas: pgas_at(end) - pgas_at(start), + over_budget: over, + carry_in: carry_at(start), + carry_out: carry_at(end), + pre_root: root_at(start), + post_root: root_at(end), + }); + index += 1; + if end >= n { + break; + } + start = end; + } + shards +} + +#[cfg(test)] +mod tests { + use super::*; + + fn boundaries(pgas: &[u64]) -> Vec { + let mut acc = 0; + pgas.iter() + .enumerate() + .map(|(i, p)| { + acc += p; + Boundary { carry: Carry { tx_index: i as u32 + 1, ..Default::default() }, gas: 0, pgas: acc, state_root: Some(B256::with_last_byte(i as u8 + 1)) } + }) + .collect() + } + + #[test] + fn cuts_at_the_budget_and_isolates_an_oversized_transaction() { + let b = boundaries(&[300, 300, 500, 1200, 100, 100]); + let s = plan(B256::ZERO, Carry::default(), &b, 1000); + let ranges: Vec<(u32, u32, bool)> = s.iter().map(|x| (x.tx_start, x.tx_end, x.over_budget)).collect(); + assert_eq!(ranges, vec![(0, 2, false), (2, 3, false), (3, 4, true), (4, 6, false)]); + assert_eq!(s[0].pre_root, B256::ZERO); + assert_eq!(s[0].post_root, s[1].pre_root); + assert_eq!(s[3].post_root, B256::with_last_byte(6)); + assert_eq!(s.iter().map(|x| x.pgas).sum::(), 2500); + } + + #[test] + fn an_empty_segment_is_one_shard() { + let s = plan(B256::ZERO, Carry::default(), &[], 1000); + assert_eq!(s.len(), 1); + assert_eq!((s[0].tx_start, s[0].tx_end), (0, 0)); + } +} diff --git a/proving/igneum-prove/core/src/shard.rs b/proving/igneum-prove/core/src/shard.rs new file mode 100644 index 000000000..dfd942b06 --- /dev/null +++ b/proving/igneum-prove/core/src/shard.rs @@ -0,0 +1,183 @@ +//! The shard statement (design 5.1, ledger P12): from the pre-root and the witness, running the executor over +//! the shard's transactions from the carried-in position yields the post-root, the shard's receipts root, its +//! gas and pgas, and the carried-out position; the prover's payout address is committed with it. + +use crate::executor::{execute_range, Carry, ShardTx}; +use crate::fixture::FixtureEnv; +use crate::witness::{self, StateWitness}; +use alloy_primitives::{Address, B256, U256}; +use serde::{Deserialize, Serialize}; + +/// What the shard guest reads. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ShardInput { + pub chain_id: u64, + pub env: FixtureEnv, + pub block_hashes: Vec<(u64, B256)>, + /// Applied by shard 0 only (the segment's rewards precede its first transaction). + pub rewards: Vec<(Address, U256)>, + pub proving_pool_credit: U256, + pub shard_index: u32, + pub txs: Vec, + pub carry_in: Carry, + /// The payout address of the prover making this proof (ledger P12). + pub prover: Address, + pub witness: StateWitness, +} + +/// The public values of a shard proof, one fixed byte layout. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ShardOutput { + pub chain_id: u64, + pub number: u64, + pub block_hash: B256, + pub shard_index: u32, + pub tx_start: u32, + pub tx_count: u32, + pub link_in: B256, + pub link_out: B256, + pub tx_acc_in: B256, + pub tx_acc_out: B256, + pub pre_root: B256, + pub post_root: B256, + pub receipts_root: B256, + pub gas_used: u64, + pub pgas_used: u64, + pub executed: u32, + pub skipped: u32, + pub prover: Address, +} + +impl ShardOutput { + pub const LEN: usize = 8 + 8 + 32 + 4 + 4 + 4 + 32 * 7 + 8 + 8 + 4 + 4 + 20; + + pub fn to_bytes(&self) -> Vec { + let mut v = Vec::with_capacity(Self::LEN); + v.extend_from_slice(&self.chain_id.to_be_bytes()); + v.extend_from_slice(&self.number.to_be_bytes()); + v.extend_from_slice(self.block_hash.as_slice()); + v.extend_from_slice(&self.shard_index.to_be_bytes()); + v.extend_from_slice(&self.tx_start.to_be_bytes()); + v.extend_from_slice(&self.tx_count.to_be_bytes()); + for w in [&self.link_in, &self.link_out, &self.tx_acc_in, &self.tx_acc_out, &self.pre_root, &self.post_root, &self.receipts_root] { + v.extend_from_slice(w.as_slice()); + } + v.extend_from_slice(&self.gas_used.to_be_bytes()); + v.extend_from_slice(&self.pgas_used.to_be_bytes()); + v.extend_from_slice(&self.executed.to_be_bytes()); + v.extend_from_slice(&self.skipped.to_be_bytes()); + v.extend_from_slice(self.prover.as_slice()); + debug_assert_eq!(v.len(), Self::LEN); + v + } + + pub fn from_bytes(b: &[u8]) -> Option { + if b.len() != Self::LEN { + return None; + } + let u64_at = |i: usize| u64::from_be_bytes(b[i..i + 8].try_into().unwrap()); + let u32_at = |i: usize| u32::from_be_bytes(b[i..i + 4].try_into().unwrap()); + let b256_at = |i: usize| B256::from_slice(&b[i..i + 32]); + Some(Self { + chain_id: u64_at(0), + number: u64_at(8), + block_hash: b256_at(16), + shard_index: u32_at(48), + tx_start: u32_at(52), + tx_count: u32_at(56), + link_in: b256_at(60), + link_out: b256_at(92), + tx_acc_in: b256_at(124), + tx_acc_out: b256_at(156), + pre_root: b256_at(188), + post_root: b256_at(220), + receipts_root: b256_at(252), + gas_used: u64_at(284), + pgas_used: u64_at(292), + executed: u32_at(300), + skipped: u32_at(304), + prover: Address::from_slice(&b[308..328]), + }) + } +} + +/// The statement, as the guest runs it and as the host checks it natively first. +pub fn shard_statement(input: &ShardInput) -> ShardOutput { + let (mut db, pre_root) = witness::load(&input.witness, &input.block_hashes); + let rewards = (input.shard_index == 0).then_some((&input.rewards[..], input.proving_pool_credit)); + let out = execute_range(&mut db, input.chain_id, &input.env, rewards, &input.txs, input.carry_in, false); + let post_root = witness::post_root(&db, &input.witness); + ShardOutput { + chain_id: input.chain_id, + number: input.env.number, + block_hash: input.env.hash, + shard_index: input.shard_index, + tx_start: input.carry_in.tx_index, + tx_count: input.txs.len() as u32, + link_in: input.carry_in.link(), + link_out: out.carry_out.link(), + tx_acc_in: input.carry_in.tx_acc, + tx_acc_out: out.carry_out.tx_acc, + pre_root, + post_root, + receipts_root: out.receipts_root, + gas_used: out.gas_used, + pgas_used: out.pgas_used, + executed: out.executed.len() as u32, + skipped: out.skipped.len() as u32, + prover: input.prover, + } +} + +/// One planned shard with its witness, as a full node builds them (design 5.1): the plan from the native trace, +/// the witness from the access log of the shard's native execution, and the native statement for the check. +pub struct BuiltShard { + pub spec: crate::plan::ShardSpec, + pub input: ShardInput, + pub output: ShardOutput, +} + +/// Plans a block into shards of at most `budget` pgas and builds every shard's input. Returns the whole-block +/// native outcome, its pre-root and the shards. Panics if a shard's statement does not reproduce the plan's +/// roots and links, or if the shards' post-state is not the block's. +pub fn build_shards(block: &crate::fixture::BlockFixture, budget: u64, prover: Address) -> (crate::executor::BlockOutcome, B256, Vec) { + use crate::executor::{execute_block, load_pre_state}; + let mut db = load_pre_state(block); + let pre_root = db.state_root(); + let txs = block.flatten(); + let outcome = execute_block(&mut db, block); + let specs = crate::plan::plan(pre_root, Carry::default(), &outcome.boundaries, budget); + + let mut state = load_pre_state(block); + let mut shards = Vec::with_capacity(specs.len()); + for spec in specs { + let range = &txs[spec.tx_start as usize..spec.tx_end as usize]; + let rewards = (spec.index == 0).then_some((&block.rewards[..], block.proving_pool_credit)); + let pre = state.clone(); + state.start_log(); + let _ = execute_range(&mut state, block.chain_id, &block.env, rewards, range, spec.carry_in, false); + let log = state.take_log(); + let witness = witness::generate(&pre, &log); + let input = ShardInput { + chain_id: block.chain_id, + env: block.env.clone(), + block_hashes: block.block_hashes.clone(), + rewards: if spec.index == 0 { block.rewards.clone() } else { Vec::new() }, + proving_pool_credit: if spec.index == 0 { block.proving_pool_credit } else { U256::ZERO }, + shard_index: spec.index, + txs: range.to_vec(), + carry_in: spec.carry_in, + prover, + witness, + }; + let output = shard_statement(&input); + assert_eq!(output.pre_root, spec.pre_root, "shard {} pre-root from the witness", spec.index); + assert_eq!(output.post_root, spec.post_root, "shard {} post-root from the witness", spec.index); + assert_eq!(output.link_out, spec.carry_out.link(), "shard {} carry", spec.index); + assert_eq!(output.pgas_used, spec.pgas, "shard {} pgas", spec.index); + assert_eq!(output.gas_used, spec.gas, "shard {} gas", spec.index); + shards.push(BuiltShard { spec, input, output }); + } + assert_eq!(state.state_root(), outcome.state_root, "the shards' post-state is the block's"); + (outcome, pre_root, shards) +} diff --git a/proving/igneum-prove/core/src/state.rs b/proving/igneum-prove/core/src/state.rs index 9f7c35afe..306cd14d5 100644 --- a/proving/igneum-prove/core/src/state.rs +++ b/proving/igneum-prove/core/src/state.rs @@ -1,6 +1,10 @@ -//! In-memory Ethereum state with the MPT state root as an output (mirror of igneum-exec `state.rs` at fb33069). +//! In-memory Ethereum state with the MPT state root as an output (mirror of igneum-exec `state.rs` at b7fca5a0), +//! with two optional modes for shard proving: an access log (the witness generator records every account and +//! slot an execution reads or writes) and a strict coverage check (the guest refuses any read the witness does +//! not prove; design 5.1). use crate::config::BLOCKHASH_WINDOW; +use crate::witness::Coverage; use alloy_primitives::{keccak256, Address, B256, U256}; use alloy_trie::TrieAccount; use revm::bytecode::Bytecode; @@ -8,13 +12,23 @@ use revm::database::{AccountState, CacheDB, EmptyDB}; use revm::primitives::KECCAK_EMPTY; use revm::state::{AccountInfo, EvmState}; use revm::{Database, DatabaseCommit}; -use std::collections::{BTreeMap, VecDeque}; +use std::cell::RefCell; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::convert::Infallible; +/// Every account and storage slot touched by an execution (reads and writes alike). +#[derive(Clone, Debug, Default)] +pub struct AccessLog { + pub accounts: BTreeSet
, + pub slots: BTreeSet<(Address, U256)>, +} + #[derive(Clone, Debug)] pub struct IgneumDb { pub cache: CacheDB, pub block_hashes: VecDeque<(u64, B256)>, + pub log: Option>, + pub strict: Option, } impl Default for IgneumDb { @@ -25,7 +39,39 @@ impl Default for IgneumDb { impl IgneumDb { pub fn new() -> Self { - Self { cache: CacheDB::new(EmptyDB::default()), block_hashes: VecDeque::new() } + Self { cache: CacheDB::new(EmptyDB::default()), block_hashes: VecDeque::new(), log: None, strict: None } + } + + pub fn start_log(&mut self) { + self.log = Some(RefCell::new(AccessLog::default())); + } + + pub fn take_log(&mut self) -> AccessLog { + self.log.take().map(|l| l.into_inner()).unwrap_or_default() + } + + fn note_account(&self, address: Address) { + if let Some(l) = &self.log { + l.borrow_mut().accounts.insert(address); + } + if let Some(c) = &self.strict { + if !c.account_covered(&address) { + panic!("witness does not cover account {address}"); + } + } + } + + fn note_slot(&self, address: Address, slot: U256) { + if let Some(l) = &self.log { + let mut l = l.borrow_mut(); + l.accounts.insert(address); + l.slots.insert((address, slot)); + } + if let Some(c) = &self.strict { + if !c.slot_covered(&address, &slot) { + panic!("witness does not cover slot {slot} of {address}"); + } + } } pub fn push_block_hash(&mut self, number: u64, hash: B256) { @@ -51,10 +97,16 @@ impl IgneumDb { } } - pub fn account(&self, address: Address) -> Option<&AccountInfo> { + /// The account as the cache holds it, without logging (the witness generator and the exporter). + pub fn account_raw(&self, address: Address) -> Option<&AccountInfo> { self.cache.cache.accounts.get(&address).filter(|a| a.account_state != AccountState::NotExisting).map(|a| &a.info) } + pub fn account(&self, address: Address) -> Option<&AccountInfo> { + self.note_account(address); + self.account_raw(address) + } + pub fn balance(&self, address: Address) -> U256 { self.account(address).map(|a| a.balance).unwrap_or(U256::ZERO) } @@ -76,10 +128,15 @@ impl IgneumDb { } pub fn storage(&self, address: Address, slot: U256) -> U256 { + self.note_slot(address, slot); + self.storage_raw(address, slot) + } + + pub fn storage_raw(&self, address: Address, slot: U256) -> U256 { self.cache.cache.accounts.get(&address).and_then(|a| a.storage.get(&slot).copied()).unwrap_or(U256::ZERO) } - /// Non-zero storage of an account, sorted by slot. + /// Non-zero storage of an account, sorted by slot (no logging: the exporter and the witness generator). pub fn storage_of(&self, address: Address) -> Vec<(U256, U256)> { let mut out: Vec<(U256, U256)> = self .cache @@ -96,6 +153,7 @@ impl IgneumDb { if wei.is_zero() { return; } + self.note_account(address); let entry = self.cache.cache.accounts.entry(address).or_default(); if entry.account_state == AccountState::NotExisting { entry.account_state = AccountState::None; @@ -107,12 +165,14 @@ impl IgneumDb { if wei.is_zero() { return; } + self.note_account(address); if let Some(entry) = self.cache.cache.accounts.get_mut(&address) { entry.info.balance = entry.info.balance.saturating_sub(wei); } } pub fn bump_nonce(&mut self, address: Address) { + self.note_account(address); let entry = self.cache.cache.accounts.entry(address).or_default(); if entry.account_state == AccountState::NotExisting { entry.account_state = AccountState::None; @@ -120,22 +180,30 @@ impl IgneumDb { entry.info.nonce += 1; } + /// The trie leaf of an account as the state root sees it: `None` for an absent or empty (EIP-161) account. + pub fn trie_account(&self, address: Address) -> Option { + let acc = self.cache.cache.accounts.get(&address)?; + if acc.account_state == AccountState::NotExisting { + return None; + } + let info = &acc.info; + let has_storage = acc.storage.values().any(|v| !v.is_zero()); + if info.balance.is_zero() && info.nonce == 0 && info.code_hash == KECCAK_EMPTY && !has_storage { + return None; + } + let storage_root = alloy_trie::root::storage_root_unhashed(acc.storage.iter().filter(|(_, v)| !v.is_zero()).map(|(k, v)| (B256::from(k.to_be_bytes::<32>()), *v))); + Some(TrieAccount { nonce: info.nonce, balance: info.balance, storage_root, code_hash: info.code_hash }) + } + /// Ethereum state root over every existing, non-empty account (EIP-161), keccak-keyed MPT through alloy-trie. + /// Only meaningful over a full state; a strict (witness) database computes its root through the witness. pub fn state_root(&self) -> B256 { + assert!(self.strict.is_none(), "state_root over a witness database: use the witness post-root"); let mut accounts: BTreeMap = BTreeMap::new(); - for (address, acc) in self.cache.cache.accounts.iter() { - if acc.account_state == AccountState::NotExisting { - continue; + for address in self.cache.cache.accounts.keys() { + if let Some(a) = self.trie_account(*address) { + accounts.insert(keccak256(address), a); } - let info = &acc.info; - let has_storage = acc.storage.values().any(|v| !v.is_zero()); - if info.balance.is_zero() && info.nonce == 0 && info.code_hash == KECCAK_EMPTY && !has_storage { - continue; - } - let storage_root = alloy_trie::root::storage_root_unhashed( - acc.storage.iter().filter(|(_, v)| !v.is_zero()).map(|(k, v)| (B256::from(k.to_be_bytes::<32>()), *v)), - ); - accounts.insert(keccak256(address), TrieAccount { nonce: info.nonce, balance: info.balance, storage_root, code_hash: info.code_hash }); } alloy_trie::root::state_root(accounts) } @@ -151,12 +219,14 @@ impl IgneumDb { impl Database for IgneumDb { type Error = Infallible; fn basic(&mut self, address: Address) -> Result, Infallible> { + self.note_account(address); Ok(self.cache.basic(address).unwrap()) } fn code_by_hash(&mut self, code_hash: B256) -> Result { Ok(self.cache.code_by_hash(code_hash).unwrap()) } fn storage(&mut self, address: Address, index: U256) -> Result { + self.note_slot(address, index); Ok(self.cache.storage(address, index).unwrap()) } fn block_hash(&mut self, number: u64) -> Result { @@ -166,6 +236,14 @@ impl Database for IgneumDb { impl DatabaseCommit for IgneumDb { fn commit(&mut self, changes: EvmState) { + if self.log.is_some() || self.strict.is_some() { + for (address, account) in &changes { + self.note_account(*address); + for slot in account.storage.keys() { + self.note_slot(*address, *slot); + } + } + } self.cache.commit(changes) } } diff --git a/proving/igneum-prove/core/src/trie.rs b/proving/igneum-prove/core/src/trie.rs new file mode 100644 index 000000000..fd8399e2f --- /dev/null +++ b/proving/igneum-prove/core/src/trie.rs @@ -0,0 +1,370 @@ +//! Partial Merkle Patricia tries for shard witnesses (design 5.1): the leaves a shard touches in full and the +//! hash of every subtree it never enters. A full node generates one from its own trie (`generate`); the guest +//! rebuilds the root from it (`root`), compares it with the shard's pre-root, executes, and rebuilds the root +//! again from the updated leaves (the same hashes) for the post-root. +//! +//! Soundness rests on two rules. First, a key is `covered` only when no retained subtree hash is a prefix of +//! its path: the guest refuses to read a key it cannot prove present or absent, so a witness that leaves an +//! account out makes the proof impossible rather than wrong. Second, every sibling of a touched path that +//! could become the sole child of a branch after a deletion is carried in full when it is a leaf, or as the +//! hash below its key when it is an extension, so the post-root computation never has to guess what is behind a +//! hash. Hashes are of branch nodes only; inline nodes (under 32 bytes of RLP, storage tries) are expanded. + +use alloy_primitives::{Bytes, B256}; +use alloy_rlp::Decodable; +use alloy_trie::nodes::TrieNode; +use alloy_trie::proof::ProofRetainer; +use alloy_trie::{HashBuilder, Nibbles, EMPTY_ROOT_HASH}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct TrieWitness { + /// (hashed key, RLP value): the leaves carried in full, sorted by key. + pub leaves: Vec<(B256, Bytes)>, + /// (nibble path, one nibble per byte; hash of the branch node there): subtrees carried as a hash, sorted. + pub hashes: Vec<(Bytes, B256)>, +} + +impl TrieWitness { + /// True when the witness proves the key present or absent: no retained hash sits on the key's path. + pub fn covers(&self, hashed_key: &B256) -> bool { + let key = Nibbles::unpack(hashed_key); + !self.hashes.iter().any(|(p, _)| key.starts_with(&Nibbles::from_nibbles_unchecked(p))) + } + + pub fn leaf(&self, hashed_key: &B256) -> Option<&[u8]> { + self.leaves.binary_search_by(|(k, _)| k.cmp(hashed_key)).ok().map(|i| self.leaves[i].1.as_ref()) + } + + /// The root the witness commits to, with its leaves replaced by `leaves` (sorted; `None` deletes). + pub fn root_with(&self, leaves: &BTreeMap>>) -> B256 { + let mut entries: Vec<(Nibbles, Entry<'_>)> = Vec::with_capacity(self.leaves.len() + self.hashes.len()); + for (k, v) in &self.leaves { + if !leaves.contains_key(k) { + entries.push((Nibbles::unpack(k), Entry::Leaf(v.as_ref()))); + } + } + for (k, v) in leaves { + if let Some(v) = v { + entries.push((Nibbles::unpack(k), Entry::Leaf(v.as_slice()))); + } + } + for (p, h) in &self.hashes { + entries.push((Nibbles::from_nibbles_unchecked(p), Entry::Hash(*h))); + } + entries.sort_by(|a, b| a.0.cmp(&b.0)); + if entries.is_empty() { + return EMPTY_ROOT_HASH; + } + if let [(p, Entry::Hash(h))] = entries.as_slice() { + if p.is_empty() { + return *h; + } + } + let mut hb = HashBuilder::default(); + for (path, e) in &entries { + match e { + Entry::Leaf(v) => hb.add_leaf(*path, v), + Entry::Hash(h) => hb.add_branch(*path, *h, false), + } + } + hb.root() + } + + pub fn root(&self) -> B256 { + self.root_with(&BTreeMap::new()) + } +} + +enum Entry<'a> { + Leaf(&'a [u8]), + Hash(B256), +} + +/// Generates the witness of `targets` over a full trie given as (hashed key, RLP value), sorted. +pub fn generate(entries: &BTreeMap>, targets: &[B256]) -> TrieWitness { + if entries.is_empty() { + return TrieWitness::default(); + } + // Every node of the trie, by path: the hash builder with a retainer over every key keeps them all. + let retainer = ProofRetainer::new(entries.keys().map(Nibbles::unpack).collect()); + let mut hb = HashBuilder::default().with_proof_retainer(retainer); + for (k, v) in entries { + hb.add_leaf(Nibbles::unpack(k), v); + } + let _root = hb.root(); + let nodes: BTreeMap = hb.take_proof_nodes().into_inner().into_iter().collect(); + + let mut w = Collector::default(); + let target_paths: Vec = targets.iter().map(Nibbles::unpack).collect(); + for key in &target_paths { + w.walk(&nodes, key, &target_paths); + } + w.finish() +} + +#[derive(Default)] +struct Collector { + leaves: BTreeMap>, + hashes: BTreeMap, +} + +impl Collector { + fn node_at<'a>(nodes: &'a BTreeMap, path: &Nibbles) -> TrieNode { + let rlp = nodes.get(path).unwrap_or_else(|| panic!("trie node at {path:?} missing from the full trie")); + TrieNode::decode(&mut rlp.as_ref()).expect("retained trie node decodes") + } + + fn add_leaf(&mut self, path: Nibbles, key: &Nibbles, value: &[u8]) { + let full = path.join(key); + assert_eq!(full.len(), 64, "a leaf path is 32 bytes of key"); + self.leaves.insert(B256::from_slice(&full.pack()), value.to_vec()); + } + + /// Carries a whole subtree in full (inline nodes, or a sibling resolved from the node map). + fn expand(&mut self, nodes: &BTreeMap, path: Nibbles, node: &TrieNode) { + match node { + TrieNode::EmptyRoot => {} + TrieNode::Leaf(l) => self.add_leaf(path, &l.key, &l.value), + TrieNode::Extension(e) => { + let child_path = path.join(&e.key); + match e.child.as_hash() { + Some(h) => { + self.hashes.insert(child_path, h); + } + None => { + let child = TrieNode::decode(&mut &e.child[..]).expect("inline child decodes"); + self.expand(nodes, child_path, &child); + } + } + } + TrieNode::Branch(b) => { + for (n, child) in b.as_ref().children() { + let Some(child) = child else { continue }; + let mut p = path; + p.push(n); + match child.as_hash() { + Some(h) => { + self.hashes.insert(p, h); + } + None => { + let c = TrieNode::decode(&mut &child[..]).expect("inline child decodes"); + self.expand(nodes, p, &c); + } + } + } + } + } + } + + /// Walks the path of one target key from the root, keeping what the guest needs to prove the key present or + /// absent and to rebuild the root after any change to the targets. + fn walk(&mut self, nodes: &BTreeMap, key: &Nibbles, targets: &[Nibbles]) { + let mut path = Nibbles::default(); + loop { + let node = Self::node_at(nodes, &path); + match &node { + TrieNode::EmptyRoot => return, + TrieNode::Leaf(l) => { + // The key itself, or the leaf that sits where the key would go (an absence proof). + self.add_leaf(path, &l.key, &l.value); + return; + } + TrieNode::Extension(e) => { + let rest = key.slice(path.len()..); + if rest.starts_with(&e.key) { + path = path.join(&e.key); + if e.child.as_hash().is_none() { + let child = TrieNode::decode(&mut &e.child[..]).expect("inline child decodes"); + self.expand(nodes, path, &child); + return; + } + } else { + // The key diverges inside the extension: the subtree below is never entered. + let child_path = path.join(&e.key); + match e.child.as_hash() { + Some(h) => { + self.hashes.insert(child_path, h); + } + None => { + let child = TrieNode::decode(&mut &e.child[..]).expect("inline child decodes"); + self.expand(nodes, child_path, &child); + } + } + return; + } + } + TrieNode::Branch(b) => { + let depth = path.len(); + let next = key.get(depth).expect("a branch below 64 nibbles"); + let children: Vec<(u8, alloy_trie::nodes::RlpNode)> = b.as_ref().children().filter_map(|(n, c)| c.map(|c| (n, c.clone()))).collect(); + let on_target = |n: u8| -> bool { + targets.iter().any(|t| t.starts_with(&path) && t.get(depth) == Some(n)) + }; + let on_target_count = children.iter().filter(|(n, _)| on_target(*n)).count(); + // After every target under this branch is deleted, one sibling may be left alone and the + // branch collapses into it; that sibling must then be known in full. + let may_collapse = children.len().saturating_sub(on_target_count) <= 1; + let mut found = false; + for (n, child) in &children { + let mut p = path; + p.push(*n); + if *n == next { + found = true; + if child.as_hash().is_none() { + let c = TrieNode::decode(&mut &child[..]).expect("inline child decodes"); + self.expand(nodes, p, &c); + return; + } + continue; + } + if on_target(*n) { + continue; // another target's walk keeps this child + } + match child.as_hash() { + Some(h) => { + if may_collapse { + match Self::node_at(nodes, &p) { + TrieNode::Branch(_) => { + self.hashes.insert(p, h); + } + other => self.expand(nodes, p, &other), + } + } else { + self.hashes.insert(p, h); + } + } + None => { + let c = TrieNode::decode(&mut &child[..]).expect("inline child decodes"); + self.expand(nodes, p, &c); + } + } + } + if !found { + return; // absent: the branch has no child on the key's nibble + } + path.push(next); + } + } + } + } + + fn finish(self) -> TrieWitness { + // A hash under a path that another target entered in full is redundant (the subtree is expanded there). + let leaves: Vec<(B256, Bytes)> = self.leaves.into_iter().map(|(k, v)| (k, Bytes::from(v))).collect(); + let hashes: Vec<(Bytes, B256)> = self + .hashes + .into_iter() + .filter(|(p, _)| !leaves.iter().any(|(k, _)| Nibbles::unpack(k).starts_with(p))) + .map(|(p, h)| (Bytes::from(p.to_vec()), h)) + .collect(); + TrieWitness { leaves, hashes } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::keccak256; + + fn full_root(entries: &BTreeMap>) -> B256 { + if entries.is_empty() { + return EMPTY_ROOT_HASH; + } + let mut hb = HashBuilder::default(); + for (k, v) in entries { + hb.add_leaf(Nibbles::unpack(k), v); + } + hb.root() + } + + fn rng(seed: &mut u64) -> u64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed + } + + #[test] + fn witness_roots_match_the_full_trie_under_updates_inserts_and_deletes() { + let mut seed = 0x9e3779b97f4a7c15u64; + for round in 0..60 { + let n = 1 + (rng(&mut seed) % 300) as usize; + let mut entries: BTreeMap> = BTreeMap::new(); + for i in 0..n { + let key = keccak256((round * 1000 + i).to_be_bytes()); + // Short values (storage style, inline leaves) and long ones (accounts) alike. + let v: Vec = if rng(&mut seed) % 2 == 0 { alloy_rlp::encode(rng(&mut seed) % 255 + 1) } else { alloy_rlp::encode(keccak256(i.to_be_bytes()).as_slice()) }; + entries.insert(key, v); + } + let keys: Vec = entries.keys().copied().collect(); + let t = 1 + (rng(&mut seed) % 12) as usize; + let mut targets: Vec = Vec::new(); + for _ in 0..t { + targets.push(keys[(rng(&mut seed) as usize) % keys.len()]); + } + // Absent keys too: some get inserted. + for j in 0..(rng(&mut seed) % 4) { + targets.push(keccak256((round * 7919 + 500 + j as usize).to_be_bytes())); + } + let w = generate(&entries, &targets); + assert_eq!(w.root(), full_root(&entries), "pre-root, round {round}"); + for k in &targets { + assert!(w.covers(k), "target covered, round {round}"); + assert_eq!(w.leaf(k).map(|v| v.to_vec()), entries.get(k).cloned(), "leaf value, round {round}"); + } + let other = keys.iter().find(|k| !targets.contains(k)); + if let Some(o) = other { + // An untouched key is usually behind a hash; when it is carried in full that is fine too. + let _ = w.covers(o); + } + // Apply changes: update, delete or insert each target. + let mut post = entries.clone(); + let mut changes: BTreeMap>> = BTreeMap::new(); + for (i, k) in targets.iter().enumerate() { + let c = match (rng(&mut seed) % 3, entries.contains_key(k)) { + (0, true) => None, + _ => Some(alloy_rlp::encode(keccak256((i as u64 + 77).to_be_bytes()).as_slice())), + }; + match &c { + None => { + post.remove(k); + } + Some(v) => { + post.insert(*k, v.clone()); + } + } + changes.insert(*k, c); + } + assert_eq!(w.root_with(&changes), full_root(&post), "post-root, round {round}, n {n}, targets {}", targets.len()); + } + } + + #[test] + fn a_tampered_leaf_changes_the_root() { + let mut entries = BTreeMap::new(); + for i in 0..40u64 { + entries.insert(keccak256(i.to_be_bytes()), alloy_rlp::encode(i + 1)); + } + let target = keccak256(3u64.to_be_bytes()); + let mut w = generate(&entries, &[target]); + let root = w.root(); + let i = w.leaves.iter().position(|(k, _)| *k == target).unwrap(); + w.leaves[i].1 = Bytes::from(alloy_rlp::encode(99u64)); + assert_ne!(w.root(), root); + } + + #[test] + fn delete_everything() { + let mut entries = BTreeMap::new(); + for i in 0..5u64 { + entries.insert(keccak256(i.to_be_bytes()), alloy_rlp::encode(i + 1)); + } + let targets: Vec = entries.keys().copied().collect(); + let w = generate(&entries, &targets); + let changes: BTreeMap>> = targets.iter().map(|k| (*k, None)).collect(); + assert_eq!(w.root_with(&changes), EMPTY_ROOT_HASH); + assert!(w.hashes.is_empty()); + } +} diff --git a/proving/igneum-prove/core/src/witness.rs b/proving/igneum-prove/core/src/witness.rs new file mode 100644 index 000000000..b9a3c2b54 --- /dev/null +++ b/proving/igneum-prove/core/src/witness.rs @@ -0,0 +1,150 @@ +//! The shard witness of design 5.1: the accounts, storage slots and trie nodes one shard touches, generated by +//! a full node from its own execution (`generate`) and checked by the guest against the shard's pre-root +//! (`load`), with the post-root rebuilt from the same trie nodes after execution (`post_root`). + +use crate::state::{AccessLog, IgneumDb}; +use crate::trie::{self, TrieWitness}; +use alloy_primitives::{keccak256, Address, Bytes, B256, U256}; +use alloy_rlp::Decodable; +use alloy_trie::{TrieAccount, EMPTY_ROOT_HASH, KECCAK_EMPTY}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct AccountWitness { + pub address: Address, + /// The account's code when it has any (checked against the leaf's code hash). + pub code: Bytes, + /// The storage slots the shard touches (their values are the storage witness's leaves). + pub slots: Vec, + pub storage: TrieWitness, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct StateWitness { + pub accounts: Vec, + /// The account trie: the touched accounts' leaves and the hashes of everything else. + pub trie: TrieWitness, +} + +impl StateWitness { + pub fn stats(&self) -> (usize, usize, usize, usize) { + let slots = self.accounts.iter().map(|a| a.slots.len()).sum(); + let leaves = self.trie.leaves.len() + self.accounts.iter().map(|a| a.storage.leaves.len()).sum::(); + let hashes = self.trie.hashes.len() + self.accounts.iter().map(|a| a.storage.hashes.len()).sum::(); + (self.accounts.len(), slots, leaves, hashes) + } +} + +/// What the strict database may read: every witnessed account and, per account, the slots its storage +/// witness proves (all of them when the storage trie is empty). +#[derive(Clone, Debug, Default)] +pub struct Coverage { + accounts: BTreeMap, +} + +impl Coverage { + pub fn account_covered(&self, address: &Address) -> bool { + self.accounts.contains_key(address) + } + pub fn slot_covered(&self, address: &Address, slot: &U256) -> bool { + self.accounts.get(address).map(|w| w.covers(&keccak256(slot.to_be_bytes::<32>()))).unwrap_or(false) + } +} + +fn storage_value_rlp(value: U256) -> Vec { + alloy_rlp::encode_fixed_size(&value).to_vec() +} + +/// Generates the witness for a shard from the full pre-state and the access log of its native execution. +pub fn generate(pre: &IgneumDb, log: &AccessLog) -> StateWitness { + assert!(pre.log.is_none() && pre.strict.is_none(), "generate from a plain full state"); + let mut entries: BTreeMap> = BTreeMap::new(); + for address in pre.addresses() { + if let Some(a) = pre.trie_account(address) { + entries.insert(keccak256(address), alloy_rlp::encode(a)); + } + } + let targets: Vec = log.accounts.iter().map(keccak256).collect(); + let trie = trie::generate(&entries, &targets); + + let mut accounts = Vec::with_capacity(log.accounts.len()); + for address in &log.accounts { + let slots: Vec = log.slots.range((*address, U256::ZERO)..=(*address, U256::MAX)).map(|(_, s)| *s).collect(); + let storage_entries: BTreeMap> = pre.storage_of(*address).into_iter().map(|(k, v)| (keccak256(k.to_be_bytes::<32>()), storage_value_rlp(v))).collect(); + let storage = if storage_entries.is_empty() { + TrieWitness::default() + } else { + let slot_targets: Vec = slots.iter().map(|s| keccak256(s.to_be_bytes::<32>())).collect(); + trie::generate(&storage_entries, &slot_targets) + }; + accounts.push(AccountWitness { address: *address, code: Bytes::from(pre.code(*address)), slots, storage }); + } + StateWitness { accounts, trie } +} + +/// Builds the strict database from the witness and returns it with the pre-root the witness commits to. +/// Panics on any inconsistency: a leaf that does not match its account, a storage witness whose root is not the +/// account's, code whose hash is not the leaf's, an account the trie does not cover. +pub fn load(w: &StateWitness, block_hashes: &[(u64, B256)]) -> (IgneumDb, B256) { + let pre_root = w.trie.root(); + let mut db = IgneumDb::new(); + let mut coverage = Coverage::default(); + for a in &w.accounts { + let hashed = keccak256(a.address); + assert!(w.trie.covers(&hashed), "account {} is not covered by the witness", a.address); + match w.trie.leaf(&hashed) { + Some(mut leaf) => { + let acc = TrieAccount::decode(&mut leaf).expect("account leaf decodes"); + assert_eq!(a.storage.root(), acc.storage_root, "storage witness root of {}", a.address); + if acc.code_hash == KECCAK_EMPTY { + assert!(a.code.is_empty(), "code carried for an account without code"); + } else { + assert_eq!(keccak256(&a.code), acc.code_hash, "code hash of {}", a.address); + } + let mut storage = Vec::with_capacity(a.slots.len()); + for slot in &a.slots { + let key = keccak256(slot.to_be_bytes::<32>()); + assert!(a.storage.covers(&key), "slot {slot} of {} is not covered", a.address); + let value = match a.storage.leaf(&key) { + Some(mut v) => U256::decode(&mut v).expect("storage leaf decodes"), + None => U256::ZERO, + }; + storage.push((*slot, value)); + } + db.insert_account(a.address, acc.nonce, acc.balance, &a.code, &storage); + } + None => { + assert_eq!(a.storage.root(), EMPTY_ROOT_HASH, "an absent account has no storage"); + assert!(a.code.is_empty(), "an absent account has no code"); + } + } + coverage.accounts.insert(a.address, a.storage.clone()); + } + for (n, h) in block_hashes { + db.push_block_hash(*n, *h); + } + db.strict = Some(coverage); + (db, pre_root) +} + +/// The state root after execution, from the witness's trie nodes and the database's final accounts. +pub fn post_root(db: &IgneumDb, w: &StateWitness) -> B256 { + let mut changes: BTreeMap>> = BTreeMap::new(); + for a in &w.accounts { + let leaf = db.account_raw(a.address).and_then(|info| { + let mut slot_changes: BTreeMap>> = BTreeMap::new(); + for slot in &a.slots { + let v = db.storage_raw(a.address, *slot); + slot_changes.insert(keccak256(slot.to_be_bytes::<32>()), (!v.is_zero()).then(|| storage_value_rlp(v))); + } + let storage_root = a.storage.root_with(&slot_changes); + if info.balance.is_zero() && info.nonce == 0 && info.code_hash == KECCAK_EMPTY && storage_root == EMPTY_ROOT_HASH { + return None; + } + Some(alloy_rlp::encode(TrieAccount { nonce: info.nonce, balance: info.balance, storage_root, code_hash: info.code_hash })) + }); + changes.insert(keccak256(a.address), leaf); + } + w.trie.root_with(&changes) +}