From 8b646a10d4bb2b4f4720fdfe661e03cb4d33e89f Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:51:13 +0000 Subject: [PATCH] fin-proof: the Merkle key table (a slot per key, one path per touched key in the fold, the full table once per certificate over the dense prefix) Co-Authored-By: Claude Fable 5.1 --- proving/igneum-prove/core/src/agg.rs | 8 +- proving/igneum-prove/fin/src/cert.rs | 38 ++-- proving/igneum-prove/fin/src/fold.rs | 83 ++++--- proving/igneum-prove/fin/src/keys.rs | 260 ++++++++++++++++++++++ proving/igneum-prove/fin/src/lib.rs | 27 +-- proving/igneum-prove/fin/src/mmr.rs | 8 +- proving/igneum-prove/fin/tests/harness.rs | 89 +++++--- proving/igneum-prove/host/src/fin.rs | 50 +++-- proving/igneum-prove/host/src/main.rs | 8 +- 9 files changed, 432 insertions(+), 139 deletions(-) create mode 100644 proving/igneum-prove/fin/src/keys.rs diff --git a/proving/igneum-prove/core/src/agg.rs b/proving/igneum-prove/core/src/agg.rs index d6ec0faee..597ab4287 100644 --- a/proving/igneum-prove/core/src/agg.rs +++ b/proving/igneum-prove/core/src/agg.rs @@ -6,6 +6,7 @@ use crate::executor::Carry; use crate::shard::ShardOutput; use alloy_primitives::{keccak256, B256}; use igneum_fin_core::fold::{fold_block, ChainBlockWitness, FoldWitness}; +use igneum_fin_core::keys::{KeyLeafWitness, WitnessKeys}; use igneum_fin_core::ring::{RingLeafWitness, WitnessRing}; use igneum_fin_core::{FinExt, FinParams, FinState}; use serde::{Deserialize, Serialize}; @@ -34,6 +35,8 @@ pub struct FinInput { pub prev_state: FinState, pub block: ChainBlockWitness, pub ring: Vec, + #[serde(default)] + pub keys: Vec, pub witness: FoldWitness, } @@ -215,8 +218,9 @@ pub fn aggregate(input: &AggInput, verify: &mut dyn FnMut(&[u32; 8], &[u8])) -> assert_eq!(f.block.number, first.number, "the finality witness is of this chain block"); assert_eq!(f.block.block_hash.as_slice(), first.block_hash.as_slice(), "the finality witness names this chain block"); let mut ring = WitnessRing::new(f.ring.clone()); - fold_block(&mut state, &f.params, &f.block, &mut ring, &f.witness, &igneum_fin_core::bls::ZkBls).expect("the finality fold"); - assert!(ring.witnesses.is_empty(), "every ring witness consumed"); + let mut keys = WitnessKeys::new(f.keys.clone()); + fold_block(&mut state, &f.params, &f.block, &mut ring, &mut keys, &f.witness, &igneum_fin_core::bls::ZkBls).expect("the finality fold"); + assert!(ring.witnesses.is_empty() && keys.witnesses.is_empty(), "every witness consumed"); state.extension() }); BlockOutput { diff --git a/proving/igneum-prove/fin/src/cert.rs b/proving/igneum-prove/fin/src/cert.rs index 16d196452..871745576 100644 --- a/proving/igneum-prove/fin/src/cert.rs +++ b/proving/igneum-prove/fin/src/cert.rs @@ -5,15 +5,16 @@ use crate::bls::Bls; use crate::mmr::{HistoryLeaf, MmrProof}; -use crate::{keys_hash, vote_message, voters_at, FinParams, FinState, KeyEntry, Lock, H, SIG_LEN}; +use crate::keys::{canonical, dense_root}; +use crate::{vote_message, voters_at, FinParams, FinState, KeyEntry, Lock, H, SIG_LEN}; use serde::{Deserialize, Serialize}; -/// A table at a chain block: the leaf that commits it and the entries. +/// A table at a chain block: the leaf that commits it and every entry with its slot, slot order. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct TableAt { pub leaf: HistoryLeaf, pub proof: MmrProof, - pub keys: Vec, + pub keys: Vec<(u64, KeyEntry)>, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -49,17 +50,16 @@ pub fn signer_positions(bitmap: &[u8], voter_count: u32) -> Vec { out } -fn check_table(state: &FinState, t: &TableAt, what: &str) -> Result<(), String> { +/// The table's entries against the leaf the history holds for that block: the dense root over the slots must be +/// the leaf's. Returns the canonical (sorted, duplicate-free) list. +fn check_table(state: &FinState, t: &TableAt, what: &str) -> Result, String> { if !t.proof.verify(&t.leaf.hash(), &state.history_root(), state.leaves) { return Err(format!("the {what} block is not in the proof's history")); } - if keys_hash(&t.keys) != t.leaf.keys_hash { + if dense_root(&t.keys, t.leaf.key_count)? != t.leaf.keys_root { return Err(format!("the {what} table is not the one the proof committed at that block")); } - if !t.keys.windows(2).all(|p| p[0].key_hash < p[1].key_hash) { - return Err(format!("the {what} table is not sorted")); - } - Ok(()) + canonical(&t.keys).map_err(|e| format!("the {what} table: {e}")) } pub fn verify_certificate(state: &mut FinState, params: &FinParams, c: &CertificateWitness, bls: &dyn Bls) -> Result<(), String> { @@ -69,7 +69,7 @@ pub fn verify_certificate(state: &mut FinState, params: &FinParams, c: &Certific if c.index <= state.lock.index { return Err(format!("certificate index {} is not above the last lock {}", c.index, state.lock.index)); } - check_table(state, &c.at, "checkpoint")?; + let at_keys = check_table(state, &c.at, "checkpoint")?; let leaf = &c.at.leaf; if leaf.block_hash != c.checkpoint { return Err("the certificate's checkpoint is not the block the history leaf names".into()); @@ -84,13 +84,10 @@ pub fn verify_certificate(state: &mut FinState, params: &FinParams, c: &Certific return Err(format!("checkpoint DAA score {} is below min_daa {} (C5)", leaf.daa, params.min_daa)); } // the canonical voter list at the checkpoint and the signers - let (voters, total) = voters_at(&c.at.keys, leaf.daa, params.dust); + let (voters, total) = voters_at(&at_keys, leaf.daa, params.dust); if voters.len() != c.voter_count as usize { return Err(format!("the certificate names {} voters, the table at the checkpoint has {}", c.voter_count, voters.len())); } - if total != leaf.total { - return Err("the table's total differs from the leaf's".into()); - } let positions = signer_positions(&c.bitmap, c.voter_count); if positions.is_empty() { return Err("the certificate has no signer".into()); @@ -99,7 +96,7 @@ pub fn verify_certificate(state: &mut FinState, params: &FinParams, c: &Certific let mut pubkeys = Vec::with_capacity(positions.len()); let mut signer_hashes: Vec = Vec::with_capacity(positions.len()); for p in &positions { - let k = &c.at.keys[voters[*p]]; + let k = &at_keys[voters[*p]]; if !k.revealed() { return Err("a signer's key is not revealed".into()); } @@ -130,20 +127,17 @@ pub fn verify_certificate(state: &mut FinState, params: &FinParams, c: &Certific // Q5, never expiring in the proof let (frozen_signed, frozen_total) = if state.lock.index > 0 { let f = c.frozen.as_ref().ok_or("a lock exists and the frozen table is missing")?; - check_table(state, f, "frozen")?; + let f_keys = check_table(state, f, "frozen")?; if f.leaf.number != state.lock.number || f.leaf.block_hash != state.lock.hash { return Err("the frozen table is not the table at the last lock's block".into()); } - let (fvoters, ftotal) = voters_at(&f.keys, f.leaf.daa, params.dust); - if ftotal != f.leaf.total { - return Err("the frozen table's total differs from its leaf's".into()); - } + let (fvoters, ftotal) = voters_at(&f_keys, f.leaf.daa, params.dust); // keys gone at the checkpoint (their leaves carried after the lock) leave the frozen denominator - let gone = |kh: &H| c.at.keys.binary_search_by(|k| k.key_hash.cmp(kh)).ok().map(|i| c.at.keys[i].is_gone(leaf.daa)).unwrap_or(false); + let gone = |kh: &H| at_keys.binary_search_by(|k| k.key_hash.cmp(kh)).ok().map(|i| at_keys[i].is_gone(leaf.daa)).unwrap_or(false); let mut left = 0u64; let mut fsigned = 0u64; for &i in &fvoters { - let k = &f.keys[i]; + let k = &f_keys[i]; if gone(&k.key_hash) { left += k.blocks; } else if signer_hashes.binary_search(&k.key_hash).is_ok() { diff --git a/proving/igneum-prove/fin/src/fold.rs b/proving/igneum-prove/fin/src/fold.rs index 5e1f3517a..beb749b19 100644 --- a/proving/igneum-prove/fin/src/fold.rs +++ b/proving/igneum-prove/fin/src/fold.rs @@ -7,7 +7,8 @@ use crate::cert::{verify_certificate, CertificateWitness}; use crate::header::{outranks, HeaderWitness}; use crate::mmr::{self, HistoryLeaf}; use crate::ring::{slot_of, RingAccess, RingEntry, RING_LEAF_DAA}; -use crate::{find_key, vote_key_hash, vote_message, voters_at, FinParams, FinState, KeyEntry, H, PUBKEY_LEN, SIG_LEN}; +use crate::keys::KeyAccess; +use crate::{vote_key_hash, vote_message, FinParams, FinState, KeyEntry, H, PUBKEY_LEN, SIG_LEN}; use serde::{Deserialize, Serialize}; /// One blue block of the chain block's past counted for its key: the chain block itself and its mergeset blues @@ -85,18 +86,28 @@ pub struct FoldReport { pub locks: u64, } -fn touch(keys: &mut Vec, key_hash: &H) -> usize { - match find_key(keys, key_hash) { - Ok(i) => i, - Err(i) => { - keys.insert(i, KeyEntry::new(*key_hash)); - i +/// Opens the key's leaf, applies `f` to its entry (a fresh one when the key has no slot), writes it back (an entry +/// that holds nothing at `daa` empties its leaf; the slot is never reused). +fn touch(state: &mut FinState, keys: &mut dyn KeyAccess, key: &H, daa: u64, f: &mut dyn FnMut(&mut KeyEntry) -> Result<(), String>) -> Result<(), String> { + let (slot, entry, siblings) = keys.touch(key, &state.keys_root, state.key_count)?; + let mut e = match entry { + Some(e) => e, + None => { + if slot != state.key_count { + return Err("a new key must take the next free slot".into()); + } + state.key_count += 1; + KeyEntry::new(*key) } - } + }; + f(&mut e)?; + let out = if e.is_empty_at(daa) { None } else { Some(e) }; + state.keys_root = keys.write(slot, out, &siblings); + Ok(()) } /// Folds chain block `block` into `state`. Errors make the proof impossible (the guest panics on them). -pub fn fold_block(state: &mut FinState, params: &FinParams, block: &ChainBlockWitness, ring: &mut dyn RingAccess, witness: &FoldWitness, bls: &dyn Bls) -> Result { +pub fn fold_block(state: &mut FinState, params: &FinParams, block: &ChainBlockWitness, ring: &mut dyn RingAccess, keys: &mut dyn KeyAccess, witness: &FoldWitness, bls: &dyn Bls) -> Result { if state.params_hash != params.hash() { return Err("the state was computed under other finality parameters".into()); } @@ -133,8 +144,10 @@ pub fn fold_block(state: &mut FinState, params: &FinParams, block: &ChainBlockWi let pos = entries.binary_search(&e).unwrap_err(); entries.insert(pos, e); state.ring_root = ring.write(slot, entries, &siblings); - let i = touch(&mut state.keys, &b.key_hash); - state.keys[i].blocks += 1; + touch(state, keys, &b.key_hash, block.daa, &mut |e| { + e.blocks += 1; + Ok(()) + })?; report.counted += 1; } for r in &reds { @@ -163,8 +176,10 @@ pub fn fold_block(state: &mut FinState, params: &FinParams, block: &ChainBlockWi let (gone, kept): (Vec, Vec) = entries.into_iter().partition(|e| e.daa <= window_start); if !gone.is_empty() { for g in gone.iter().filter(|g| g.blue) { - let i = find_key(&state.keys, &g.key_hash).map_err(|_| format!("expired block {} names a key not in the table", hex32(&g.block_hash)))?; - state.keys[i].blocks = state.keys[i].blocks.checked_sub(1).ok_or("a key's block count went below zero")?; + touch(state, keys, &g.key_hash, block.daa, &mut |e| { + e.blocks = e.blocks.checked_sub(1).ok_or("a key's block count went below zero")?; + Ok(()) + })?; report.expired += 1; } state.ring_root = ring.write(slot, kept, &siblings); @@ -179,11 +194,13 @@ pub fn fold_block(state: &mut FinState, params: &FinParams, block: &ChainBlockWi return Err("a key reveal's proof of possession does not verify".into()); } let kh = vote_key_hash(&r.pubkey); - let i = touch(&mut state.keys, &kh); - if state.keys[i].revealed() && state.keys[i].pubkey != r.pubkey { - return Err("a key reveal names another key for a revealed hash".into()); - } - state.keys[i].pubkey = r.pubkey; + touch(state, keys, &kh, block.daa, &mut |e| { + if e.revealed() && e.pubkey != r.pubkey { + return Err("a key reveal names another key for a revealed hash".into()); + } + e.pubkey = r.pubkey; + Ok(()) + })?; } for e in &witness.evidence { if e.hash_a == e.hash_b { @@ -198,12 +215,14 @@ pub fn fold_block(state: &mut FinState, params: &FinParams, block: &ChainBlockWi return Err("an equivocation vote does not verify".into()); } let kh = vote_key_hash(&e.pubkey); - let i = touch(&mut state.keys, &kh); let until = e.carrier_daa.saturating_add(params.equivocation_ban); - state.keys[i].ban_until = state.keys[i].ban_until.max(until); - if !state.keys[i].revealed() { - state.keys[i].pubkey = e.pubkey; - } + touch(state, keys, &kh, block.daa, &mut |k| { + k.ban_until = k.ban_until.max(until); + if !k.revealed() { + k.pubkey = e.pubkey; + } + Ok(()) + })?; } for l in &witness.leaves { if l.carrier_daa > block.daa { @@ -213,11 +232,14 @@ pub fn fold_block(state: &mut FinState, params: &FinParams, block: &ChainBlockWi return Err("a leave does not verify".into()); } let kh = vote_key_hash(&l.pubkey); - let i = touch(&mut state.keys, &kh); - if state.keys[i].leave_until == 0 { - state.keys[i].leave_from = l.carrier_daa.saturating_add(params.leave_delay); - state.keys[i].leave_until = l.carrier_daa.saturating_add(params.weight_window); - } + let (from, until) = (l.carrier_daa.saturating_add(params.leave_delay), l.carrier_daa.saturating_add(params.weight_window)); + touch(state, keys, &kh, block.daa, &mut |k| { + if k.leave_until == 0 { + k.leave_from = from; + k.leave_until = until; + } + Ok(()) + })?; } // 4. the certificates that landed with this block, against the history as it stood before it (every witness @@ -226,6 +248,7 @@ pub fn fold_block(state: &mut FinState, params: &FinParams, block: &ChainBlockWi verify_certificate(state, params, c, bls)?; report.locks += 1; } + // slots whose entries emptied before this fold's touches stay empty; nothing to retain // 5. staleness: a window without a lock (the frozen table never expires in the proof, design 4.4) if state.lock.index > 0 && block.daa >= state.lock.daa.saturating_add(params.weight_window) { @@ -240,9 +263,7 @@ pub fn fold_block(state: &mut FinState, params: &FinParams, block: &ChainBlockWi state.end_blue_score = h.blue_score; state.end_blue_work = h.blue_work.clone(); } - state.keys.retain(|k| !k.is_empty_at(block.daa)); - let (_, total) = voters_at(&state.keys, block.daa, params.dust); - let leaf = HistoryLeaf { number: block.number, block_hash: block.block_hash, daa: block.daa, table_root: state.table_root(), keys_hash: state.keys_hash(), total }; + let leaf = HistoryLeaf { number: block.number, block_hash: block.block_hash, daa: block.daa, table_root: state.table_root(), keys_root: state.keys_root, key_count: state.key_count }; mmr::append(&mut state.peaks, &mut state.leaves, leaf.hash()); Ok(report) } diff --git a/proving/igneum-prove/fin/src/keys.rs b/proving/igneum-prove/fin/src/keys.rs new file mode 100644 index 000000000..4afce7ac0 --- /dev/null +++ b/proving/igneum-prove/fin/src/keys.rs @@ -0,0 +1,260 @@ +//! The key table as a Merkle tree (docs/design/finality-in-proof.md section 2, the scale form): 2^KEY_DEPTH leaf +//! slots, a key takes the next free slot the first time it is seen and keeps it until its entry empties (a slot is +//! never reused), so the non-empty leaves are a prefix `0..key_count` and the fold touches one path per key it +//! changes instead of hashing the whole table at every block. The certificate step takes every entry once and +//! rebuilds the root over that dense prefix (about 2N hashes at N keys), sorts by key hash for the canonical voter +//! list of spec 3.10 C3 and refuses a key that appears twice. +//! +//! What a lying witness can do: insert a key a second time at a fresh slot (the guest cannot know the key has a +//! slot already). The split is refused at the next certificate (duplicate key hashes in the full table), so no +//! certificate verifies while it stands, and the native compare vetoes the record on the chain. + +use crate::{sha256, KeyEntry, H, ZERO}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +pub const KEY_DEPTH: usize = 24; + +pub fn leaf_hash(entry: Option<&KeyEntry>) -> H { + match entry { + None => ZERO, + Some(e) => { + let mut buf = Vec::with_capacity(1 + KeyEntry::LEN); + buf.push(5u8); + e.write(&mut buf); + sha256(&[&buf]) + } + } +} + +pub fn node_hash(left: &H, right: &H) -> H { + sha256(&[&[6u8], left, right]) +} + +pub fn defaults() -> Vec { + let mut d = Vec::with_capacity(KEY_DEPTH + 1); + d.push(ZERO); + for l in 1..=KEY_DEPTH { + let below = d[l - 1]; + d.push(node_hash(&below, &below)); + } + d +} + +pub fn empty_root() -> H { + defaults()[KEY_DEPTH] +} + +pub fn root_from_path(slot: u64, leaf: H, siblings: &[H]) -> H { + assert_eq!(siblings.len(), KEY_DEPTH, "a key path has {KEY_DEPTH} siblings"); + let mut h = leaf; + let mut idx = slot; + for s in siblings { + h = if idx & 1 == 0 { node_hash(&h, s) } else { node_hash(s, &h) }; + idx >>= 1; + } + h +} + +/// The root over the dense prefix: `entries` in slot order (every non-empty leaf below `key_count`), empty slots +/// between them the zero leaf, everything from `key_count` on the default subtrees. +pub fn dense_root(entries: &[(u64, KeyEntry)], key_count: u64) -> Result { + let d = defaults(); + let mut level: Vec = Vec::with_capacity(key_count as usize); + let mut next_slot = 0u64; + for (slot, e) in entries { + if *slot < next_slot || *slot >= key_count { + return Err(format!("key table entries out of order or past key_count ({slot}, {key_count})")); + } + while next_slot < *slot { + level.push(ZERO); + next_slot += 1; + } + level.push(leaf_hash(Some(e))); + next_slot += 1; + } + while next_slot < key_count { + level.push(ZERO); + next_slot += 1; + } + for l in 0..KEY_DEPTH { + if level.is_empty() { + return Ok(d[KEY_DEPTH]); + } + let mut next = Vec::with_capacity(level.len().div_ceil(2)); + for i in (0..level.len()).step_by(2) { + let r = level.get(i + 1).copied().unwrap_or(d[l]); + next.push(node_hash(&level[i], &r)); + } + level = next; + } + Ok(level[0]) +} + +/// The canonical voter order: by key hash, no key twice. +pub fn canonical(entries: &[(u64, KeyEntry)]) -> Result, String> { + let mut v: Vec = entries.iter().map(|(_, e)| e.clone()).collect(); + v.sort_by(|a, b| a.key_hash.cmp(&b.key_hash)); + if v.windows(2).any(|p| p[0].key_hash == p[1].key_hash) { + return Err("a key appears twice in the table".into()); + } + Ok(v) +} + +/// One key leaf opened for the guest: the slot the key sits at (or the next free slot when absent), its entry and +/// the siblings as they stand at that moment of the fold. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct KeyLeafWitness { + pub slot: u64, + pub entry: Option, + pub siblings: Vec, +} + +pub trait KeyAccess { + /// Opens the leaf of `key`: its slot, its entry (None when the key has no slot: the slot is then `key_count`, + /// the next free one) and the siblings, checked against `root`. + fn touch(&mut self, key: &H, root: &H, key_count: u64) -> Result<(u64, Option, Vec), String>; + /// Writes `entry` (None empties the leaf) to `slot` and returns the new root. + fn write(&mut self, slot: u64, entry: Option, siblings: &[H]) -> H; +} + +/// The guest's table: a queue of witnesses. +pub struct WitnessKeys { + pub witnesses: std::collections::VecDeque, +} + +impl WitnessKeys { + pub fn new(witnesses: Vec) -> Self { + Self { witnesses: witnesses.into() } + } +} + +impl KeyAccess for WitnessKeys { + fn touch(&mut self, key: &H, root: &H, key_count: u64) -> Result<(u64, Option, Vec), String> { + let w = self.witnesses.pop_front().ok_or("key witness missing")?; + match &w.entry { + Some(e) => { + if e.key_hash != *key || w.slot >= key_count { + return Err("key witness names another key or a slot past the table".into()); + } + } + None => { + if w.slot != key_count { + return Err(format!("a new key takes slot {key_count}, the witness names {}", w.slot)); + } + } + } + if root_from_path(w.slot, leaf_hash(w.entry.as_ref()), &w.siblings) != *root { + return Err("key witness does not open the table root".into()); + } + Ok((w.slot, w.entry, w.siblings)) + } + + fn write(&mut self, slot: u64, entry: Option, siblings: &[H]) -> H { + root_from_path(slot, leaf_hash(entry.as_ref()), siblings) + } +} + +/// The native table (the tracker, the host, the tests): every entry by slot, the key index, a sparse node map, +/// and the witnesses recorded for the guest. +#[derive(Clone, Debug)] +pub struct SparseKeys { + defaults: Vec, + nodes: HashMap<(u8, u64), H>, + entries: Vec>, + index: HashMap, + pub recorded: Vec, +} + +impl Default for SparseKeys { + fn default() -> Self { + Self::new() + } +} + +impl SparseKeys { + pub fn new() -> Self { + Self { defaults: defaults(), nodes: HashMap::new(), entries: Vec::new(), index: HashMap::new(), recorded: Vec::new() } + } + + fn node(&self, level: u8, idx: u64) -> H { + self.nodes.get(&(level, idx)).copied().unwrap_or(self.defaults[level as usize]) + } + + pub fn root(&self) -> H { + self.node(KEY_DEPTH as u8, 0) + } + + pub fn key_count(&self) -> u64 { + self.entries.len() as u64 + } + + pub fn siblings(&self, slot: u64) -> Vec { + let mut idx = slot; + let mut out = Vec::with_capacity(KEY_DEPTH); + for level in 0..KEY_DEPTH as u8 { + out.push(self.node(level, idx ^ 1)); + idx >>= 1; + } + out + } + + pub fn get(&self, key: &H) -> Option<&KeyEntry> { + self.index.get(key).and_then(|&s| self.entries[s as usize].as_ref()) + } + + /// Every non-empty entry with its slot, slot order (the certificate step's table). + pub fn all_entries(&self) -> Vec<(u64, KeyEntry)> { + self.entries.iter().enumerate().filter_map(|(i, e)| e.as_ref().map(|e| (i as u64, e.clone()))).collect() + } + + pub fn set(&mut self, slot: u64, entry: Option) { + if slot as usize >= self.entries.len() { + self.entries.resize(slot as usize + 1, None); + } + if let Some(old) = &self.entries[slot as usize] { + self.index.remove(&old.key_hash); + } + if let Some(e) = &entry { + self.index.insert(e.key_hash, slot); + } + let mut h = leaf_hash(entry.as_ref()); + self.entries[slot as usize] = entry; + let mut idx = slot; + for level in 0..=KEY_DEPTH as u8 { + if h == self.defaults[level as usize] { + self.nodes.remove(&(level, idx)); + } else { + self.nodes.insert((level, idx), h); + } + if level == KEY_DEPTH as u8 { + break; + } + let sib = self.node(level, idx ^ 1); + h = if idx & 1 == 0 { node_hash(&h, &sib) } else { node_hash(&sib, &h) }; + idx >>= 1; + } + } + + pub fn take_witnesses(&mut self) -> Vec { + std::mem::take(&mut self.recorded) + } +} + +impl KeyAccess for SparseKeys { + fn touch(&mut self, key: &H, root: &H, key_count: u64) -> Result<(u64, Option, Vec), String> { + if self.root() != *root || self.key_count() != key_count { + return Err("the native key table differs from the state's".into()); + } + let slot = self.index.get(key).copied().unwrap_or(key_count); + let entry = if slot < key_count { self.entries[slot as usize].clone() } else { None }; + let siblings = self.siblings(slot); + self.recorded.push(KeyLeafWitness { slot, entry: entry.clone(), siblings: siblings.clone() }); + Ok((slot, entry, siblings)) + } + + fn write(&mut self, slot: u64, entry: Option, _siblings: &[H]) -> H { + self.set(slot, entry); + self.root() + } +} diff --git a/proving/igneum-prove/fin/src/lib.rs b/proving/igneum-prove/fin/src/lib.rs index 59948d753..23433ae6f 100644 --- a/proving/igneum-prove/fin/src/lib.rs +++ b/proving/igneum-prove/fin/src/lib.rs @@ -13,6 +13,7 @@ pub mod bls; pub mod cert; pub mod fold; pub mod header; +pub mod keys; pub mod mmr; pub mod ring; pub mod tracker; @@ -165,16 +166,6 @@ impl KeyEntry { } } -/// The key table: sorted by key hash, which is the canonical voter order of spec 3.10 C3. -pub fn keys_hash(keys: &[KeyEntry]) -> H { - let mut buf = Vec::with_capacity(8 + keys.len() * KeyEntry::LEN); - buf.extend_from_slice(&(keys.len() as u64).to_le_bytes()); - for k in keys { - k.write(&mut buf); - } - sha256(&[b"igneum-fin-keys-v1", &buf]) -} - /// The canonical voter list at `daa` (positions index a certificate's bitmap) and the total weight. pub fn voters_at(keys: &[KeyEntry], daa: u64, dust: u64) -> (Vec, u64) { let mut positions = Vec::new(); @@ -223,8 +214,9 @@ pub struct FinState { pub end_blue_work: Vec, /// The first chain block this attestation counted from (section 1 of the design: `history_first`). pub history_first: u64, - /// Sorted by key hash. - pub keys: Vec, + /// The key table's Merkle root and the number of slots taken (`keys`). + pub keys_root: H, + pub key_count: u64, /// Root of the block ring (`ring`). pub ring_root: H, /// The history MMR's peaks, left to right, and its leaf count (`mmr`). @@ -310,7 +302,8 @@ impl FinState { end_blue_score: 0, end_blue_work: Vec::new(), history_first: first_number, - keys: Vec::new(), + keys_root: keys::empty_root(), + key_count: 0, ring_root: ring::empty_root(), peaks: Vec::new(), leaves: 0, @@ -319,13 +312,9 @@ impl FinState { } } - pub fn keys_hash(&self) -> H { - keys_hash(&self.keys) - } - - /// `sha256("igneum-fin-state-v1" || params_hash || end_daa || end_number || keys_hash || ring_root)`. + /// `sha256("igneum-fin-state-v1" || params || end block || keys_root || key_count || ring_root)`. pub fn table_root(&self) -> H { - sha256(&[b"igneum-fin-state-v1", &self.params_hash, &self.end_daa.to_le_bytes(), &self.end_number.to_le_bytes(), &self.end_hash, &self.end_blue_score.to_le_bytes(), &(self.end_blue_work.len() as u64).to_le_bytes(), &self.end_blue_work, &self.keys_hash(), &self.ring_root]) + sha256(&[b"igneum-fin-state-v1", &self.params_hash, &self.end_daa.to_le_bytes(), &self.end_number.to_le_bytes(), &self.end_hash, &self.end_blue_score.to_le_bytes(), &(self.end_blue_work.len() as u64).to_le_bytes(), &self.end_blue_work, &self.keys_root, &self.key_count.to_le_bytes(), &self.ring_root]) } pub fn history_root(&self) -> H { diff --git a/proving/igneum-prove/fin/src/mmr.rs b/proving/igneum-prove/fin/src/mmr.rs index 936a5d8b4..add19b070 100644 --- a/proving/igneum-prove/fin/src/mmr.rs +++ b/proving/igneum-prove/fin/src/mmr.rs @@ -12,14 +12,14 @@ pub struct HistoryLeaf { pub daa: u64, /// The table root (keys and ring) after this block was folded. pub table_root: H, - /// The keys hash alone, so a certificate's table witness can be checked without the ring. - pub keys_hash: H, - pub total: u64, + /// The key table's root and slot count alone, so a certificate's table witness is checked without the ring. + pub keys_root: H, + pub key_count: u64, } impl HistoryLeaf { pub fn hash(&self) -> H { - sha256(&[&[4u8], &self.number.to_le_bytes(), &self.block_hash, &self.daa.to_le_bytes(), &self.table_root, &self.keys_hash, &self.total.to_le_bytes()]) + sha256(&[&[4u8], &self.number.to_le_bytes(), &self.block_hash, &self.daa.to_le_bytes(), &self.table_root, &self.keys_root, &self.key_count.to_le_bytes()]) } } diff --git a/proving/igneum-prove/fin/tests/harness.rs b/proving/igneum-prove/fin/tests/harness.rs index f5ec8dbb1..1dd18c95e 100644 --- a/proving/igneum-prove/fin/tests/harness.rs +++ b/proving/igneum-prove/fin/tests/harness.rs @@ -10,9 +10,10 @@ use igneum_fin_core::bls::{Bls, ZkBls}; use igneum_fin_core::cert::{signer_positions, CertificateWitness, TableAt}; use igneum_fin_core::fold::{fold_block, BlueBlock, ChainBlockWitness, FoldWitness, LeaveWitness, Reveal}; use igneum_fin_core::mmr::{HistoryLeaf, Mmr}; +use igneum_fin_core::keys::{canonical, SparseKeys, WitnessKeys}; use igneum_fin_core::ring::WitnessRing; use igneum_fin_core::tracker::SparseRing; -use igneum_fin_core::{keys_hash, vote_key_hash, vote_message, voters_at, FinExt, FinParams, FinState, KeyEntry, DST_LEAVE, DST_POP, DST_VOTE, H, PUBKEY_LEN, SIG_LEN}; +use igneum_fin_core::{vote_key_hash, vote_message, voters_at, FinExt, FinParams, FinState, KeyEntry, DST_LEAVE, DST_POP, DST_VOTE, H, PUBKEY_LEN, SIG_LEN}; use std::collections::HashMap; struct Key { @@ -71,11 +72,12 @@ struct Sim { keys: Vec, /// (number, hash, daa, blues) per chain block blocks: Vec, - /// the full table after each block (number -> keys) and the history leaves, from the tracker's fold - tables: HashMap>, + /// the full table after each block (number -> entries by slot) and the history leaves, from the tracker's fold + tables: HashMap>, mmr: Mmr, leaves: HashMap, tracker: SparseRing, + keytree: SparseKeys, state: FinState, bls: BlstBls, } @@ -84,7 +86,7 @@ impl Sim { fn new(params: FinParams, labels: &[&str]) -> Self { let keys: Vec = labels.iter().map(|l| key(l)).collect(); let state = FinState::empty(¶ms, 0); - Self { params, keys, blocks: Vec::new(), tables: HashMap::new(), mmr: Mmr::default(), leaves: HashMap::new(), tracker: SparseRing::new(), state, bls: BlstBls } + Self { params, keys, blocks: Vec::new(), tables: HashMap::new(), mmr: Mmr::default(), leaves: HashMap::new(), tracker: SparseRing::new(), keytree: SparseKeys::new(), state, bls: BlstBls } } /// Chain block `n` at DAA `n`, mined by key `miner`, with `extra` more blue blocks by the keys named; folds it. @@ -99,7 +101,7 @@ impl Sim { // whenever a key whose entry aged out mines again) let mut witness = witness; for b in &block.blues { - let known = igneum_fin_core::find_key(&self.state.keys, &b.key_hash).map(|i| self.state.keys[i].revealed()).unwrap_or(false); + let known = self.keytree.get(&b.key_hash).map(|e| e.revealed()).unwrap_or(false); if !known && !witness.reveals.iter().any(|r| vote_key_hash(&r.pubkey) == b.key_hash) { let k = self.keys.iter().find(|k| k.hash == b.key_hash).unwrap(); witness.reveals.push(reveal(k)); @@ -107,32 +109,45 @@ impl Sim { } let before = self.state.clone(); let ring_before = self.tracker.clone(); - let r = fold_block(&mut self.state, &self.params, &block, &mut self.tracker, &witness, &self.bls); + let keys_before = self.keytree.clone(); + let r = fold_block(&mut self.state, &self.params, &block, &mut self.tracker, &mut self.keytree, &witness, &self.bls); let witnesses = self.tracker.take_witnesses(); + let key_witnesses = self.keytree.take_witnesses(); match r { Ok(_) => {} Err(e) => { self.state = before; self.tracker = ring_before; + self.keytree = keys_before; return Err(e); } } // the guest's path: the same fold through the witnesses, from the same previous state, must agree let mut replay = before; let mut ring = WitnessRing::new(witnesses); - fold_block(&mut replay, &self.params, &block, &mut ring, &witness, &ZkBls).expect("the witnessed replay folds"); + let mut keys = WitnessKeys::new(key_witnesses); + fold_block(&mut replay, &self.params, &block, &mut ring, &mut keys, &witness, &ZkBls).expect("the witnessed replay folds"); assert_eq!(replay.extension(), self.state.extension(), "the guest's fold and the tracker's agree at block {n}"); - assert!(ring.witnesses.is_empty(), "every witness consumed"); - let (_, total) = voters_at(&self.state.keys, n, self.params.dust); - let leaf = HistoryLeaf { number: n, block_hash: block.block_hash, daa: n, table_root: self.state.table_root(), keys_hash: self.state.keys_hash(), total }; + assert!(ring.witnesses.is_empty() && keys.witnesses.is_empty(), "every witness consumed"); + let leaf = HistoryLeaf { number: n, block_hash: block.block_hash, daa: n, table_root: self.state.table_root(), keys_root: self.state.keys_root, key_count: self.state.key_count }; self.mmr.append(leaf.hash()); assert_eq!(self.mmr.root(), self.state.history_root()); self.leaves.insert(n, leaf); - self.tables.insert(n, self.state.keys.clone()); + self.tables.insert(n, self.keytree.all_entries()); self.blocks.push(block); Ok(()) } + /// The canonical table after block `number`. + fn table(&self, number: u64) -> Vec { + canonical(&self.tables[&number]).unwrap() + } + + /// A key's blocks in the current table. + fn blocks_of(&self, key: &Key) -> u64 { + self.keytree.get(&key.hash).map(|e| e.blocks).unwrap_or(0) + } + /// The brute-force window count: blue blocks per key with DAA in (daa - W, daa]. fn brute(&self, daa: u64) -> HashMap { let mut m = HashMap::new(); @@ -153,7 +168,7 @@ impl Sim { /// A certificate for index `index` over chain block `number`, signed by `signers`. fn certificate(&self, index: u64, number: u64, signers: &[usize]) -> CertificateWitness { let checkpoint = self.blocks[number as usize].block_hash; - let table = &self.tables[&number]; + let table = &self.table(number); let (voters, _) = voters_at(table, number, self.params.dust); let msg = vote_message(&self.params.chain_id, index, &checkpoint); let mut sigs = Vec::new(); @@ -191,12 +206,11 @@ fn the_carried_table_equals_a_brute_force_count_of_the_window_at_every_block() { let extra: Vec = if n % 7 == 0 { vec![1, 2] } else if n % 11 == 0 { vec![3] } else { vec![] }; sim.mine(miner, &extra, FoldWitness::default()).unwrap(); let brute = sim.brute(n); - for k in &sim.state.keys { + for (_, k) in sim.keytree.all_entries() { assert_eq!(k.blocks, brute.get(&k.key_hash).copied().unwrap_or(0), "key blocks at block {n}"); } for (kh, b) in &brute { - let i = igneum_fin_core::find_key(&sim.state.keys, kh).expect("every key with blocks is in the table"); - assert_eq!(sim.state.keys[i].blocks, *b); + assert_eq!(sim.keytree.get(kh).expect("every key with blocks is in the table").blocks, *b); } let ring_count: u64 = sim.tracker.all_entries().iter().filter(|e| e.blue).count() as u64; assert_eq!(ring_count, brute.values().sum::(), "the ring holds exactly the window at block {n}"); @@ -214,18 +228,17 @@ fn a_block_counted_twice_is_refused_and_ageing_is_exact_at_the_edge() { let dup = BlueBlock { block_hash: hash_of("dup", 1), key_hash: sim.keys[1].hash, daa: n }; let block = ChainBlockWitness { number: n, block_hash: hash_of("chain", n), daa: n, blues: vec![BlueBlock { block_hash: hash_of("chain", n), key_hash: sim.keys[0].hash, daa: n }, dup.clone(), dup], headers: Vec::new() }; let mut s = sim.state.clone(); - let err = fold_block(&mut s, &sim.params, &block, &mut sim.tracker, &FoldWitness::default(), &sim.bls).unwrap_err(); + let err = fold_block(&mut s, &sim.params, &block, &mut sim.tracker.clone(), &mut sim.keytree.clone(), &FoldWitness::default(), &sim.bls).unwrap_err(); assert!(err.contains("counted twice"), "{err}"); - sim.tracker.take_witnesses(); // ageing: with W = 200, the block at DAA d leaves exactly when the chain reaches d + 200 let mut sim = Sim::new(params(), &["a", "b"]); for _ in 0..201u64 { sim.mine(0, &[], FoldWitness::default()).unwrap(); } // blocks 0..=200 mined; at DAA 200 the window is (0, 200]: block 0 is out, 200 blocks counted - assert_eq!(sim.state.keys[0].blocks, 200); + assert_eq!(sim.blocks_of(&sim.keys[0]), 200); sim.mine(0, &[], FoldWitness::default()).unwrap(); - assert_eq!(sim.state.keys[0].blocks, 200, "one in, one out"); + assert_eq!(sim.blocks_of(&sim.keys[0]), 200, "one in, one out"); } /// Builds a chain past the first-month gate with four keys of weight 40, 30, 20 and 10 percent, every key revealed. @@ -246,7 +259,8 @@ fn known_failed_first_todays_light_client_takes_the_voter_list_from_a_node_and_t // keys a, b, c, d hold 40, 30, 20, 10 percent of the real 200-block window; e is the attacker with nothing let mut sim = chain_past_the_gate(&["a", "b", "c", "d", "e"]); let cp = 250u64; - let real_table = sim.tables[&cp].clone(); + let real_entries = sim.tables[&cp].clone(); + let real_table = sim.table(cp); let (real_voters, real_total) = voters_at(&real_table, cp, sim.params.dust); assert_eq!(real_voters.len(), 4, "e mined nothing and is no voter"); @@ -260,7 +274,7 @@ fn known_failed_first_todays_light_client_takes_the_voter_list_from_a_node_and_t e_entry.pubkey = e.pk; e_entry.blocks = real_total * 7 / 3 + 1; let pos = forged.binary_search_by(|k| k.key_hash.cmp(&e.hash)).unwrap_err(); - forged.insert(pos, e_entry); + forged.insert(pos, e_entry.clone()); let (forged_voters, forged_total) = voters_at(&forged, cp, sim.params.dust); let checkpoint = sim.blocks[cp as usize].block_hash; let msg = vote_message(&sim.params.chain_id, 9, &checkpoint); @@ -280,7 +294,9 @@ fn known_failed_first_todays_light_client_takes_the_voter_list_from_a_node_and_t // is refused (the table at the checkpoint is the one the proof committed), and with the real table it is // refused too (e holds nothing there and is no voter: the bitmap names a position outside the list, or e's // weight is zero). - let forged_at = TableAt { leaf: sim.leaves[&cp].clone(), proof: sim.mmr.proof(cp).unwrap(), keys: forged.clone() }; + let mut forged_entries = real_entries.clone(); + forged_entries.push((real_entries.len() as u64 + 7, e_entry.clone())); + let forged_at = TableAt { leaf: sim.leaves[&cp].clone(), proof: sim.mmr.proof(cp).unwrap(), keys: forged_entries }; let cert = CertificateWitness { index: 9, checkpoint, voter_count: forged_voters.len() as u32, bitmap: bitmap.clone(), signature: sig, signer_points: Vec::new(), at: forged_at, frozen: None }; let mut w = FoldWitness::default(); w.certificates.push(cert); @@ -323,7 +339,7 @@ fn two_thirds_is_inclusive_and_under_it_is_refused() { sim.mine((n % 6) as usize, &[], FoldWitness::default()).unwrap(); } let cp = 240u64; - let (_, total) = voters_at(&sim.tables[&cp], cp, sim.params.dust); + let (_, total) = voters_at(&sim.table(cp), cp, sim.params.dust); assert_eq!(total, 240); let three = sim.certificate(8, cp, &[0, 1, 2]); let mut w = FoldWitness::default(); @@ -366,8 +382,8 @@ fn the_frozen_table_holds_a_side_without_its_partner_and_a_leave_releases_it_aft sim.mine(0, &[], FoldWitness::default()).unwrap(); } let n = sim.blocks.len() as u64 - 1; - let (_, total) = voters_at(&sim.state.keys, n, sim.params.dust); - let a_blocks = sim.state.keys.iter().find(|k| k.key_hash == sim.keys[0].hash).unwrap().blocks; + let (_, total) = voters_at(&sim.table(n), n, sim.params.dust); + let a_blocks = sim.blocks_of(&sim.keys[0]); assert!(FinParams::floor_met(a_blocks, total), "a holds two thirds of the sliding table by now ({a_blocks} of {total})"); let cert = sim.certificate(12, n, &[0]); let mut w = FoldWitness::default(); @@ -453,14 +469,14 @@ fn a_light_client_answers_final_at_checkpoint_from_the_extension_and_a_history_p let mut bad = sim.leaves[&240].clone(); bad.block_hash = hash_of("other", 240); assert!(!sim.mmr.proof(240).unwrap().verify(&bad.hash(), &ext.history_root, leaves)); - assert_eq!(keys_hash(&sim.tables[&250]), sim.leaves[&250].keys_hash); + assert_eq!(igneum_fin_core::keys::dense_root(&sim.tables[&250], sim.leaves[&250].key_count).unwrap(), sim.leaves[&250].keys_root); } #[test] fn the_guest_curve_and_blst_agree_on_real_certificates_and_disagree_with_nothing() { let sim = chain_past_the_gate(&["a", "b", "c"]); let cert = sim.certificate(9, 250, &[0, 1]); - let table = &sim.tables[&250]; + let table = &sim.table(250); let (voters, _) = voters_at(table, 250, sim.params.dust); let pks: Vec<[u8; PUBKEY_LEN]> = signer_positions(&cert.bitmap, cert.voter_count).iter().map(|&p| table[voters[p]].pubkey).collect(); let msg = vote_message(&sim.params.chain_id, 9, &cert.checkpoint); @@ -495,6 +511,7 @@ fn level_one_pins_the_blues_to_headers_and_parent_links() { let p = params(); let mut state = FinState::empty(&p, 0); let mut ring = SparseRing::new(); + let mut kt = SparseKeys::new(); let keys: Vec = ["a", "b", "c"].iter().map(|l| key(l)).collect(); let hdr = |parents: Vec, daa: u64, blue_score: u64, work: u8, key: &Key, blue: bool| HeaderWitness { version: 2, parents_by_level: vec![parents], hash_merkle_root: hash_of("m", daa), accepted_id_merkle_root: ZERO_H, utxo_commitment: ZERO_H, timestamp: 1_000 + daa, bits: 0x1e00ffff, nonce: daa * 7, daa_score: daa, blue_work: vec![work], blue_score, pruning_point: ZERO_H, vote_key_hash: key.hash, blue }; // block 0 (genesis-like, no parents), by a @@ -502,8 +519,9 @@ fn level_one_pins_the_blues_to_headers_and_parent_links() { let b0 = ChainBlockWitness { number: 0, block_hash: h0.hash(), daa: 0, blues: vec![BlueBlock { block_hash: h0.hash(), key_hash: keys[0].hash, daa: 0 }], headers: vec![h0.clone()] }; let mut w = FoldWitness::default(); w.reveals = keys.iter().map(reveal).collect(); - fold_block(&mut state, &p, &b0, &mut ring, &w, &BlstBls).unwrap(); + fold_block(&mut state, &p, &b0, &mut ring, &mut kt, &w, &BlstBls).unwrap(); ring.take_witnesses(); + kt.take_witnesses(); // a side block s by b (parent: block 0) and a red r by c (parent: block 0); block 1 by a with parents [0, s, r] let hs = hdr(vec![h0.hash()], 1, 1, 2, &keys[1], true); let hr = hdr(vec![h0.hash()], 1, 1, 1, &keys[2], false); @@ -512,33 +530,34 @@ fn level_one_pins_the_blues_to_headers_and_parent_links() { // known-failed cases first let mut swapped = good.clone(); swapped.blues[1].key_hash = keys[2].hash; - let e = fold_block(&mut state.clone(), &p, &swapped, &mut ring.clone(), &FoldWitness::default(), &BlstBls).unwrap_err(); + let e = fold_block(&mut state.clone(), &p, &swapped, &mut ring.clone(), &mut kt.clone(), &FoldWitness::default(), &BlstBls).unwrap_err(); assert!(e.contains("differs from its header"), "{e}"); let mut miscount = good.clone(); miscount.blues.push(BlueBlock { block_hash: hr.hash(), key_hash: keys[2].hash, daa: 1 }); - let e = fold_block(&mut state.clone(), &p, &miscount, &mut ring.clone(), &FoldWitness::default(), &BlstBls).unwrap_err(); + let e = fold_block(&mut state.clone(), &p, &miscount, &mut ring.clone(), &mut kt.clone(), &FoldWitness::default(), &BlstBls).unwrap_err(); assert!(e.contains("blue score") || e.contains("differs from its header"), "{e}"); let mut unreached = good.clone(); let hx = hdr(vec![hash_of("nowhere", 9)], 1, 1, 1, &keys[1], false); unreached.headers.push(hx); - let e = fold_block(&mut state.clone(), &p, &unreached, &mut ring.clone(), &FoldWitness::default(), &BlstBls).unwrap_err(); + let e = fold_block(&mut state.clone(), &p, &unreached, &mut ring.clone(), &mut kt.clone(), &FoldWitness::default(), &BlstBls).unwrap_err(); assert!(e.contains("not reached"), "{e}"); let mut wrong_sp = good.clone(); wrong_sp.headers[0].parents_by_level = vec![vec![hs.hash(), hr.hash()]]; wrong_sp.block_hash = wrong_sp.headers[0].hash(); wrong_sp.blues[0].block_hash = wrong_sp.block_hash; - let e = fold_block(&mut state.clone(), &p, &wrong_sp, &mut ring.clone(), &FoldWitness::default(), &BlstBls).unwrap_err(); + let e = fold_block(&mut state.clone(), &p, &wrong_sp, &mut ring.clone(), &mut kt.clone(), &FoldWitness::default(), &BlstBls).unwrap_err(); assert!(e.contains("not a direct parent"), "{e}"); // the good block folds: b credited for s, c not credited for r, r in the ring uncounted - fold_block(&mut state, &p, &good, &mut ring, &FoldWitness::default(), &BlstBls).unwrap(); + fold_block(&mut state, &p, &good, &mut ring, &mut kt, &FoldWitness::default(), &BlstBls).unwrap(); ring.take_witnesses(); - let w_of = |k: &Key| igneum_fin_core::find_key(&state.keys, &k.hash).map(|i| state.keys[i].blocks).unwrap_or(0); + kt.take_witnesses(); + let w_of = |k: &Key| kt.get(&k.hash).map(|e| e.blocks).unwrap_or(0); assert_eq!((w_of(&keys[0]), w_of(&keys[1]), w_of(&keys[2])), (2, 1, 0)); assert!(ring.all_entries().iter().any(|e| e.block_hash == hr.hash() && !e.blue)); // block 2 replays r as a blue: refused by the ring let h2 = hdr(vec![h1.hash(), hr.hash()], 2, 4, 20, &keys[0], true); let replay = ChainBlockWitness { number: 2, block_hash: h2.hash(), daa: 2, blues: vec![BlueBlock { block_hash: h2.hash(), key_hash: keys[0].hash, daa: 2 }, BlueBlock { block_hash: hr.hash(), key_hash: keys[2].hash, daa: 1 }], headers: vec![h2.clone(), { let mut x = hr.clone(); x.blue = true; x }] }; - let e = fold_block(&mut state.clone(), &p, &replay, &mut ring.clone(), &FoldWitness::default(), &BlstBls).unwrap_err(); + let e = fold_block(&mut state.clone(), &p, &replay, &mut ring.clone(), &mut kt.clone(), &FoldWitness::default(), &BlstBls).unwrap_err(); assert!(e.contains("counted twice") || e.contains("not a direct parent") || e.contains("listed twice"), "{e}"); } diff --git a/proving/igneum-prove/host/src/fin.rs b/proving/igneum-prove/host/src/fin.rs index f078d33d1..eb75a6b36 100644 --- a/proving/igneum-prove/host/src/fin.rs +++ b/proving/igneum-prove/host/src/fin.rs @@ -6,6 +6,7 @@ use anyhow::{anyhow, bail, Context, Result}; use igneum_fin_core::cert::TableAt; use igneum_fin_core::fold::{fold_block, ChainBlockWitness, FoldWitness}; use igneum_fin_core::mmr::{HistoryLeaf, MmrProof}; +use igneum_fin_core::keys::{KeyLeafWitness, SparseKeys, WitnessKeys}; use igneum_fin_core::ring::{RingLeafWitness, WitnessRing}; use igneum_fin_core::{FinExt, FinParams, FinState}; use igneum_prove_core::agg::{BlockOutput, FinInput}; @@ -17,6 +18,8 @@ pub struct BlockFin { pub block: ChainBlockWitness, pub ring: Vec, #[serde(default)] + pub keys: Vec, + #[serde(default)] pub witness: FoldWitness, } @@ -51,11 +54,12 @@ pub fn prepare(file: &FinWitnessFile, first_number: u64, rooted: bool) -> Result if b.block.number != first_number + i as u64 { bail!("finality witness {i} is of chain block {}, expected {}", b.block.number, first_number + i as u64); } - let input = FinInput { params: file.params.clone(), prev_state: state.clone(), block: b.block.clone(), ring: b.ring.clone(), witness: b.witness.clone() }; + let input = FinInput { params: file.params.clone(), prev_state: state.clone(), block: b.block.clone(), ring: b.ring.clone(), keys: b.keys.clone(), witness: b.witness.clone() }; let mut ring = WitnessRing::new(b.ring.clone()); - fold_block(&mut state, &file.params, &b.block, &mut ring, &b.witness, &igneum_fin_core::bls::ZkBls).map_err(|e| anyhow!("finality fold of chain block {} natively: {e}", b.block.number))?; - if !ring.witnesses.is_empty() { - bail!("finality witness of chain block {} carries {} ring leaves the fold did not open", b.block.number, ring.witnesses.len()); + let mut keys = WitnessKeys::new(b.keys.clone()); + fold_block(&mut state, &file.params, &b.block, &mut ring, &mut keys, &b.witness, &igneum_fin_core::bls::ZkBls).map_err(|e| anyhow!("finality fold of chain block {} natively: {e}", b.block.number))?; + if !ring.witnesses.is_empty() || !keys.witnesses.is_empty() { + bail!("finality witness of chain block {} carries {} ring and {} key leaves the fold did not open", b.block.number, ring.witnesses.len(), keys.witnesses.len()); } out.push(input); } @@ -131,7 +135,7 @@ pub fn load_query(path: &str) -> Result { /// Certificate witnesses carry tables; this is what one weighs on the wire. pub fn table_bytes(t: &TableAt) -> usize { - t.keys.len() * igneum_fin_core::KeyEntry::LEN + 32 * (t.proof.siblings.len() + t.proof.peaks.len()) + t.keys.len() * (8 + igneum_fin_core::KeyEntry::LEN) + 32 * (t.proof.siblings.len() + t.proof.peaks.len()) } // --------------------------------------------------------------------------------------------------------------- @@ -145,6 +149,7 @@ use igneum_fin_core::fold::{BlueBlock, Reveal}; use igneum_fin_core::mmr::Mmr; use igneum_fin_core::tracker::SparseRing; use igneum_fin_core::{vote_key_hash, vote_message, voters_at, KeyEntry, DST_POP, DST_VOTE, H}; +use igneum_fin_core::keys::canonical; use igneum_prove_core::Fixture; struct SynthKey { @@ -184,6 +189,7 @@ pub fn synth(fixtures: &[Fixture], keys: usize, voters: usize, window_blocks: u6 let root_number = first.block.env.number - prefix; let mut state = FinState::empty(¶ms, root_number); let mut ring = SparseRing::new(); + let mut keys_tree = SparseKeys::new(); let bls = igneum_fin_core::bls::ZkBls; let mut mmr = Mmr::default(); for i in 0..prefix { @@ -191,28 +197,29 @@ pub fn synth(fixtures: &[Fixture], keys: usize, voters: usize, window_blocks: u6 let daa = first_daa - prefix + i; let k = &synth_keys[(i as usize) % synth_keys.len()]; let k2 = &synth_keys[(i as usize * 7 + 3) % synth_keys.len()]; - let block = ChainBlockWitness { number: n, block_hash: synth_hash("prefix", n), daa, blues: vec![BlueBlock { block_hash: synth_hash("prefix", n), key_hash: k.hash, daa }, BlueBlock { block_hash: synth_hash("prefix-side", n), key_hash: k2.hash, daa }] }; + let block = ChainBlockWitness { number: n, block_hash: synth_hash("prefix", n), daa, blues: vec![BlueBlock { block_hash: synth_hash("prefix", n), key_hash: k.hash, daa }, BlueBlock { block_hash: synth_hash("prefix-side", n), key_hash: k2.hash, daa }], headers: Vec::new() }; let mut w = FoldWitness::default(); - reveals_for(&state, &synth_keys, &block, &mut w); - fold_block(&mut state, ¶ms, &block, &mut ring, &w, &bls).map_err(|e| anyhow!("prefix fold {n}: {e}"))?; + reveals_for(&keys_tree, &synth_keys, &block, &mut w); + fold_block(&mut state, ¶ms, &block, &mut ring, &mut keys_tree, &w, &bls).map_err(|e| anyhow!("prefix fold {n}: {e}"))?; ring.take_witnesses(); - let (_, total) = voters_at(&state.keys, daa, params.dust); - mmr.append(HistoryLeaf { number: n, block_hash: block.block_hash, daa, table_root: state.table_root(), keys_hash: state.keys_hash(), total }.hash()); + keys_tree.take_witnesses(); + mmr.append(HistoryLeaf { number: n, block_hash: block.block_hash, daa, table_root: state.table_root(), keys_root: state.keys_root, key_count: state.key_count }.hash()); } let root_state = state.clone(); // the run over the fixtures, witnesses recorded; tables and leaves kept for the certificate - let mut tables: Vec<(u64, Vec, HistoryLeaf)> = Vec::new(); + let mut tables: Vec<(u64, Vec<(u64, KeyEntry)>, HistoryLeaf)> = Vec::new(); let mut blocks = Vec::new(); for (i, f) in fixtures.iter().enumerate() { let n = f.block.env.number; let daa = (daa_base + i as u64).max(state.end_daa); let k = &synth_keys[i % synth_keys.len()]; let k2 = &synth_keys[(i * 5 + 1) % synth_keys.len()]; - let block = ChainBlockWitness { number: n, block_hash: f.block.env.hash.0, daa, blues: vec![BlueBlock { block_hash: f.block.env.hash.0, key_hash: k.hash, daa }, BlueBlock { block_hash: synth_hash("side", n), key_hash: k2.hash, daa }] }; + let block = ChainBlockWitness { number: n, block_hash: f.block.env.hash.0, daa, blues: vec![BlueBlock { block_hash: f.block.env.hash.0, key_hash: k.hash, daa }, BlueBlock { block_hash: synth_hash("side", n), key_hash: k2.hash, daa }], headers: Vec::new() }; let mut w = FoldWitness::default(); - reveals_for(&state, &synth_keys, &block, &mut w); + reveals_for(&keys_tree, &synth_keys, &block, &mut w); if i + 1 == fixtures.len() && fixtures.len() > cert_back && cert_back > 0 { - let (cp_number, cp_keys, cp_leaf) = tables[tables.len() - cert_back].clone(); + let (cp_number, cp_entries, cp_leaf) = tables[tables.len() - cert_back].clone(); + let cp_keys = canonical(&cp_entries).map_err(|e| anyhow!(e))?; let (vlist, _) = voters_at(&cp_keys, cp_leaf.daa, params.dust); let mut heavy: Vec = vlist.clone(); heavy.sort_by_key(|&v| std::cmp::Reverse(cp_keys[v].blocks)); @@ -239,22 +246,21 @@ pub fn synth(fixtures: &[Fixture], keys: usize, voters: usize, window_blocks: u6 let signer_points: Vec> = by_pos.iter().map(|(_, v)| { let kh = cp_keys[**v].key_hash; synth_keys.iter().find(|k| k.hash == kh).unwrap().sk.sk_to_pk().serialize().to_vec() }).collect(); let position = cp_number - root_state.history_first; let proof = mmr.proof(position).ok_or_else(|| anyhow!("no history proof for {cp_number}"))?; - w.certificates.push(CertificateWitness { index, checkpoint: cp_leaf.block_hash, voter_count: vlist.len() as u32, bitmap, signature: agg, signer_points, at: TableAt { leaf: cp_leaf, proof, keys: cp_keys }, frozen: None }); + w.certificates.push(CertificateWitness { index, checkpoint: cp_leaf.block_hash, voter_count: vlist.len() as u32, bitmap, signature: agg, signer_points, at: TableAt { leaf: cp_leaf, proof, keys: cp_entries }, frozen: None }); } - fold_block(&mut state, ¶ms, &block, &mut ring, &w, &bls).map_err(|e| anyhow!("fold {n}: {e}"))?; - let (_, total) = voters_at(&state.keys, daa, params.dust); - let leaf = HistoryLeaf { number: n, block_hash: block.block_hash, daa, table_root: state.table_root(), keys_hash: state.keys_hash(), total }; + fold_block(&mut state, ¶ms, &block, &mut ring, &mut keys_tree, &w, &bls).map_err(|e| anyhow!("fold {n}: {e}"))?; + let leaf = HistoryLeaf { number: n, block_hash: block.block_hash, daa, table_root: state.table_root(), keys_root: state.keys_root, key_count: state.key_count }; mmr.append(leaf.hash()); - tables.push((n, state.keys.clone(), leaf)); - blocks.push(BlockFin { block, ring: ring.take_witnesses(), witness: w }); + tables.push((n, keys_tree.all_entries(), leaf)); + blocks.push(BlockFin { block, ring: ring.take_witnesses(), keys: keys_tree.take_witnesses(), witness: w }); } Ok(FinWitnessFile { format: FORMAT.into(), params, root_state, blocks }) } /// W1 as the tracker applies it: a key's reveal rides with its first block in the table. -fn reveals_for(state: &FinState, keys: &[SynthKey], block: &ChainBlockWitness, w: &mut FoldWitness) { +fn reveals_for(table: &SparseKeys, keys: &[SynthKey], block: &ChainBlockWitness, w: &mut FoldWitness) { for b in &block.blues { - let known = igneum_fin_core::find_key(&state.keys, &b.key_hash).map(|i| state.keys[i].revealed()).unwrap_or(false); + let known = table.get(&b.key_hash).map(|e| e.revealed()).unwrap_or(false); if !known && !w.reveals.iter().any(|r| vote_key_hash(&r.pubkey) == b.key_hash) { if let Some(k) = keys.iter().find(|k| k.hash == b.key_hash) { w.reveals.push(Reveal { pubkey: k.pk, pop: k.sk.sign(&k.pk, DST_POP, &[]).compress() }); diff --git a/proving/igneum-prove/host/src/main.rs b/proving/igneum-prove/host/src/main.rs index 53515206e..58e68098a 100644 --- a/proving/igneum-prove/host/src/main.rs +++ b/proving/igneum-prove/host/src/main.rs @@ -97,7 +97,7 @@ fn run() -> Result<()> { let file = fin::synth(&fixtures, keys, voters, window, cert_back)?; let text = serde_json::to_string(&file)?; std::fs::write(&out, &text).with_context(|| format!("write {out}"))?; - println!("RESULT fin-synth: {} blocks {}..={}, {} keys ({} in the table), {} signers, window {} blocks, certificate at block {} over {} back; root state {} keys; {} bytes of JSON in {:.1} s at {}", file.blocks.len(), fixtures[0].block.env.number, fixtures[fixtures.len() - 1].block.env.number, keys, file.root_state.keys.len(), voters, window, file.blocks.len(), cert_back, file.root_state.keys.len(), text.len(), t.elapsed().as_secs_f64(), now()); + println!("RESULT fin-synth: {} blocks {}..={}, {} keys ({} slots in the table), {} signers, window {} blocks, certificate at block {} over {} back; {} bytes of JSON in {:.1} s at {}", file.blocks.len(), fixtures[0].block.env.number, fixtures[fixtures.len() - 1].block.env.number, keys, file.root_state.key_count, voters, window, file.blocks.len(), cert_back, text.len(), t.elapsed().as_secs_f64(), now()); return Ok(()); } if mode == "fin-execute" { @@ -127,13 +127,13 @@ fn run() -> Result<()> { let ext = out_fin.fin.clone().ok_or_else(|| anyhow!("no extension in the output"))?; let (c0, c1) = (rep_plain.total_instruction_count(), rep_fin.total_instruction_count()); let sys: Vec<(String, u64)> = rep_fin.syscall_counts.iter().map(|(k, v)| (format!("{k:?}"), *v)).filter(|(_, v)| *v > 0).collect(); - println!("RESULT fin-execute block {}: plain {} cycles ({:.2} s), with finality {} cycles ({:.2} s), extra {} cycles; keys {} witness {} bytes certificates {}; lock index {} at block {} ({} of {}); syscalls {:?} at {}", f.block.env.number, c0, dt_plain.as_secs_f64(), c1, dt_fin.as_secs_f64(), c1.saturating_sub(c0), file.root_state.keys.len(), witness_bytes, certs, ext.lock.index, ext.lock.number, ext.lock.signed, ext.lock.total, sys, now()); + println!("RESULT fin-execute block {}: plain {} cycles ({:.2} s), with finality {} cycles ({:.2} s), extra {} cycles; key slots {} witness {} bytes certificates {}; lock index {} at block {} ({} of {}); syscalls {:?} at {}", f.block.env.number, c0, dt_plain.as_secs_f64(), c1, dt_fin.as_secs_f64(), c1.saturating_sub(c0), file.root_state.key_count, witness_bytes, certs, ext.lock.index, ext.lock.number, ext.lock.signed, ext.lock.total, sys, now()); rows.push(serde_json::json!({ "number": f.block.env.number, "plain_cycles": c0, "fin_cycles": c1, "extra_cycles": c1.saturating_sub(c0), "witness_bytes": witness_bytes, "certificates": certs, "lock_index": ext.lock.index, "syscalls": sys.iter().map(|(k, v)| serde_json::json!({"name": k, "count": v})).collect::>() })); prev_plain = Some(out_plain.to_bytes()); prev_fin = Some(out_fin.to_bytes()); } results.insert("blocks".into(), rows.into()); - results.insert("keys".into(), file.root_state.keys.len().into()); + results.insert("keys".into(), file.root_state.key_count.into()); drop(sp1); return finish(results, out_path.clone()); } @@ -748,7 +748,7 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_ if inputs.len() != built.len() { bail!("{p} carries {} finality witnesses for {} blocks", inputs.len(), built.len()); } - println!("RESULT chain fin: {} finality witnesses folded natively, root state ends at chain block {} with {} keys, {} ring leaves opened, {} certificates at {}", inputs.len(), file.root_state.end_number, file.root_state.keys.len(), file.blocks.iter().map(|b| b.ring.len()).sum::(), file.blocks.iter().map(|b| b.witness.certificates.len()).sum::(), now()); + println!("RESULT chain fin: {} finality witnesses folded natively, root state ends at chain block {} with {} key slots, {} ring and {} key leaves opened, {} certificates at {}", inputs.len(), file.root_state.end_number, file.root_state.key_count, file.blocks.iter().map(|b| b.ring.len()).sum::(), file.blocks.iter().map(|b| b.keys.len()).sum::(), file.blocks.iter().map(|b| b.witness.certificates.len()).sum::(), now()); inputs } };