diff --git a/app/igneum-app/src/prover.rs b/app/igneum-app/src/prover.rs index 9d1ed0bd0..c1053eee2 100644 --- a/app/igneum-app/src/prover.rs +++ b/app/igneum-app/src/prover.rs @@ -350,6 +350,10 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { // proving v1 segment path: (first, last, aggregator wei) of the segment records this machine submitted let mut submitted_segments: Vec<(u64, u64, u128)> = Vec::new(); let mut last_segment_secs: Option = None; + // segment records the node refused by the chain rule ("does not chain to ... pending"): held and offered again + // every pass until the segment's deadline (the fresh-record window of spec 7.8 is the segment length in DAA on + // the rule as shipped, 6 October 2026; from the fresh-rule switch the first retry lands) + let mut held_segments: Vec = Vec::new(); let mut last_verifier_read = Instant::now() - Duration::from_secs(600); let mut asked_restart = false; // macOS and Linux: the host sits next to the engine, so its pinned ids are read at once, proving on or off @@ -483,6 +487,37 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { }); } } + // held segment records: offered again, dropped past the deadline + if !held_segments.is_empty() { + let tip_daa = evm_rpc(&shared, "igneum_getProvingStatus", json!([]), Duration::from_secs(10)).ok().and_then(|st| st["tipDaa"].as_str().and_then(|x| u64::from_str_radix(x.trim_start_matches("0x"), 16).ok())).unwrap_or(0); + let mut keep = Vec::new(); + for h in held_segments.drain(..) { + match retry_held(&shared, &h, tip_daa) { + Retry::Accepted => { + shared.event("proving", &format!("segment {}..{} record accepted on retry {} (held {} s)", h.first, h.last, h.tries + 1, h.since.elapsed().as_secs())); + submitted_segments.push((h.first, h.last, h.agg_wei)); + set(&shared, |p| { + p.segments_submitted += 1; + p.aggregated += 1; + p.segment_note = format!("segment {}..{} accepted on retry", h.first, h.last); + }); + } + Retry::Expired(why) => { + shared.log(&format!("prover: segment {}..{} record dropped after {} tries: {why}", h.first, h.last, h.tries)); + } + Retry::Again(why) => { + let mut h = h; + h.tries += 1; + if h.tries % 30 == 1 { + shared.log(&format!("prover: segment {}..{} record held (try {}): {why}", h.first, h.last, h.tries)); + } + keep.push(h); + } + } + } + held_segments = keep; + set(&shared, |p| p.segments_held = held_segments.len() as u32); + } // paid segments among what we submitted for (first, last, wei) in submitted_segments.clone() { let paid = evm_rpc(&shared, "igneum_getSegmentRecords", json!([format!("{first:#x}")]), Duration::from_secs(10)) @@ -525,7 +560,21 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { attempted_segments.insert(seg.first); let started = Instant::now(); match prove_segment(&shared, t, &seg, &keys[0].0, &payout, prev_file.as_deref(), &expected_pv, &mut submitted) { - Ok(agg_wei) => { + Ok(SegmentOutcome::Held(h)) => { + let secs = started.elapsed().as_secs_f64(); + last_segment_secs = Some(secs); + shared.event("proving", &format!("segment {}..{}: {} shards proven and submitted in {secs:.0} s; the segment record is held ({})", seg.first, seg.last, seg.shards.len(), h.why)); + set(&shared, |p| { + p.segment_last_s = secs; + p.status = "submitted".into(); + p.message = format!("segment {}..{}: shards submitted, the segment record waits for the chain rule", seg.first, seg.last); + p.segment_note = format!("segment {}..{} proven whole in {secs:.0} s; its record is held: {}", seg.first, seg.last, h.why); + p.current = String::new(); + }); + held_segments.push(h); + set(&shared, |p| p.segments_held = held_segments.len() as u32); + } + Ok(SegmentOutcome::Submitted(agg_wei)) => { let secs = started.elapsed().as_secs_f64(); last_segment_secs = Some(secs); submitted_segments.push((seg.first, seg.last, agg_wei)); @@ -649,6 +698,44 @@ fn loop_forever(shared: Arc, bin_dir: PathBuf) { } } +/// A segment record the node refused by the chain rule, kept with its proof for another offer. +pub struct HeldSegment { + pub first: u64, + pub last: u64, + pub deadline_daa: u64, + pub record: String, + pub proof_path: PathBuf, + pub agg_wei: u128, + pub why: String, + pub tries: u32, + pub since: Instant, +} + +pub enum SegmentOutcome { + Submitted(u128), + Held(HeldSegment), +} + +pub enum Retry { + Accepted, + Again(String), + Expired(String), +} + +/// Offers a held segment record again: accepted, held for another pass, or dropped past the segment's deadline. +fn retry_held(shared: &Shared, h: &HeldSegment, tip_daa: u64) -> Retry { + if tip_daa > 0 && tip_daa + 1 > h.deadline_daa { + return Retry::Expired(format!("past the deadline DAA {} at tip DAA {tip_daa}", h.deadline_daa)); + } + let Ok(proof) = std::fs::read(&h.proof_path) else { return Retry::Expired(format!("proof file {} gone", h.proof_path.display())) }; + let proof_hex = format!("0x{}", proof.iter().map(|b| format!("{b:02x}")).collect::()); + match evm_rpc(shared, "igneum_submitSegmentRecord", json!([{ "record": h.record, "proof": proof_hex }]), Duration::from_secs(60)) { + Ok(r) if r["accepted"].as_bool().unwrap_or(false) => Retry::Accepted, + Ok(r) => Retry::Again(r["reason"].as_str().unwrap_or("?").to_string()), + Err(e) => Retry::Again(e), + } +} + /// Proving v1 segment path, the choice: the node's v1 status (active, the grid start, the segment length, the /// deadline clock), the work list grouped into whole untouched segments (`segments::whole_segments`), the /// candidates inside the deadline ranked for this key, then for the best three the node's segment statement: @@ -714,7 +801,7 @@ fn pick_segment(shared: &Shared, work: &[Work], key_hash: &str, attempted: &mut /// one host run (`--mode chain --save-shards`, `--prev` when the chain continues) that proves every shard and /// aggregates the segment, then every shard record signed and submitted (the shard payouts) and the segment /// record signed and submitted (the aggregator share). Returns the segment's aggregator wei. -fn prove_segment(shared: &Shared, t: &Tools, seg: &crate::segments::SegmentWork, label: &str, payout: &str, prev_file: Option<&str>, expected_pv: &str, submitted: &mut Vec<(u64, String, u32, u128)>) -> Result { +fn prove_segment(shared: &Shared, t: &Tools, seg: &crate::segments::SegmentWork, label: &str, payout: &str, prev_file: Option<&str>, expected_pv: &str, submitted: &mut Vec<(u64, String, u32, u128)>) -> Result { let (first, last) = (seg.first, seg.last); let dir = shared.runtime.app_dir.join("proving").join(format!("seg-{first}")); let _ = std::fs::create_dir_all(&dir); @@ -802,18 +889,25 @@ fn prove_segment(shared: &Shared, t: &Tools, seg: &crate::segments::SegmentWork, let proof = std::fs::read(to_win(&proof_file)).map_err(|e| format!("segment proof file {proof_file}: {e}"))?; let proof_hex = format!("0x{}", proof.iter().map(|b| format!("{b:02x}")).collect::()); let r = evm_rpc(shared, "igneum_submitSegmentRecord", json!([{ "record": record, "proof": proof_hex }]), Duration::from_secs(60))?; - if !r["accepted"].as_bool().unwrap_or(false) { - return Err(format!("segment record refused: {}", r["reason"].as_str().unwrap_or("?"))); - } - set(shared, |p| p.aggregated += 1); let hexu = |x: &Value| x.as_str().and_then(|s| u128::from_str_radix(s.trim_start_matches("0x"), 16).ok()).unwrap_or(0); - let agg_wei = evm_rpc(shared, "igneum_getSegmentStatement", json!([format!("{first:#x}")]), Duration::from_secs(10)).map(|s| hexu(&s["aggregatorWei"])).unwrap_or(0); - // the shard proofs stay for the aggregator of the next segment (the chain link is the segment proof, read back - // from the pool); the fixtures and the export go + let stmt = evm_rpc(shared, "igneum_getSegmentStatement", json!([format!("{first:#x}")]), Duration::from_secs(10)).unwrap_or(Value::Null); + let agg_wei = hexu(&stmt["aggregatorWei"]); + // the fixtures and the export go; the proofs stay (the next segment's chain link, and a held record's offer) for b in first..=last { let _ = std::fs::remove_file(dir.join(format!("block-{b}.json"))); } - Ok(agg_wei) + if !r["accepted"].as_bool().unwrap_or(false) { + let why = r["reason"].as_str().unwrap_or("?").to_string(); + // the chain rule's refusal ("does not chain to ... pending until DAA ..."): held, not failed; anything else + // (a bad statement, a late carrier) is an error + if why.contains("does not chain") { + let deadline = hexu(&stmt["status"]["deadline_daa"]) as u64; + return Ok(SegmentOutcome::Held(HeldSegment { first, last, deadline_daa: if deadline > 0 { deadline } else { u64::MAX }, record, proof_path: to_win(&proof_file), agg_wei, why, tries: 0, since: Instant::now() })); + } + return Err(format!("segment record refused: {why}")); + } + set(shared, |p| p.aggregated += 1); + Ok(SegmentOutcome::Submitted(agg_wei)) } /// Proving v1 (spec 7.8): one aggregation attempt. When the node reports v1 active, takes the newest executed diff --git a/app/igneum-app/src/state.rs b/app/igneum-app/src/state.rs index 2f94de27d..554371acc 100644 --- a/app/igneum-app/src/state.rs +++ b/app/igneum-app/src/state.rs @@ -211,6 +211,8 @@ pub struct ProvingState { #[serde(serialize_with = "u128_string")] pub segment_paid_wei: u128, pub segment_last_s: f64, + /// segment records the node refused by the chain rule and this machine offers again each pass + pub segments_held: u32, } #[derive(Clone, Serialize, Default)] diff --git a/app/igneum-app/ui/app.js b/app/igneum-app/ui/app.js index feeb5dc85..ebd6f8f21 100644 --- a/app/igneum-app/ui/app.js +++ b/app/igneum-app/ui/app.js @@ -1013,7 +1013,7 @@ if (typeof document !== 'undefined') (function () { setText('pv-paid-sub', pv.paid_wei ? (Number(pv.paid_wei) / 1e18).toFixed(4) + ' IGN earned' : 'shards paid out'); // proving v1 segments (6 October 2026): whole segments this machine proved, and the segment path's last line var segLine = ''; - if (pv.segments_submitted) segLine = 'Segments: ' + pv.segments_submitted + ' proven whole, ' + (pv.segments_paid || 0) + ' paid' + (pv.segment_paid_wei && Number(pv.segment_paid_wei) ? ' (' + (Number(pv.segment_paid_wei) / 1e18).toFixed(4) + ' IGN to the aggregator)' : '') + (pv.segment_last_s ? ', the last in ' + Math.round(pv.segment_last_s) + ' s' : '') + '.'; + if (pv.segments_submitted) segLine = 'Segments: ' + pv.segments_submitted + ' proven whole, ' + (pv.segments_paid || 0) + ' paid' + (pv.segment_paid_wei && Number(pv.segment_paid_wei) ? ' (' + (Number(pv.segment_paid_wei) / 1e18).toFixed(4) + ' IGN to the aggregator)' : '') + (pv.segment_last_s ? ', the last in ' + Math.round(pv.segment_last_s) + ' s' : '') + (pv.segments_held ? ', ' + pv.segments_held + ' record' + (pv.segments_held > 1 ? 's' : '') + ' held for the chain rule' : '') + '.'; if (pv.segment_note && enabled) segLine += (segLine ? ' ' : '') + pv.segment_note + '.'; $('pv-seg-note').hidden = !segLine; setText('pv-seg-note', segLine); var v = View.verifierWords(pv); diff --git a/infra/fast-time/override-60x.json b/infra/fast-time/override-60x.json index a9a2da61c..08f56e9ee 100644 --- a/infra/fast-time/override-60x.json +++ b/infra/fast-time/override-60x.json @@ -58,6 +58,7 @@ "proving_v1_activation_daa": 18446744073709551615, "proving_v1_segment_blocks": 8, "proving_v1_unproven_daa": 10, + "proving_v1_fresh_rule_daa": 0, "proving_v1_aggregator_share_bps": 1000, "fees": {"pgas": {"version": 1, "cycles_per_pgas": 1000, "intrinsic_pgas_per_tx": 300, "modexp_base": 10, "modexp_per_byte_numer": 1, "modexp_per_byte_denom": 10}, "block_proving_gas_limit": 120000, "shard_proving_gas_budget": 30000, "min_execution_base_fee_wei": 100000000000, "min_proving_base_fee_wei": 10000000000000, "initial_execution_base_fee_wei": 100000000000, "initial_proving_base_fee_wei": 10000000000000, "base_fee_change_denominator": 8} }