release 0.3.12: merge proving-v1 app f0a40cd (the segment-aligned prover, the held fresh record, the fresh-record rule harness)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
commit
e651e281d4
18 changed files with 1771 additions and 17 deletions
|
|
@ -30,6 +30,7 @@ mod jobrun;
|
|||
mod jobbuild;
|
||||
mod prover;
|
||||
mod provedefault;
|
||||
mod segments;
|
||||
mod verifier;
|
||||
mod wslhost;
|
||||
mod sweep;
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@ pub struct Work {
|
|||
pub shard_wei: u128,
|
||||
/// The first of this machine's keys that is assigned (the label that signs).
|
||||
pub key_hash: String,
|
||||
/// The chain block's DAA score (the deadline clock of spec 7.8).
|
||||
pub daa: u64,
|
||||
}
|
||||
|
||||
/// Parses the node's work list. Newest first, as the node returns it.
|
||||
|
|
@ -67,6 +69,7 @@ pub fn parse_work(v: &Value) -> Vec<Work> {
|
|||
in_pool: w["pool"].as_array().map(|p| !p.is_empty()).unwrap_or(false),
|
||||
shard_wei: hexu(&w["shardWei"]),
|
||||
key_hash: w["assignedKeys"].as_array().and_then(|k| k.first()).and_then(|k| k.as_str()).unwrap_or("").to_string(),
|
||||
daa: hexu(&w["daaScore"]) as u64,
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
|
|
@ -344,6 +347,13 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
let ram_mb = crate::detect::total_ram_mb();
|
||||
let mut last_probe = Instant::now() - Duration::from_secs(600);
|
||||
let mut submitted: Vec<(u64, String, u32, u128)> = Vec::new();
|
||||
// proving v1 segment path: (first, last, aggregator wei) of the segment records this machine submitted
|
||||
let mut submitted_segments: Vec<(u64, u64, u128)> = Vec::new();
|
||||
let mut last_segment_secs: Option<f64> = None;
|
||||
// segment records the node refused by the chain rule ("does not chain to ... pending"): held and offered again
|
||||
// every pass until the segment's deadline (the fresh-record window of spec 7.8 is the segment length in DAA on
|
||||
// the rule as shipped, 6 October 2026; from the fresh-rule switch the first retry lands)
|
||||
let mut held_segments: Vec<HeldSegment> = Vec::new();
|
||||
let mut last_verifier_read = Instant::now() - Duration::from_secs(600);
|
||||
let mut asked_restart = false;
|
||||
// macOS and Linux: the host sits next to the engine, so its pinned ids are read at once, proving on or off
|
||||
|
|
@ -450,7 +460,7 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
});
|
||||
continue;
|
||||
}
|
||||
let work = match evm_rpc(&shared, "igneum_getAssignedShards", json!([keys.iter().map(|(_, h)| h.clone()).collect::<Vec<_>>(), 60]), Duration::from_secs(10)) {
|
||||
let work = match evm_rpc(&shared, "igneum_getAssignedShards", json!([keys.iter().map(|(_, h)| h.clone()).collect::<Vec<_>>(), crate::segments::WORK_LOOKBACK]), Duration::from_secs(10)) {
|
||||
Ok(v) => parse_work(&v),
|
||||
Err(e) => {
|
||||
set(&shared, |p| {
|
||||
|
|
@ -477,6 +487,52 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
});
|
||||
}
|
||||
}
|
||||
// held segment records: offered again, dropped past the deadline
|
||||
if !held_segments.is_empty() {
|
||||
let tip_daa = evm_rpc(&shared, "igneum_getProvingStatus", json!([]), Duration::from_secs(10)).ok().and_then(|st| st["tipDaa"].as_str().and_then(|x| u64::from_str_radix(x.trim_start_matches("0x"), 16).ok())).unwrap_or(0);
|
||||
let mut keep = Vec::new();
|
||||
for h in held_segments.drain(..) {
|
||||
match retry_held(&shared, &h, tip_daa) {
|
||||
Retry::Accepted => {
|
||||
shared.event("proving", &format!("segment {}..{} record accepted on retry {} (held {} s)", h.first, h.last, h.tries + 1, h.since.elapsed().as_secs()));
|
||||
submitted_segments.push((h.first, h.last, h.agg_wei));
|
||||
set(&shared, |p| {
|
||||
p.segments_submitted += 1;
|
||||
p.aggregated += 1;
|
||||
p.segment_note = format!("segment {}..{} accepted on retry", h.first, h.last);
|
||||
});
|
||||
}
|
||||
Retry::Expired(why) => {
|
||||
shared.log(&format!("prover: segment {}..{} record dropped after {} tries: {why}", h.first, h.last, h.tries));
|
||||
}
|
||||
Retry::Again(why) => {
|
||||
let mut h = h;
|
||||
h.tries += 1;
|
||||
if h.tries % 30 == 1 {
|
||||
shared.log(&format!("prover: segment {}..{} record held (try {}): {why}", h.first, h.last, h.tries));
|
||||
}
|
||||
keep.push(h);
|
||||
}
|
||||
}
|
||||
}
|
||||
held_segments = keep;
|
||||
set(&shared, |p| p.segments_held = held_segments.len() as u32);
|
||||
}
|
||||
// paid segments among what we submitted
|
||||
for (first, last, wei) in submitted_segments.clone() {
|
||||
let paid = evm_rpc(&shared, "igneum_getSegmentRecords", json!([format!("{first:#x}")]), Duration::from_secs(10))
|
||||
.ok()
|
||||
.map(|r| !r["paid"].is_null() && r["paid"]["payout"].as_str().map(|a| a.eq_ignore_ascii_case(&payout_address(&shared))).unwrap_or(false))
|
||||
.unwrap_or(false);
|
||||
if paid {
|
||||
submitted_segments.retain(|x| x.0 != first);
|
||||
shared.event("proving", &format!("segment {first}..{last} paid {} IGN to the aggregator", wei as f64 / 1e18));
|
||||
set(&shared, |p| {
|
||||
p.segments_paid += 1;
|
||||
p.segment_paid_wei += wei;
|
||||
});
|
||||
}
|
||||
}
|
||||
let assigned = work.iter().filter(|w| w.assigned).count() as u32;
|
||||
set(&shared, |p| {
|
||||
p.assigned = assigned;
|
||||
|
|
@ -496,10 +552,60 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
}
|
||||
}
|
||||
}
|
||||
// proving v1 segment path (src/segments.rs, 6 October 2026): a whole segment first, the newest shard only
|
||||
// when no whole segment qualifies
|
||||
let payout = shared.settings.lock().unwrap().address.clone();
|
||||
if payout.len() == 42 {
|
||||
if let Some((seg, prev_file, expected_pv)) = pick_segment(&shared, &work, &keys[0].1, &mut attempted_segments, last_segment_secs) {
|
||||
attempted_segments.insert(seg.first);
|
||||
let started = Instant::now();
|
||||
match prove_segment(&shared, t, &seg, &keys[0].0, &payout, prev_file.as_deref(), &expected_pv, &mut submitted) {
|
||||
Ok(SegmentOutcome::Held(h)) => {
|
||||
let secs = started.elapsed().as_secs_f64();
|
||||
last_segment_secs = Some(secs);
|
||||
shared.event("proving", &format!("segment {}..{}: {} shards proven and submitted in {secs:.0} s; the segment record is held ({})", seg.first, seg.last, seg.shards.len(), h.why));
|
||||
set(&shared, |p| {
|
||||
p.segment_last_s = secs;
|
||||
p.status = "submitted".into();
|
||||
p.message = format!("segment {}..{}: shards submitted, the segment record waits for the chain rule", seg.first, seg.last);
|
||||
p.segment_note = format!("segment {}..{} proven whole in {secs:.0} s; its record is held: {}", seg.first, seg.last, h.why);
|
||||
p.current = String::new();
|
||||
});
|
||||
held_segments.push(h);
|
||||
set(&shared, |p| p.segments_held = held_segments.len() as u32);
|
||||
}
|
||||
Ok(SegmentOutcome::Submitted(agg_wei)) => {
|
||||
let secs = started.elapsed().as_secs_f64();
|
||||
last_segment_secs = Some(secs);
|
||||
submitted_segments.push((seg.first, seg.last, agg_wei));
|
||||
shared.event("proving", &format!("segment {}..{}: {} shards proven, aggregated and submitted in {secs:.0} s", seg.first, seg.last, seg.shards.len()));
|
||||
set(&shared, |p| {
|
||||
p.segments_submitted += 1;
|
||||
p.segment_last_s = secs;
|
||||
p.status = "submitted".into();
|
||||
p.message = format!("segment {}..{} submitted; paid when a block carries it", seg.first, seg.last);
|
||||
p.segment_note = format!("segment {}..{} proven whole in {secs:.0} s", seg.first, seg.last);
|
||||
p.current = String::new();
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
shared.log(&format!("prover: segment {}..{}: {e}", seg.first, seg.last));
|
||||
set(&shared, |p| {
|
||||
p.failed += 1;
|
||||
p.status = "idle".into();
|
||||
p.message = e.clone();
|
||||
p.segment_note = e;
|
||||
p.current = String::new();
|
||||
});
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let Some(w) = choose(&work, &attempted) else {
|
||||
set(&shared, |p| {
|
||||
p.status = if submitted.is_empty() { "idle".into() } else { "submitted".into() };
|
||||
p.message = if assigned == 0 { "no shard assigned to this machine and none open in the last 60 blocks".into() } else { "every assigned and open shard is proven or paid".into() };
|
||||
p.message = if assigned == 0 { format!("no shard assigned to this machine and none open in the last {} blocks", crate::segments::WORK_LOOKBACK) } else { "every assigned and open shard is proven or paid".into() };
|
||||
});
|
||||
continue;
|
||||
};
|
||||
|
|
@ -592,6 +698,218 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
}
|
||||
}
|
||||
|
||||
/// A segment record the node refused by the chain rule, kept with its proof for another offer.
|
||||
pub struct HeldSegment {
|
||||
pub first: u64,
|
||||
pub last: u64,
|
||||
pub deadline_daa: u64,
|
||||
pub record: String,
|
||||
pub proof_path: PathBuf,
|
||||
pub agg_wei: u128,
|
||||
pub why: String,
|
||||
pub tries: u32,
|
||||
pub since: Instant,
|
||||
}
|
||||
|
||||
pub enum SegmentOutcome {
|
||||
Submitted(u128),
|
||||
Held(HeldSegment),
|
||||
}
|
||||
|
||||
pub enum Retry {
|
||||
Accepted,
|
||||
Again(String),
|
||||
Expired(String),
|
||||
}
|
||||
|
||||
/// Offers a held segment record again: accepted, held for another pass, or dropped past the segment's deadline.
|
||||
fn retry_held(shared: &Shared, h: &HeldSegment, tip_daa: u64) -> Retry {
|
||||
if tip_daa > 0 && tip_daa + 1 > h.deadline_daa {
|
||||
return Retry::Expired(format!("past the deadline DAA {} at tip DAA {tip_daa}", h.deadline_daa));
|
||||
}
|
||||
let Ok(proof) = std::fs::read(&h.proof_path) else { return Retry::Expired(format!("proof file {} gone", h.proof_path.display())) };
|
||||
let proof_hex = format!("0x{}", proof.iter().map(|b| format!("{b:02x}")).collect::<String>());
|
||||
match evm_rpc(shared, "igneum_submitSegmentRecord", json!([{ "record": h.record, "proof": proof_hex }]), Duration::from_secs(60)) {
|
||||
Ok(r) if r["accepted"].as_bool().unwrap_or(false) => Retry::Accepted,
|
||||
Ok(r) => Retry::Again(r["reason"].as_str().unwrap_or("?").to_string()),
|
||||
Err(e) => Retry::Again(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Proving v1 segment path, the choice: the node's v1 status (active, the grid start, the segment length, the
|
||||
/// deadline clock), the work list grouped into whole untouched segments (`segments::whole_segments`), the
|
||||
/// candidates inside the deadline ranked for this key, then for the best three the node's segment statement:
|
||||
/// executed and pending; the previous segment either paid with its proof in this node's pool (the chain continues,
|
||||
/// `--prev`) or not paid and with no verified record of it waiting in the pool (fresh). Returns the segment, the
|
||||
/// previous proof's host path when the chain continues, and the public values the node expects.
|
||||
fn pick_segment(shared: &Shared, work: &[Work], key_hash: &str, attempted: &mut HashSet<u64>, last_secs: Option<f64>) -> Option<(crate::segments::SegmentWork, Option<String>, String)> {
|
||||
let hexu = |x: &Value| x.as_str().and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()).unwrap_or(0);
|
||||
let st = evm_rpc(shared, "igneum_getProvingStatus", json!([]), Duration::from_secs(10)).ok()?;
|
||||
let v1 = &st["v1"];
|
||||
if !v1["active"].as_bool().unwrap_or(false) || v1["start"].is_null() {
|
||||
return None;
|
||||
}
|
||||
let (start, n, unproven, tip_daa) = (hexu(&v1["start"]), hexu(&v1["segmentBlocks"]).max(1), hexu(&v1["unprovenDaa"]), hexu(&st["tipDaa"]));
|
||||
let segs = crate::segments::whole_segments(start, n, unproven, work);
|
||||
let need = crate::segments::need_daa(last_secs);
|
||||
let cands = crate::segments::candidates(&segs, tip_daa, need, key_hash, attempted);
|
||||
if cands.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let dir = shared.runtime.app_dir.join("proving");
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
let wsl = cfg!(windows);
|
||||
let as_host_path = |p: &Path| if wsl { wsl_path(p) } else { p.display().to_string() };
|
||||
for seg in cands.into_iter().take(3) {
|
||||
let Ok(stmt) = evm_rpc(shared, "igneum_getSegmentStatement", json!([format!("{:#x}", seg.first)]), Duration::from_secs(10)) else { continue };
|
||||
if !stmt["executed"].as_bool().unwrap_or(false) || stmt["status"]["status"].as_str() != Some("pending") {
|
||||
attempted.insert(seg.first);
|
||||
continue;
|
||||
}
|
||||
let prev = &stmt["previous"];
|
||||
if prev.is_null() {
|
||||
// fresh only when no record of the previous segment is waiting to be carried (the chain rule would
|
||||
// refuse a fresh record once that one pays)
|
||||
if seg.first >= start + n {
|
||||
let p = evm_rpc(shared, "igneum_getSegmentRecords", json!([format!("{:#x}", seg.first - n)]), Duration::from_secs(10)).unwrap_or(Value::Null);
|
||||
let waiting = p["pool"].as_array().map(|a| a.iter().any(|e| e["verified"] == json!(true) && e["includedIn"].is_null())).unwrap_or(false);
|
||||
if waiting || !p["paid"].is_null() {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return Some((seg, None, stmt["publicValuesFresh"].as_str().unwrap_or("").to_string()));
|
||||
}
|
||||
if prev["proofInPool"] != json!(true) {
|
||||
continue;
|
||||
}
|
||||
let Ok(got) = evm_rpc(shared, "igneum_getSegmentProofBytes", json!([prev["first"], prev["keyHash"]]), Duration::from_secs(60)) else { continue };
|
||||
let hex = got["proof"].as_str().unwrap_or("").trim_start_matches("0x").to_string();
|
||||
if hex.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let bytes: Vec<u8> = (0..hex.len() / 2).map(|k| u8::from_str_radix(&hex[2 * k..2 * k + 2], 16).unwrap_or(0)).collect();
|
||||
let f = dir.join(format!("prev-{}.bin", seg.first));
|
||||
if std::fs::write(&f, bytes).is_err() {
|
||||
continue;
|
||||
}
|
||||
return Some((seg, Some(as_host_path(&f)), stmt["publicValuesContinuing"].as_str().unwrap_or("").to_string()));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Proving v1 segment path, the work: one export of the chain to the segment's last block, one fixture per block,
|
||||
/// one host run (`--mode chain --save-shards`, `--prev` when the chain continues) that proves every shard and
|
||||
/// aggregates the segment, then every shard record signed and submitted (the shard payouts) and the segment
|
||||
/// record signed and submitted (the aggregator share). Returns the segment's aggregator wei.
|
||||
fn prove_segment(shared: &Shared, t: &Tools, seg: &crate::segments::SegmentWork, label: &str, payout: &str, prev_file: Option<&str>, expected_pv: &str, submitted: &mut Vec<(u64, String, u32, u128)>) -> Result<SegmentOutcome, String> {
|
||||
let (first, last) = (seg.first, seg.last);
|
||||
let dir = shared.runtime.app_dir.join("proving").join(format!("seg-{first}"));
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
let as_host_path = |p: &Path| if t.wsl { wsl_path(p) } else { p.display().to_string() };
|
||||
set(shared, |p| {
|
||||
p.status = "proving".into();
|
||||
p.current = format!("segment {first}..{last} ({} shards)", seg.shards.len());
|
||||
p.started_at = crate::platform::unix_now_f();
|
||||
p.message = "exporting the chain and cutting the segment's blocks".into();
|
||||
});
|
||||
shared.log(&format!("prover: segment {first}..{last} claimed ({} shards{}): export, cut, chain ({}), sign, submit", seg.shards.len(), if prev_file.is_some() { ", continuing the previous segment's proof" } else { ", fresh" }, if t.cuda { "CUDA" } else { "CPU" }));
|
||||
// 1. export once, cut every block
|
||||
let seq = dir.join("seq.json");
|
||||
let export = evm_rpc(shared, "igneum_exportSegments", json!(["0x0", format!("{last:#x}")]), Duration::from_secs(300))?;
|
||||
std::fs::write(&seq, export.to_string()).map_err(|e| e.to_string())?;
|
||||
let mut fixtures: Vec<String> = Vec::new();
|
||||
for b in first..=last {
|
||||
let fixture = dir.join(format!("block-{b}.json"));
|
||||
let (ok, out) = run_tool(shared, t, &t.export, &[as_host_path(&seq), b.to_string(), as_host_path(&fixture)], &[], Duration::from_secs(600), &dir.join(format!("export-{b}.log")));
|
||||
if !ok || !fixture.exists() {
|
||||
return Err(format!("exporter, block {b}: {}", out.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("failed")));
|
||||
}
|
||||
fixtures.push(as_host_path(&fixture));
|
||||
}
|
||||
let _ = std::fs::remove_file(&seq);
|
||||
// 2. the chain: every shard proven, every block aggregated with the previous, in one process
|
||||
set(shared, |p| p.message = format!("proving {} shards and aggregating segment {first}..{last} ({})", seg.shards.len(), if t.cuda { "GPU" } else { "CPU, slow" }));
|
||||
let results = dir.join("chain-results.json");
|
||||
let mut args: Vec<String> = vec!["--mode".into(), "chain".into(), "--chain".into(), fixtures.join(","), "--prover".into(), payout.to_string(), "--save-shards".into(), "--out".into(), as_host_path(&results)];
|
||||
if let Some(pf) = prev_file {
|
||||
args.push("--prev".into());
|
||||
args.push(pf.to_string());
|
||||
}
|
||||
let (ok, out) = run_tool(shared, t, &t.host, &args, &[("SP1_PROVER", if t.cuda { "cuda" } else { "cpu" }), ("RUST_LOG", "off")], Duration::from_secs(3 * 3600), &dir.join("chain.log"));
|
||||
if !ok || !results.exists() {
|
||||
let last_line = out.lines().rev().find(|l| l.contains("RESULT") || l.contains("rror")).unwrap_or("failed").to_string();
|
||||
let hint = if last_line.contains("PermissionDenied") { " (a GPU-server socket /tmp/sp1-cuda-*.sock owned by another user: the root-socket class)" } else { "" };
|
||||
return Err(format!("chain: {last_line}{hint}"));
|
||||
}
|
||||
let res: Value = serde_json::from_str(&std::fs::read_to_string(&results).map_err(|e| e.to_string())?).map_err(|e| e.to_string())?;
|
||||
let to_win = |f: &str| if t.wsl { PathBuf::from(f.replace("/mnt/c/", "C:/")) } else { PathBuf::from(f) };
|
||||
// 3. the shard records
|
||||
let chain = chain_name(shared);
|
||||
let mut shard_ok = 0usize;
|
||||
for b in res["blocks"].as_array().cloned().unwrap_or_default() {
|
||||
for r in b["shard_records"].as_array().cloned().unwrap_or_default() {
|
||||
let (number, hash, shard) = (r["number"].as_u64().unwrap_or(0), r["block_hash"].as_str().unwrap_or("").to_string(), r["shard"].as_u64().unwrap_or(0) as u32);
|
||||
let statement = r["statement"].as_str().unwrap_or("").to_string();
|
||||
let proof_sha = r["proof_sha256"].as_str().unwrap_or("").to_string();
|
||||
let proof_file = r["proof_file"].as_str().unwrap_or("").to_string();
|
||||
let sg = crate::detect::run_timeout(crate::platform::quiet(&mut Command::new(&t.miner)).args(["sign-record", label, &chain, &hash, &number.to_string(), &shard.to_string(), payout, &statement, &proof_sha]), None, Duration::from_secs(20)).ok_or("sign-record did not run")?;
|
||||
let signed: Value = serde_json::from_str(sg.lines().last().unwrap_or("")).map_err(|_| format!("sign-record: {}", sg.trim()))?;
|
||||
let record = signed["record"].as_str().ok_or("sign-record gave no record")?.to_string();
|
||||
let proof = std::fs::read(to_win(&proof_file)).map_err(|e| format!("proof file {proof_file}: {e}"))?;
|
||||
let proof_hex = format!("0x{}", proof.iter().map(|b| format!("{b:02x}")).collect::<String>());
|
||||
let out = evm_rpc(shared, "igneum_submitProofRecord", json!([{ "record": record, "proof": proof_hex }]), Duration::from_secs(60))?;
|
||||
if out["accepted"].as_bool().unwrap_or(false) {
|
||||
shard_ok += 1;
|
||||
let wei = seg.shards.iter().position(|(n, _, s)| *n == number && *s == shard).map(|_| seg.shard_wei / seg.shards.len().max(1) as u128).unwrap_or(0);
|
||||
submitted.push((number, hash.clone(), shard, wei));
|
||||
} else {
|
||||
shared.log(&format!("prover: segment {first}..{last}: block {number} shard {shard} record refused: {}", out["reason"].as_str().unwrap_or("?")));
|
||||
}
|
||||
}
|
||||
}
|
||||
if shard_ok != seg.shards.len() {
|
||||
return Err(format!("{shard_ok} of {} shard records accepted; the segment record is not submitted", seg.shards.len()));
|
||||
}
|
||||
set(shared, |p| {
|
||||
p.proved += shard_ok as u32;
|
||||
p.submitted += shard_ok as u32;
|
||||
});
|
||||
// 4. the segment record: the aggregated statement against the node's native one (every field but provers)
|
||||
let pv = res["segment_public_values"].as_str().ok_or("no public values in the chain results")?.to_string();
|
||||
let proof_sha = res["segment_proof_sha256"].as_str().ok_or("no segment proof hash in the chain results")?.to_string();
|
||||
let proof_file = res["segment_proof_file"].as_str().ok_or("no segment proof file in the chain results")?.to_string();
|
||||
let strip = |h: &str| { let h = h.trim_start_matches("0x"); if h.len() == 680 { format!("{}{}", &h[..472], &h[536..]) } else { h.to_string() } };
|
||||
if strip(&pv) != strip(expected_pv) {
|
||||
return Err(format!("the aggregated statement differs from the node's native statement (it would be vetoed); ours {} node {}", &pv[..66.min(pv.len())], &expected_pv[..66.min(expected_pv.len())]));
|
||||
}
|
||||
let last_hash = seg.shards.iter().rev().find(|(n, _, _)| *n == last).map(|(_, h, _)| h.clone()).ok_or("no last block hash")?;
|
||||
let sg = crate::detect::run_timeout(crate::platform::quiet(&mut Command::new(&t.miner)).args(["sign-segment-record", label, &chain, &first.to_string(), &last.to_string(), &last_hash, payout, &pv, &proof_sha]), None, Duration::from_secs(20)).ok_or("sign-segment-record did not run")?;
|
||||
let signed: Value = serde_json::from_str(sg.lines().last().unwrap_or("")).map_err(|_| format!("sign-segment-record: {}", sg.trim()))?;
|
||||
let record = signed["record"].as_str().ok_or("sign-segment-record gave no record")?.to_string();
|
||||
let proof = std::fs::read(to_win(&proof_file)).map_err(|e| format!("segment proof file {proof_file}: {e}"))?;
|
||||
let proof_hex = format!("0x{}", proof.iter().map(|b| format!("{b:02x}")).collect::<String>());
|
||||
let r = evm_rpc(shared, "igneum_submitSegmentRecord", json!([{ "record": record, "proof": proof_hex }]), Duration::from_secs(60))?;
|
||||
let hexu = |x: &Value| x.as_str().and_then(|s| u128::from_str_radix(s.trim_start_matches("0x"), 16).ok()).unwrap_or(0);
|
||||
let stmt = evm_rpc(shared, "igneum_getSegmentStatement", json!([format!("{first:#x}")]), Duration::from_secs(10)).unwrap_or(Value::Null);
|
||||
let agg_wei = hexu(&stmt["aggregatorWei"]);
|
||||
// the fixtures and the export go; the proofs stay (the next segment's chain link, and a held record's offer)
|
||||
for b in first..=last {
|
||||
let _ = std::fs::remove_file(dir.join(format!("block-{b}.json")));
|
||||
}
|
||||
if !r["accepted"].as_bool().unwrap_or(false) {
|
||||
let why = r["reason"].as_str().unwrap_or("?").to_string();
|
||||
// the chain rule's refusal ("does not chain to ... pending until DAA ..."): held, not failed; anything else
|
||||
// (a bad statement, a late carrier) is an error
|
||||
if why.contains("does not chain") {
|
||||
let deadline = hexu(&stmt["status"]["deadline_daa"]) as u64;
|
||||
return Ok(SegmentOutcome::Held(HeldSegment { first, last, deadline_daa: if deadline > 0 { deadline } else { u64::MAX }, record, proof_path: to_win(&proof_file), agg_wei, why, tries: 0, since: Instant::now() }));
|
||||
}
|
||||
return Err(format!("segment record refused: {why}"));
|
||||
}
|
||||
set(shared, |p| p.aggregated += 1);
|
||||
Ok(SegmentOutcome::Submitted(agg_wei))
|
||||
}
|
||||
|
||||
/// Proving v1 (spec 7.8): one aggregation attempt. When the node reports v1 active, takes the newest executed
|
||||
/// segment that is still pending and not yet attempted here, needs one shard proof per shard of every block in
|
||||
/// this node's pool (`igneum_getProofBytes`, a verified one when there is one) and, when the previous segment is
|
||||
|
|
|
|||
214
app/igneum-app/src/segments.rs
Normal file
214
app/igneum-app/src/segments.rs
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
//! Proving v1 (spec 7.8): segment-aligned work for the prover loop (6 October 2026).
|
||||
//!
|
||||
//! The shipped loop took the newest open shard each pass, so one prover scattered one block in about 45 across
|
||||
//! the segment grid and no segment ever had all its blocks proven (node 1, 04:16Z: pending 55, proven 0). Here a
|
||||
//! free prover claims a whole segment (`proving_v1_segment_blocks` consecutive chain blocks), proves every shard
|
||||
//! of it in order from one export in one host run (`--mode chain --save-shards`), submits the shard records and the
|
||||
//! aggregated segment record, then takes the next. One card completes whole segments at its own rate instead of
|
||||
//! completing none.
|
||||
//!
|
||||
//! The choice is deterministic per prover: among the untouched whole segments still inside their deadline by a
|
||||
//! margin, the lowest FNV-1a of (first block, this prover's key hash) wins, so several provers spread over the
|
||||
//! candidates without a coordinator; the per-block fallback (`prover::choose`) stays for the passes where no whole
|
||||
//! segment qualifies.
|
||||
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
|
||||
use crate::prover::Work;
|
||||
|
||||
/// The least time a claimed segment is given before its deadline (DAA units, about one a second on devnet): the
|
||||
/// chain of 8 empty blocks took 135.6 s cold beside the miner (bench-log, 5 October 2026), so 240 leaves the
|
||||
/// submission and the carrying block inside the window.
|
||||
pub const SEGMENT_MARGIN_MIN_DAA: u64 = 240;
|
||||
/// The margin grows with what the last segment actually took, times this.
|
||||
pub const SEGMENT_MARGIN_FACTOR: f64 = 1.5;
|
||||
/// How far back the work list reaches (chain blocks): the record window, so every open segment inside the
|
||||
/// deadline is visible.
|
||||
pub const WORK_LOOKBACK: u64 = 600;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct SegmentWork {
|
||||
pub first: u64,
|
||||
pub last: u64,
|
||||
/// the last block's DAA score; the deadline is it plus `proving_v1_unproven_daa`
|
||||
pub last_daa: u64,
|
||||
pub deadline_daa: u64,
|
||||
/// (chain block number, block hash, shard index) in chain order
|
||||
pub shards: Vec<(u64, String, u32)>,
|
||||
/// the shard payouts summed (what the shards earn when carried)
|
||||
pub shard_wei: u128,
|
||||
}
|
||||
|
||||
/// The segment holding chain block `number` on the grid that starts at `start`.
|
||||
pub fn segment_of(start: u64, n: u64, number: u64) -> (u64, u64) {
|
||||
let n = n.max(1);
|
||||
let k = number.saturating_sub(start) / n;
|
||||
(start + k * n, start + k * n + n - 1)
|
||||
}
|
||||
|
||||
/// The DAA margin a segment must have before its deadline: the floor, or 1.5 times the last segment's wall time.
|
||||
pub fn need_daa(last_segment_secs: Option<f64>) -> u64 {
|
||||
let from_last = last_segment_secs.map(|s| (s * SEGMENT_MARGIN_FACTOR).ceil() as u64).unwrap_or(0);
|
||||
from_last.max(SEGMENT_MARGIN_MIN_DAA)
|
||||
}
|
||||
|
||||
/// Groups the node's work list into whole, untouched segments: every block of the segment is in the list, every
|
||||
/// listed shard is open (past its exclusive window), unpaid and not in this node's pool from us. A segment with a
|
||||
/// block missing (inside the exclusive window, or outside the lookback) or a shard already paid is not a candidate.
|
||||
pub fn whole_segments(start: u64, n: u64, unproven_daa: u64, work: &[Work]) -> Vec<SegmentWork> {
|
||||
let n = n.max(1);
|
||||
let mut by_block: BTreeMap<u64, Vec<&Work>> = BTreeMap::new();
|
||||
for w in work.iter().filter(|w| w.number >= start) {
|
||||
by_block.entry(w.number).or_default().push(w);
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
for number in by_block.keys() {
|
||||
let (first, last) = segment_of(start, n, *number);
|
||||
if !seen.insert(first) {
|
||||
continue;
|
||||
}
|
||||
let mut shards = Vec::new();
|
||||
let mut wei: u128 = 0;
|
||||
let mut last_daa = 0;
|
||||
let mut whole = true;
|
||||
for b in first..=last {
|
||||
let Some(entries) = by_block.get(&b) else {
|
||||
whole = false;
|
||||
break;
|
||||
};
|
||||
let mut e: Vec<&&Work> = entries.iter().collect();
|
||||
e.sort_by_key(|w| w.shard);
|
||||
e.dedup_by_key(|w| w.shard);
|
||||
if e.iter().any(|w| !w.open || w.paid || w.in_pool) {
|
||||
whole = false;
|
||||
break;
|
||||
}
|
||||
for w in e {
|
||||
shards.push((w.number, w.hash.clone(), w.shard));
|
||||
wei = wei.saturating_add(w.shard_wei);
|
||||
if b == last {
|
||||
last_daa = w.daa;
|
||||
}
|
||||
}
|
||||
}
|
||||
if whole && !shards.is_empty() {
|
||||
out.push(SegmentWork { first, last, last_daa, deadline_daa: last_daa.saturating_add(unproven_daa), shards, shard_wei: wei });
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// FNV-1a 64 of the segment's first block and this prover's key hash: the per-prover rank.
|
||||
pub fn rank(first: u64, key_hash: &str) -> u64 {
|
||||
let mut h: u64 = 0xcbf29ce484222325;
|
||||
for b in first.to_be_bytes().iter().chain(key_hash.as_bytes()) {
|
||||
h ^= *b as u64;
|
||||
h = h.wrapping_mul(0x100000001b3);
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
/// The segments to try, best first: inside the deadline by `need` DAA at `tip_daa`, not attempted, ranked by
|
||||
/// `rank(first, key)` (ties by the older first block).
|
||||
pub fn candidates(segs: &[SegmentWork], tip_daa: u64, need: u64, key_hash: &str, attempted: &HashSet<u64>) -> Vec<SegmentWork> {
|
||||
let mut c: Vec<SegmentWork> = segs.iter().filter(|s| !attempted.contains(&s.first) && s.deadline_daa >= tip_daa.saturating_add(1).saturating_add(need)).cloned().collect();
|
||||
c.sort_by(|a, b| rank(a.first, key_hash).cmp(&rank(b.first, key_hash)).then(a.first.cmp(&b.first)));
|
||||
c
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn w(number: u64, shard: u32, daa: u64, open: bool, paid: bool, in_pool: bool) -> Work {
|
||||
Work { number, hash: format!("0x{number:064x}"), shard, pgas: 0, tx_count: 0, assigned: false, open, paid, in_pool, shard_wei: 10, key_hash: String::new(), daa }
|
||||
}
|
||||
|
||||
fn grid(start: u64, n: u64, segments: u64, daa0: u64) -> Vec<Work> {
|
||||
(0..segments * n).map(|i| w(start + i, 0, daa0 + i, true, false, false)).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_grid_is_counted_from_the_first_v1_block() {
|
||||
assert_eq!(segment_of(100, 8, 100), (100, 107));
|
||||
assert_eq!(segment_of(100, 8, 107), (100, 107));
|
||||
assert_eq!(segment_of(100, 8, 108), (108, 115));
|
||||
assert_eq!(segment_of(100, 8, 123), (116, 123));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_whole_open_unpaid_untouched_segments_qualify() {
|
||||
let mut work = grid(100, 4, 3, 1000); // 100..111, three segments
|
||||
work.retain(|x| x.number != 105); // 104..107 has a block missing (inside its exclusive window, say)
|
||||
work.iter_mut().find(|x| x.number == 110).unwrap().paid = true; // 108..111 has a paid shard
|
||||
let segs = whole_segments(100, 4, 600, &work);
|
||||
assert_eq!(segs.len(), 1);
|
||||
assert_eq!((segs[0].first, segs[0].last), (100, 103));
|
||||
assert_eq!(segs[0].shards.len(), 4);
|
||||
assert_eq!(segs[0].last_daa, 1003);
|
||||
assert_eq!(segs[0].deadline_daa, 1603);
|
||||
assert_eq!(segs[0].shard_wei, 40);
|
||||
// a shard of ours already in the pool, or one still exclusive, also disqualifies
|
||||
let mut work = grid(100, 4, 1, 1000);
|
||||
work[1].in_pool = true;
|
||||
assert!(whole_segments(100, 4, 600, &work).is_empty());
|
||||
let mut work = grid(100, 4, 1, 1000);
|
||||
work[3].open = false;
|
||||
assert!(whole_segments(100, 4, 600, &work).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_block_with_several_shards_lists_them_in_order() {
|
||||
let mut work = grid(100, 2, 1, 1000);
|
||||
work.push(w(101, 1, 1001, true, false, false));
|
||||
work.push(w(100, 1, 1000, true, false, false));
|
||||
let segs = whole_segments(100, 2, 600, &work);
|
||||
assert_eq!(segs[0].shards.iter().map(|(n, _, s)| (*n, *s)).collect::<Vec<_>>(), vec![(100, 0), (100, 1), (101, 0), (101, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_deadline_margin_and_the_attempted_set_filter_the_candidates() {
|
||||
let work = grid(100, 8, 4, 1000); // 100..131, deadlines 1607, 1615, 1623, 1631
|
||||
let segs = whole_segments(100, 8, 600, &work);
|
||||
assert_eq!(segs.len(), 4);
|
||||
// at tip DAA 1380 with a 240 margin only the segments with a deadline at or past 1621 remain
|
||||
let c = candidates(&segs, 1380, 240, "0xkey", &HashSet::new());
|
||||
let firsts: Vec<u64> = c.iter().map(|s| s.first).collect();
|
||||
assert_eq!(firsts.len(), 2);
|
||||
assert!(firsts.contains(&116) && firsts.contains(&124));
|
||||
let mut attempted = HashSet::new();
|
||||
attempted.insert(firsts[0]);
|
||||
let c2 = candidates(&segs, 1380, 240, "0xkey", &attempted);
|
||||
assert_eq!(c2.len(), 1);
|
||||
assert_eq!(c2[0].first, firsts[1]);
|
||||
// past every deadline: nothing
|
||||
assert!(candidates(&segs, 1700, 240, "0xkey", &HashSet::new()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_order_is_deterministic_per_key_and_differs_between_keys() {
|
||||
let work = grid(100, 8, 6, 1000);
|
||||
let segs = whole_segments(100, 8, 600, &work);
|
||||
let a = candidates(&segs, 1000, 240, "0xaaaa", &HashSet::new());
|
||||
let a2 = candidates(&segs, 1000, 240, "0xaaaa", &HashSet::new());
|
||||
assert_eq!(a, a2);
|
||||
assert_eq!(a.len(), 6);
|
||||
// two provers rank the six candidates differently (the spread); the sets are the same
|
||||
let b = candidates(&segs, 1000, 240, "0xbbbb", &HashSet::new());
|
||||
let (fa, fb): (Vec<u64>, Vec<u64>) = (a.iter().map(|s| s.first).collect(), b.iter().map(|s| s.first).collect());
|
||||
let mut sa = fa.clone();
|
||||
let mut sb = fb.clone();
|
||||
sa.sort();
|
||||
sb.sort();
|
||||
assert_eq!(sa, sb);
|
||||
assert_ne!(fa, fb, "two keys should not rank six segments identically");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_margin_follows_the_last_segment_time() {
|
||||
assert_eq!(need_daa(None), 240);
|
||||
assert_eq!(need_daa(Some(100.0)), 240);
|
||||
assert_eq!(need_daa(Some(190.0)), 285);
|
||||
}
|
||||
}
|
||||
|
|
@ -221,6 +221,15 @@ pub struct ProvingState {
|
|||
/// proving v1: segment records this machine aggregated and submitted, and the aggregator's last line
|
||||
pub aggregated: u32,
|
||||
pub segment_note: String,
|
||||
/// proving v1 segment path (6 October 2026): whole segments this machine proved and submitted, paid, and what
|
||||
/// they paid (wei as a decimal string, see `paid_wei`); the last segment's wall time
|
||||
pub segments_submitted: u32,
|
||||
pub segments_paid: u32,
|
||||
#[serde(serialize_with = "u128_string")]
|
||||
pub segment_paid_wei: u128,
|
||||
pub segment_last_s: f64,
|
||||
/// segment records the node refused by the chain rule and this machine offers again each pass
|
||||
pub segments_held: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Default)]
|
||||
|
|
|
|||
|
|
@ -1054,6 +1054,11 @@ if (typeof document !== 'undefined') (function () {
|
|||
setText('pv-submitted', String(pv.submitted || 0));
|
||||
setText('pv-paid', String(pv.paid || 0));
|
||||
setText('pv-paid-sub', pv.paid_wei ? (Number(pv.paid_wei) / 1e18).toFixed(4) + ' IGN earned' : 'shards paid out');
|
||||
// proving v1 segments (6 October 2026): whole segments this machine proved, and the segment path's last line
|
||||
var segLine = '';
|
||||
if (pv.segments_submitted) segLine = 'Segments: ' + pv.segments_submitted + ' proven whole, ' + (pv.segments_paid || 0) + ' paid' + (pv.segment_paid_wei && Number(pv.segment_paid_wei) ? ' (' + (Number(pv.segment_paid_wei) / 1e18).toFixed(4) + ' IGN to the aggregator)' : '') + (pv.segment_last_s ? ', the last in ' + Math.round(pv.segment_last_s) + ' s' : '') + (pv.segments_held ? ', ' + pv.segments_held + ' record' + (pv.segments_held > 1 ? 's' : '') + ' held for the chain rule' : '') + '.';
|
||||
if (pv.segment_note && enabled) segLine += (segLine ? ' ' : '') + pv.segment_note + '.';
|
||||
$('pv-seg-note').hidden = !segLine; setText('pv-seg-note', segLine);
|
||||
var v = View.verifierWords(pv);
|
||||
setText('pv-verifier', v.word); $('pv-verifier').className = 'big-word ' + v.tone;
|
||||
$('pv-verifier-note').hidden = !pv.verifier_note; setText('pv-verifier-note', pv.verifier_note || '');
|
||||
|
|
|
|||
|
|
@ -219,6 +219,7 @@
|
|||
<div class="cell"><div class="k">proven</div><div class="v" id="pv-submitted">0</div><div class="s">proofs sent to the node</div></div>
|
||||
<div class="cell"><div class="k">paid</div><div class="v" id="pv-paid">0</div><div class="s" id="pv-paid-sub">shards paid out</div></div>
|
||||
</div>
|
||||
<p class="note" id="pv-seg-note" hidden></p>
|
||||
<div class="grid2">
|
||||
<div class="card">
|
||||
<div class="card-head"><h3>Verifier</h3><div class="eyebrow">the node's check</div></div>
|
||||
|
|
|
|||
|
|
@ -2086,3 +2086,140 @@ Reported by the aggregation-cost agent: PC 2's `/api/state` answered `{}` (2 byt
|
|||
|
||||
Cause: `ProvingState.paid_wei: u128` and serde_json `to_value` (1.0.151, `value/ser.rs` `serialize_u128`: u64 range or an error); the error became `json!({})`. Fix: the field serialises as a decimal string; `state_json` logs the error once. Test `a_paid_total_over_u64_max_still_serialises_the_whole_state` (`cargo test --offline -q paid_wei`, 1 passed).
|
||||
|
||||
| The fast-time 3-node harness (`tools/proving-v1/net.mjs`, 29950+, suffix 956, every node in trust mode, three vmine voters, v0 at DAA 60, v1 at DAA 120, 4 blocks a segment, unproven after 60 DAA, a tenth to the aggregator; fork b177718e built on this Mac) | run 2, 19:13:01Z to 19:16:19Z, under the run lock: PASSED, 21 checks in 197.3 s (`tools/proving-v1/report-2026-10-05.json`). v1 start = chain block 119 on all three nodes; the native statement identical on all three. Known-finished: segment 119..122's fresh-chain record submitted to n1 at t=131.1 s, relayed, verified (trust) and PAID on n0 1.0 s later at chain block 129, 253,611,648,000,000,000 wei = a tenth of the four credits, the same on every node, the payout address holding it. Chain rule: segment 123..126's fresh-chain record refused ("does not chain to segment 119..122 ... proven (record paid at chain block 129)"), the continuing one (chain_len 8) accepted and paid. Known-failed: segment 127..130 left without a record: a fresh-chain record for 131..134 refused while 127..130 was pending ("pending until DAA 191"); at DAA 192 the status read unproven, a late record for 127..130 refused ("unproven: carried after the deadline"), the fresh-chain record for 131..134 accepted and paid with chain_len 4; `segmentsInWindow` proven 3, unproven 1. The shard side: a v1 shard's `shardWei` = 90% of its block's credit. Run 1 (19:10Z) failed in its own tooling (the signer's argument order), fixed |
|
||||
|
||||
## 5 October 2026 (night), aggregation cost on the RTX 5090: what a per-block aggregation spends and what each lever gives (proving engineer, agg-cost)
|
||||
|
||||
the project lead, 5 October 2026: "fix everything else in the numbers tonight". The number under test: the chained segment aggregation cost 9.6 to 9.7 s a block on PC 2's 5090 while the card mined (`chain-pc2-pv1c`, the entry above), 2.2 s on 4 October with the card to itself. Target: under 3 s a block, the miner's slowdown of the prover under 1.5x, the proof statement unchanged. Branch `agg-cost` (worktree `igneum-wt-agg-cost`, from `proving-v1` 219517f). Host changes (statement untouched, `elf/` untouched): the aggregation's stdin build timed apart from the prove call, the deferred-proof count and the SP1 knobs in the RESULT lines, `--mode chain --save-shards` (every shard's compressed proof written next to the results, so `--mode aggregate` re-runs the same proofs under other settings). Jobs: `agg-cost-pc2-1` (21:01:20Z to 21:25:11Z, `tools/proving-v1/pc2-agg-cost.ps1`, the package `igneum-prove-wsl2-aggcost.zip` fetched by `fetch-prove-aggcost` 20:55:39Z, built in WSL2 against the live target dir in 5 s, installed to `/opt/igneum-aggcost`, the live `/opt/igneum` untouched, `--mode id` the pinned pair) and `agg-cost-pc2-2` (21:34:00Z, the same script). The live prover was switched OFF for the runs (its sp1-gpu-server would otherwise be shared through `/tmp/sp1-cuda-0.sock` and carry its own environment; `gpu_server_before running=0`) and ON again at the end. Fixtures: four consecutive live blocks cut from PC 2's own node (86165..86168 at tip 86195, one empty shard each, every one MATCHES natively), the same four for every phase of job 1. App 0.3.9 on PC 2 throughout.
|
||||
|
||||
Known-finished case of the host changes before the GPU (this Mac, CPU, run lock, 20:41Z to 20:44Z): `--mode chain` over `fixtures/chain/block-81046.json` with `--save-shards` (shard 38.5 s, aggregate 43.4 s, the proof file written), then `--mode aggregate` over that saved shard proof with `SP1_WORKER_VERIFY_INTERMEDIATES=false` (46.6 s, the same statement `0x3dedb8ea...`), `--mode verify-segment` VERIFIED in 0.027 s; known-failed: a wrong statement NOT VERIFIED in 0.027 s. Unit tests: `cargo test --release -p igneum-prove-core -p igneum-prove-host`: core 8 passed, host 9 passed and 1 ignored (build lock, 20:53Z).
|
||||
|
||||
### Lever 1, the profile: where a per-block aggregation goes
|
||||
|
||||
| What | Measured (job `agg-cost-pc2-1`) |
|
||||
|---|---|
|
||||
| The host's own share of an aggregation (the stdin build: the AggInput, the proof clones into the request) | 0.000 s on every block, mining or idle (the `stdin` field of every `RESULT chain block` line): everything is inside the one `prove().compressed()` call to the GPU server |
|
||||
| The GPU server's log at `RUST_LOG=info` (phase A0, the same chain of 1, stderr captured) | 1 line: sp1-gpu-server 6.8.1 prints no spans and no timings, so the step costs below are read from the deferred-proof count, not from a profiler |
|
||||
| Aggregation with 1 deferred proof (the first block, no previous proof) against 2 (every chained block), the card mining | 7.9 s against 9.6, 9.6, 9.8 s: the second deferred proof costs 1.7 to 1.9 s under the miner |
|
||||
| The same, the miners paused (phase C, the same fixtures, 21:05:51Z) | 1.7 s against 2.1, 2.1, 2.2 s: the second deferred proof costs 0.4 to 0.5 s alone |
|
||||
| The shard proof of an empty shard | 7.4 to 7.8 s mining, 1.9 to 2.2 s alone |
|
||||
| A whole block (one empty shard plus its aggregation) | 17.1 to 17.3 s mining (end to end 67.4 s for 4 blocks), 4.1 s alone (16.4 s for 4) |
|
||||
| GPU utilisation over the phase (1-s `nvidia-smi` samples) | 93.9% mining (80 samples, the miner's), 15.8% alone (32 samples): the prover alone keeps the card busy a sixth of the time. Its work is short GPU bursts between CPU phases (the executor, the witness and recursion-program generation run on the CPU inside the server), and the miner's kernels fill the gaps |
|
||||
| GPU memory peak | 16,195 MiB mining (the miner's 3.4 GB resident), 14,483 MiB alone |
|
||||
| The slowdown by the miner, same fixtures, same host, 2 min apart | shards 3.6x, the first aggregation 4.6x, a chained aggregation 4.5x, a block 4.2x |
|
||||
| Setup per host process (client plus two key setups) | 13.0 to 15.7 s, mining or not |
|
||||
|
||||
Reading. An aggregation is three or four recursion steps on the card (the aggregator guest's one core shard, its lift, one deferred program per verified proof, the compose), each a burst of under half a second when the card is free. The chained aggregation's extra deferred proof is the only part that grows with the chain rule, 0.4 to 0.5 s alone. Everything else the 9.7 s holds is the miner: with the card at 94% from the lottery kernels, every prover burst waits for a time slice, and a 2.1-s aggregation becomes 9.7 s. The 4 October 2.2 s (two shards, no previous proof, the card to itself) and tonight's 1.7 s (one shard) and 2.1 s (one shard plus the previous proof) agree within the deferred count.
|
||||
|
||||
### Lever 2, batch and tree folds (estimate from the measured step costs; the statement is pinned, no guest was changed tonight)
|
||||
|
||||
A fold of K blocks' shard proofs plus the previous segment proof in ONE aggregator call would cost one core shard, one lift, K + 1 deferred programs and the compose tree in place of K chained aggregations. From the measured rows (alone: a 1-deferred aggregation 1.7 s, each further deferred proof 0.45 s; mining: 7.9 s and 1.8 s):
|
||||
|
||||
| Fold | Deferred proofs per call | Per block, card alone (estimate) | Per block, card mining (estimate) | Rule |
|
||||
|---|---|---|---|---|
|
||||
| chained, as pinned (measured) | 2 | 2.1 s | 9.7 s | one call per block |
|
||||
| batch of 4 | 5 | (1.7 + 4 x 0.45) / 4 = 0.9 s | (7.9 + 4 x 1.8) / 4 = 3.8 s | one call per 4 blocks |
|
||||
| batch of 8 | 9 | (1.7 + 8 x 0.45) / 8 = 0.7 s | (7.9 + 8 x 1.8) / 8 = 2.8 s | one call per 8 blocks |
|
||||
| tree of 4 (2 + 2, then the pair) | 3 per call, 3 calls | 3 x (1.7 + 2 x 0.45) / 4 = 1.9 s | 3 x (7.9 + 2 x 1.8) / 4 = 8.6 s | no gain over the chain: every call pays the fixed part |
|
||||
|
||||
Reading. A batch fold halves to quarters the per-block aggregation but changes the aggregator's statement (`AggInput` carries one block's shards and the guest asserts one block hash), so it is a new pinned guest and a new program id: a provers-off drain and a rollout (proving/README.md, pinned guests). It does not reach 3 s on a mining card by itself (2.8 s at K = 8 is on the line), and the shard proof beside it stays 7.4 s a block on a mining card. The lever that moves both is the card's other job, lever 4. A tree fold gains nothing here because the fixed part of a call (the core shard and the lift) dominates the per-proof part 4 to 1.
|
||||
|
||||
### Levers 3 and 4, two streams and the miner's kernels (job `agg-cost-pc2-2` and the re-run)
|
||||
|
||||
Job `agg-cost-pc2-2` (21:34:00Z to 21:49:22Z) ran with the 5090 idle throughout: job 1's `/api/resume` had left the worker off (below), so the rows that needed the miner (the batch-log2 curve, the two streams beside the miner, the time-slice policy, the chosen combination) are void and wait for a re-run; the idle rows are measured.
|
||||
|
||||
| What | Measured (job `agg-cost-pc2-2`, card idle) |
|
||||
|---|---|
|
||||
| Aggregate-only over job 1's four saved shard proofs (`--mode aggregate --proofs b1;b2;b3;b4 --parent ...`, one process, the same statement `0x3a995f24...` as the chain run), default knobs (phase B0, then C1) | 1.7, 2.0, 2.0, 2.0 s (1, 2, 2, 2 deferred proofs), 8.1 s for four; C1: 1.8, 2.1, 2.1, 2.1 s, 8.3 s |
|
||||
| The same with `SP1_WORKER_VERIFY_INTERMEDIATES=false` (phase B; the server inherits the host's environment, the knob printed in the `sp1 knobs` line) | 1.7, 2.0, 2.0, 2.0 s, 7.8 s for four: no gain (0.3 s over four, inside the run-to-run spread of 0.2 s). The knobs that change the recursion shape (`SP1_WORKER_MAX_COMPOSE_ARITY`, `MAX_REDUCE_ARITY`) were not tried: a different shape is a different recursion key set and the pinned verifier would refuse the proof |
|
||||
| A 4-deferred aggregation (block-344-shards4, four prototype shards of 6.75 M pgas, phase C2) | shards 42.8 s (10.7 s each, the 4 October 10.2 to 10.7 s), aggregation 2.4 s with 4 deferred proofs; GPU peak 28,402 MiB (the prototype shard's 28.3 GB), utilisation 27.7% over the phase. With 1.7 s at one deferred proof and 2.0 to 2.1 s at two: 0.25 s per further deferred proof alone, so a batch of 8 would cost about 3.5 s a call, 0.45 s a block (estimate, the pinned statement forbids it) |
|
||||
| Two host processes at once on the one card (phase G0: chains of 2 on disjoint blocks, started 2 s apart) | both connected to ONE sp1-gpu-server (the first process's child; the socket is per device, `/tmp/sp1-cuda-0.sock`): process 1 shard 2.2 and 3.5 s, aggregation 3.0 and 4.0 s (12.9 s for 2 blocks against 8.2 s alone); process 2 shard 3.3 s, aggregation 3.6 s, then its second block died with `CudaClientError: Failed to read the response: early eof` when process 1 finished and its server exited. GPU 24,911 MiB, utilisation 12.4% and 13.1%. Two streams through SP1 6.8.1's server are serialised on one socket and the second dies with the first: no throughput gain (3 blocks in 33 s against 4 in 16.4 s) and a failure mode; lever 3 is closed on this SP1 version |
|
||||
| Job 3 (`agg-cost-pc2-3`, 22:41:15Z, app 0.3.10, the same script with the socket rule and a card switch): phase A, the app's 5090 miner at 117.0 MH/s mean (n 3, STATUS lines 22:44:45Z to 22:46:11Z), four fresh live blocks 90896..90899 | shards 8.0, 7.8, 7.6, 7.8 s; aggregations 8.0 s (1 deferred), 10.0, 10.0, 10.0 s (2 deferred); 69.5 s for four, 17.8 s a block; GPU 93.8%, peak 16,245 MiB: the job-1 baseline reproduced 100 min later on other blocks |
|
||||
| Job 3's own-miner phases | void: the state reads came back empty (the class below), the card switch did nothing, phase D launched my miner beside the app's (the app's dropped to 62.2 MH/s, mine read 60.6 MH/s), then PC 2's app restarted at 23:03:30Z and the job died with it; no curve point |
|
||||
| The GPU time-slice policy (`nvidia-smi compute-policy --set-timeslice`, the restore job `agg-cost-restore-1`, 23:16:53Z) | "Not Supported" on PC 2 (RTX 5090, driver 13.3, the Windows nvidia-smi, not elevated): the lever is closed on this driver; an elevated try is not worth a slot, the error is the driver's, not a permission's |
|
||||
| The own-miner phases of job 2 | void: no 5090 miner was running to copy the command line from (the worker off since 21:25Z) |
|
||||
|
||||
The curve, job `agg-cost-pc2-6` (01:12:09Z to 01:24:14Z, app 0.3.11, PC 2 to itself; every phase closed before the next job landed on PC 2 at 01:24:21Z). The app's 5090 miner switched off through `/api/cards` (the keys from `settings.json`; the worker was still alive after 120 s, `/api/pause` as the fallback stopped it in 5 s), then the job's OWN miner on the 5090 with the app's command line (`igneum-miner mine ... --worker igneum-worker-cuda.exe --identities 8 --worker-args "--device 0 --pack packs\devnet --race off [--batch-log2 B]"`, the base variant, its STATUS line every 10 s), the same four live blocks 96556..96559 (one empty shard each) proven by `--mode chain` under it, the miner's rate from its own `now=` field (the first two lines skipped). `--batch-log2 B` sets the worker's nonces per kernel launch (2^B; 22 is the worker's default, 4,194,304 nonces, about 35 ms a launch at 120 MH/s; `proto-cuda/nvrtc/worker.cpp`).
|
||||
|
||||
| batch-log2 | Shard proof (4, s) | Aggregation (1 deferred, then 2) (s) | A block (s) | GPU util. (%) | GPU peak (MiB) | Own miner (MH/s wall, n) | Against the card alone (4.1 s a block) |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 22 (the default), phase D | 8.1, 7.8, 7.9, 7.8 | 8.4; 10.3, 10.0, 10.4 | 18.1 | 95.5 | 16,580 | 103.9 (9) | 4.4x |
|
||||
| 20, E20 | 8.1, 7.8, 7.8, 7.8 | 8.4; 10.3, 10.1, 10.1 | 18.0 | 94.9 | 16,461 | 103.7 (8) | 4.4x |
|
||||
| 18, E18 | 7.0, 6.7, 6.7, 6.7 | 7.2; 8.9, 8.8, 8.8 | 15.6 | 91.5 | 16,487 | 99.3 (7), minus 4.4% | 3.8x |
|
||||
| 16, E16 | 5.1, 4.9, 4.9, 4.9 | 5.0; 6.1, 6.2, 6.2 | 11.1 | 85.3 | 16,519 | 83.8 (6), minus 19% | 2.7x |
|
||||
| 16 again, phase H (the job's own choice: the shortest chain) | 5.0, 4.9, 4.8, 4.9 | 4.9; 6.1, 6.2, 6.2 | 11.1 | 85.7 | 16,487 | 84.0 (6) | 2.7x |
|
||||
|
||||
Reading. Between 2^22 and 2^20 nothing moves: the card's time-slice scheduler alternates the two contexts whatever the kernel length above a few milliseconds. From 2^18 down the miner's launches get short enough (about 2 ms at 2^18, 0.5 ms at 2^16) that the prover's bursts find the card sooner, and the miner pays in launch overhead and idle gaps: at 2^16 the prover runs 1.6x faster (18.1 to 11.1 s a block, the chained aggregation 10.2 to 6.2 s) for a fifth of the hash rate, and it is still 2.7x slower than on a card to itself. The trade is about 1 MH/s per 0.37 s of block time at the 2^16 point, and the 3-s aggregation and the 1.5x slowdown are not reachable on a mining card by the kernel length; a 2^14 point (approximate, extrapolated) would be about 8 s a block at about 65 MH/s. The phase E0 (a 4-deferred aggregation under the miner) failed in 0.1 s: its proof paths pointed at `/` where job 1 had left its shard proofs, but job 2's block-344 proofs sit in job 2's own folder (`$JOB` was exported from job 2 on); the 4-deferred cost under the miner stays an estimate (lever 2 above). The app's own 5090 miner ran at 117 MH/s (job 3, 22:44Z) and 110 to 129 MH/s (its STATUS lines at 01:10Z) with the prover beside it, against my miner's 104 MH/s at the default batch: my miner runs the base variant with `--race off` (no tuning file on PC 2), so the curve's rates are relative to each other, not to the app's.
|
||||
|
||||
### Lever 5, the host side under WSL2 (what the chain-mode numbers leave out)
|
||||
|
||||
| What | Measured |
|
||||
|---|---|
|
||||
| The export (`igneum_exportSegments` 0..tip, 75 to 77 MB over curl.exe to a file on `C:`) | 1.1 to 1.5 s |
|
||||
| The cut (`igneum-prove-export` replaying from genesis, then `--mode native`), four blocks | 18 s for four including the native checks (21:01:28Z to 21:01:46Z), about 4 s a block; the export's file sits on `/mnt/c` |
|
||||
| The key setup per host process | 13.0 to 15.7 s on PC 2 (8.0 to 8.5 s on the Mac CPU): `--mode chain` and `--mode aggregate` pay it once per process, the app's loop pays it per shard |
|
||||
| The proof file write through the WSL2 bridge | the 4 October entry ("shard proving on the RTX 5090"): 24 min of unbuffered `save` across `/mnt/c`, fixed by the 4 MB buffer; tonight `--save-shards` wrote the four 1.27 MB proofs inside the chain phase with no visible gap (the A phase's 80.4 s wall against 67.4 s of proving plus 13.0 s of setup) |
|
||||
| Native Linux | not measured: no native Linux machine with an NVIDIA card exists in the project tonight, and the 4 October numbers were also WSL2 (Ubuntu 24.04 under PC 2's Windows). The WSL2 cost inside a `prove()` call is not separable from here; the host-side pieces above are what a native box would also skip or keep |
|
||||
|
||||
### What went wrong, measured
|
||||
|
||||
| What | Fixed |
|
||||
|---|---|
|
||||
| Job 1's per-phase command ran with `$JOB` empty (the bash variables of `vars.sh` were set, not exported, and the command runs in a child bash): `--out /results-A.json`, the saved shard proofs in `/` on the WSL root, so the aggregate-only phases B0, B, C1 and the prototype-shard phase C2 failed in 0.0 s ("No such file") | `export` in `vars.sh`; job 2 reads the proofs from `/` |
|
||||
| Job 1's own-miner phases launched the iGPU miner (the first `igneum-miner mine` process matched; the 5090's is the second) and `if (StartMiner ...)` was always true (PowerShell: a function's emitted RESULT strings are part of its output), so D and E ran with the 5090 idle and the AMD iGPU at 3.4 MH/s: three more idle replicates of the chain (2.0 to 2.2 s shards, 1.8 and 2.2 s aggregations), no curve | the miner matched on `igneum-worker-cuda`, the outcome in a script-scope flag, `--race off` for the own miner (no tuning file on PC 2; a race costs up to 120 s a start) |
|
||||
| Job 1's `/api/resume` at 21:25:11Z answered ok and the 5090 miner stayed off (card state `off`, hash 0.0, 1,760 MiB on the card) until the 0.3.10 restart; job 2 waited its full 600 s for a hash rate and ran its mining phases void | the restore job `tools/proving-v1/pc2-agg-cost-restore.ps1` also posts `/api/start`; the Counter ASIC coordinator opened a task chip for the resume defect |
|
||||
| Jobs 3 and 4 (`agg-cost-pc2-3` 22:41Z on app 0.3.10, `agg-cost-pc2-4` 00:18Z on 0.3.11): every `/api/state` read came back as the two bytes `{}` (job 4's raw-body print: `raw_len=2`; the same reads gave the full state on 0.3.9 at 21:01Z and the AMD agent saw the empty reply at 22:22Z), so the card switch found no card, the app's 5090 miner kept mining, and job 3 ran a second miner beside it (two miners at about 60 MH/s each) while job 4's double-mining guard voided its own-miner phases. The class is the app's, not the reader's: `state_json()` (engine.rs:180) does `serde_json::to_value(st).unwrap_or(json!({}))`, and the value that fails is `ProvingState.paid_wei: u128` (serde_json 1.0.151 refuses a u128 over u64::MAX, 18.45 IGN; the proving-v1 agent's diagnosis): a paid shard averages 1.23 IGN, so the reply empties about 15 paid shards after every app start and comes back at the next restart, which matches the times (full at 21:01Z with paid_wei 0, empty from 22:22Z after the prover had paid from 22:02Z). Fixed on the app branch proving-v1 at 6714a45 (paid_wei as a decimal string, the error logged, an `{"error":...}` reply on any future failure) | job 5 reads the card keys from the app's `settings.json` (`cards`: key to enabled and identities), restores the 5090's 8 identities first (the restore job of 23:16:53Z had set 2: its parser read the next card's value), refuses before any pause when it cannot name the card, waits on the CUDA worker process count for the card to stop, and checks the worker is back at the end |
|
||||
| Job 5 (`agg-cost-pc2-5`, 01:10:44Z) failed at PowerShell's parse in 1 s: `$RestoreIdentities:` inside a double-quoted string (a drive-qualified variable); no card or miner touched | `${RestoreIdentities}:`; the other `$name:` shapes are inside single-quoted bash here-strings |
|
||||
| Job 6's identities step found `settings.json` already at 8 identities under the active key `nvidia:0:NVIDIA GeForce RTX 5090` (a stale key `nvidia:NVIDIA GeForce RTX 5090` carries 2), so no change was sent; job 6's `/api/cards` with the 5090 disabled answered ok but the worker ran on for 120 s, `/api/pause` stopped it in 5 s, and at the end `/api/resume` brought it back in 5 s on 0.3.11 | the card switch keeps the pause as its fallback; the resume path works on 0.3.11 |
|
||||
| PC 2 ran three jobs at once from 01:24Z (`run-prover-on-pc2-20261006` at 01:24:21Z, the ledger suites build at 01:26:15Z, while agg-cost-pc2-6's closing report was still being uploaded): the app does not serialise jobs, "one job per machine at a time" holds only by the coordinator's word; job 6 had closed at 01:24:14Z, so its rows are clean | nothing of mine to fix; a rule for the job runner |
|
||||
| The make-package gate ran the exporter's side files (`block-N.json.node-plan.json`) as fixtures and failed; its execute step took the exclusive `measure` lock for a cycle count and queued 25 min behind a packbench run | the glob skips `.node-plan.json`; the execute step runs under the `run` lock (a count, not a time) |
|
||||
|
||||
### 6 October 2026, 07:12Z to 07:17Z, the host's chain mode with --save-shards records and --prev, on the Mac's CPU
|
||||
|
||||
`tools/lock/with-lock.sh run`, `SP1_PROVER=cpu igneum-prove-host --mode chain --chain proving/fixtures/chain/block-81046.json,block-81047.json --save-shards --out chain-a.json`, then `--chain block-81048.json --save-shards --prev segment-81047-aggregated.bin --out chain-b.json` (the app branch at ce8f34a, Apple M5 Max, CPU prover). The flags the app's segment path needs, before PC 2 (approximate figures: a CPU run, one sample each):
|
||||
|
||||
| Step | Value |
|
||||
|---|---|
|
||||
| Shard proof, CPU, empty block | 34.7 s and 36.3 s |
|
||||
| Aggregation, CPU, 1 then 2 deferred proofs | 39.1 s, 50.8 s |
|
||||
| Chain of 2, end to end | 160.9 s |
|
||||
| Per-shard records written | 2 (number, block_hash, shard, statement, proof_sha256, proof_bytes 1,272,897, proof_file, prove_seconds) |
|
||||
| `--prev` run: base_chain_len, final chain_len | 2, 3 (the chain continued; a wrong previous proof is refused by number and parent hash) |
|
||||
|
||||
### 6 October 2026, 07:52Z to 08:24Z, the segment-aligned prover beside the miner on PC 2's RTX 5090 (job `segments-pc2-pv1c`)
|
||||
|
||||
`tools/proving-v1/pc2-segments.ps1` (app branch 330207d; the host from the package `igneum-prove-wsl2-segal`, built on PC 2 in 7 s warm to `/opt/igneum-segal`, pinned guests unchanged); the app's own prover OFF for the run through `/api/prove`, ON again at the end; the app's miner running (8 identities, batch-log2 22); `SP1_PROVER=cuda`, the stock 6.8.1 GPU server; a 1-s nvidia-smi sampler under every chain. Payouts read on node 1 (read-only, `igneum_getProofRecords` per block at 08:30Z). The miner's rate from the app's uploaded log (`status: ... MH/s` every 30 s, run win-1ccfe586-20261005-235130).
|
||||
|
||||
| Figure | Value | Note |
|
||||
|---|---|---|
|
||||
| Segments claimed in 30 min | 9 (114470, 114654, 114862, 115022, 115198, 115366, 115542, 115710, 115870) | one every 210 s; 32.1 min of loop |
|
||||
| Candidates per pass | 32 to 38 whole segments inside the margin | margin 580 to 589 DAA at claim |
|
||||
| Export (the chain to the segment's last block) | 99.6 to 100.6 MB in 1.4 to 1.6 s | once per segment |
|
||||
| Cut (8 fixtures, the exporter) | 45.1 to 46.0 s | the exporter replays from genesis per block; the next lever |
|
||||
| Chain run wall (8 shards, 8 aggregations, one key setup) | 159.7 to 160.6 s | host `--mode chain --save-shards` |
|
||||
| Shard proofs, 8 per segment | 63.0 to 63.5 s (7.9 s a shard) | empty blocks |
|
||||
| Aggregation, 8 chained | 80.4 to 81.1 s (10.1 s a block) | the fixed cost per block beside the miner |
|
||||
| End to end per segment (export, cut, chain, sign, submit) | 210.0 to 211.2 s | |
|
||||
| GPU memory peak during a chain | 16,484 to 17,573 MiB (miner resident) | the 24 GB tier's gate holds |
|
||||
| GPU utilisation during a chain | 94.9 to 95.3% | |
|
||||
| Shard records accepted | 72 of 72 | 8 per segment |
|
||||
| Shard records paid on chain | 72 of 72 | 0.905 to 2.719 IGN a shard (90% of the credit); carried 180 to 226 blocks after the block |
|
||||
| Segment records accepted | 0 of 9 | every one refused: "does not chain to segment N-8..N-1 (chain_len 8), which is pending until DAA ..." |
|
||||
| Miner alone (the app's prover off), 07:25 to 07:51Z | 117.86 MH/s mean (n=52) | min 46.37 is the switch-off dip at 07:22Z |
|
||||
| Miner beside the segment prover, 07:55 to 08:24Z | 104.90 MH/s mean (n=58, min 98.39, max 119.24) | 12.96 MH/s = 11.0% of the miner, at 95% GPU utilisation from the prover |
|
||||
| The 0.3.11 prover as shipped beside the miner (5 October row) | 5.0 MH/s = 4.0% | one shard per 46 s; this run proves 8 shards per 210 s, 2.8x the shards |
|
||||
| Node 1's v1 window at 08:24Z | pending 59, proven 0, unproven 16, paid segments 0 | unchanged by the run: the chain rule |
|
||||
|
||||
What the refusal is (the fork, `igneum/exec/src/proving.rs` `check_segment_record`): a fresh record (chain_len = N) is valid only when the previous segment is UNPROVEN at the carrier, and the record's own deadline is the previous segment's deadline plus one segment length in DAA, so a fresh record is valid for 8 DAA (about 8 s) per segment and must be carried inside them. With one prover every previous segment is pending at proof time. Fixed on the fork branch behind `proving_v1_fresh_rule_daa` (0f0dda95): from the switch a fresh record is valid whenever the previous segment is not proven; the app holds a refused record and offers it again every pass until the deadline (272b025).
|
||||
|
||||
Run b (`segments-pc2-pv1b`, 07:20Z to 07:51Z) claimed nothing in 88 passes: the driver's segment keys were doubles against int64 hashtable keys (fixed in 330207d); its 30 minutes are the miner-alone baseline above.
|
||||
|
||||
### 6 October 2026, 08:26Z to 08:35Z, the fast-time harness on the fresh-record rule (Mac, `tools/lock/with-lock.sh run`)
|
||||
|
||||
`IGNEUM_PV1_BIN=vendor/igneum-node/target-pv1/release node tools/proving-v1/net.mjs --segment 8 --unproven 10 [--fresh-rule 0]` (fork 0f0dda95, 3 nodes at 60x, ports 29950+):
|
||||
|
||||
| Case | Checks | Time |
|
||||
|---|---|---|
|
||||
| The rule as shipped (no switch): fresh refused while the previous segment is pending (known-failed), accepted after it is unproven | 22 passed | 166.2 s |
|
||||
| `--fresh-rule 0`: fresh accepted while the previous segment is pending, `freshAdmissible` true, still refused after a proven one, the second offer a duplicate ("segment already paid") | 23 passed | 139.9 s |
|
||||
|
||||
|
|
|
|||
|
|
@ -132,8 +132,8 @@ The GPU server of SP1 6.8.1 sets the memory, not the shard: a floor of 13.9 GB f
|
|||
|---|---|---|---|
|
||||
| 32 GB (RTX 5090) | the prototype shard, 28.3 GB, 10.8 s; the v1 shard 20.4 GB, 4.3 s | the prototype shard 30.1 GB, 33 s; the v1 shard 22.2 GB, 13.2 s | on, mine and prove, today |
|
||||
| 24 GB (RTX 4090, 3090) | the v1 shard 20.4 GB; the prototype shard does NOT fit (28.3 GB) | the v1 shard 22.2 GB measured on the 5090's allocation (2.3 GB spare on a 24 GB card; approximate for the card itself) | on, mine and prove, with the line "until the devnet's fee switch its shards are the prototype size, which needs 32 GB, so this card proves from the switch on" |
|
||||
| 16 GB (RTX 5080, 4080) | the shipped server: an empty shard only (13.9 GB); the patched server v3 b37defef at threshold 2^27: the v1 shard 12,915 MiB and 4.3 s, the prototype shard 13,459 MiB and 16.8 s (measured by the prover-floor agent on the 5090's allocation, job `floor-sweep-3`, 00:13 to 00:17Z 6 October; 2^27 + 2^26 gives 16,115 MiB, over the card) | the shipped server: nothing (15.7 GB for an empty shard); the patched server: about 14.7 GB at 2^27 beside the miner (approximate: the measured 1.8 GB the miner adds; the pair is the agent's sweep 4) | off on the shipped server, with the line; on once the patched server ships (the packaging row below) and the pair is measured |
|
||||
| 12 GB (RTX 3060, 4070) | the shipped server: nothing (the floor is 13.9 GB, and the server refuses the card outright); the patched server v3 b37defef at threshold 2^26 (`SP1_GPU_ELEMENT_THRESHOLD=67108864`, the 12 GB profile): **the v1 shard 10,291 MiB and 5.7 s, an empty shard 9,971 MiB and 3.3 s**, the card's 2,089 MiB idle inside the peak and the server's own working set about 8.2 GB (6,535 MiB after Setup), so a 12 GB card proves alone with about 3 GB over it (measured by the prover-floor agent on the 5090's allocation, `floor-sweep-3`; the on-order RTX 3060 run is pending) | the pair (the miner's 1.7 to 1.8 GB and 3x beside it) is the agent's sweep 4; under 9.0 GB mine-and-prove is not yet shown | off on the shipped server; "proves alone" on the patched one once it ships (the packaging row below), mine-and-prove after the pair. the project lead's "make sure we can prove on 12 GB cards" is answered on the 5090's allocation and OPEN on the card itself: the prover-floor agent (branch prover-floor, 5 October night) read SP1 v6.8.1's GPU server source (`sp1-gpu/crates/prover_components/src/builder.rs` lines 35 to 39): it reads the card's memory, adds 4 and panics under 24 ("Unsupported GPU memory ... must be at least 24GB"), and builds its core (ELEMENT_THRESHOLD 2^28 + 2^27 elements + 2^21), recursion (2^27), shrink (2^25) and wrap (85 M element) provers at Setup whatever the mode, which is the 13.9 GB floor; no knob reaches them, so the fix is a server rebuilt from source on PC 2 (WSL2, nvcc 12.8, CUDA_ARCHS=120) with those sizes cut, measured on the same fixtures and recipe as the curve above (D2 carries the curve) |
|
||||
| 16 GB (RTX 5080, 4080) | the shipped server: an empty shard only (13.9 GB); the patched server v3 b37defef at threshold 2^27: the v1 shard 12,915 MiB and 4.3 s, the prototype shard 13,459 MiB and 16.8 s (measured by the prover-floor agent on the 5090's allocation, job `floor-sweep-3`, 00:13 to 00:17Z 6 October; 2^27 + 2^26 gives 16,115 MiB, over the card) | the shipped server: nothing (15.7 GB for an empty shard); the patched server at 2^27 beside the miner (the 5090 mining at 95%, 338 W, same card; `floor-sweep-4`, 00:35 to 00:38Z): the v1 shard 14,786 MiB total with the miner's 3,833 MiB resident inside it, the server's own 10,953 MiB, 17.4 s a shard; on a 16 GB card that is 10.95 GB server + 1.7 GB miner = 12.7 GB plus the display, under the 15.0 GB line | off on the shipped server, with the line; on (mines and proves, 17 s a v1 shard, 4.3x the alone time) once the patched server ships (the packaging row below) |
|
||||
| 12 GB (RTX 3060, 4070) | the shipped server: nothing (the floor is 13.9 GB, and the server refuses the card outright); the patched server v3 b37defef at threshold 2^26 (`SP1_GPU_ELEMENT_THRESHOLD=67108864`, the 12 GB profile): **the v1 shard 10,291 MiB and 5.7 s, an empty shard 9,971 MiB and 3.3 s**, the card's 2,089 MiB idle inside the peak and the server's own working set about 8.2 GB (6,535 MiB after Setup), so a 12 GB card proves alone with about 3 GB over it (measured by the prover-floor agent on the 5090's allocation, `floor-sweep-3`; the on-order RTX 3060 run is pending) | measured beside the miner (`floor-sweep-4`): at 2^26 the v1 shard 12,066 MiB total with the miner's 3,833 MiB inside, the server's own 8,233 MiB, 24.4 s; the empty shard 11,586 MiB, 13.0 s; 2^25 gains nothing (12,066 MiB, 43.5 s). On a 12 GB card that is 8.2 GB server + 1.7 GB miner = 9.9 GB before the display, over the 9.0 GB line the project lead set, so mine-and-prove on 12 GB is NOT claimed | off on the shipped server; "proves alone" on the patched one once it ships (the packaging row below); mine-and-prove stays off on 12 GB (9.9 GB plus the display, over the 9.0 GB line). The public gate stays "12 GB proves; 16 GB mines and proves", both on the patched server, both pending a run on the card itself. the project lead's "make sure we can prove on 12 GB cards" is answered on the 5090's allocation and OPEN on the card itself: the prover-floor agent (branch prover-floor, 5 October night) read SP1 v6.8.1's GPU server source (`sp1-gpu/crates/prover_components/src/builder.rs` lines 35 to 39): it reads the card's memory, adds 4 and panics under 24 ("Unsupported GPU memory ... must be at least 24GB"), and builds its core (ELEMENT_THRESHOLD 2^28 + 2^27 elements + 2^21), recursion (2^27), shrink (2^25) and wrap (85 M element) provers at Setup whatever the mode, which is the 13.9 GB floor; no knob reaches them, so the fix is a server rebuilt from source on PC 2 (WSL2, nvcc 12.8, CUDA_ARCHS=120) with those sizes cut, measured on the same fixtures and recipe as the curve above (D2 carries the curve) |
|
||||
| under 12 GB | nothing | nothing | off, mine only |
|
||||
| AMD-only and Apple machines | nothing on the GPU: no zkVM proves on an AMD GPU today (`docs/analysis/amd-proving.md`, branch amd-prove); the CPU prover is about 5 minutes a shard at a 30 GB RSS whatever the shard size (PC 1, bench-log "the SP1 CPU prover on PC 1") | | off, "mines and does not prove"; the only non-NVIDIA path with a shipped backend is RISC Zero's Metal prover behind the `ProofSystem` seam (a second guest and pinned id, a verifier for both formats, no shared aggregation): an open item, not 0.3.11 |
|
||||
|
||||
|
|
@ -143,10 +143,81 @@ The aggregation-cost agent's first rows (branch agg-cost, 5 October 2026 night,
|
|||
|
||||
The re-plans of block 344 at 2.25 M and 4.5 M pgas peak at 28.3 to 28.4 GB alone (the server's buffers step up between 4.7 M and 20 M cycles and are flat to 60 M), so no shard size between the v1 budget and the prototype one changes a tier; with the miner the adopted shard proves 3.1x slower (13.2 s against 4.2 s) and the chained aggregation 9.7 s against 2.5 s: a mining 24 GB card delivers one adopted-size shard plus one aggregation in about 23 s, inside T by 25x.
|
||||
|
||||
## Segment-aligned proving (6 October 2026, the project lead: "find a way to solve this")
|
||||
|
||||
**The fault was the work order, not the rules.** The shipped loop took the newest open shard each pass (`prover::choose`), so one prover scattered one block in about 45 across the segment grid and no segment ever had all its blocks proven: pending 56, proven 0, unproven 20 at 06:28Z. No consensus parameter moves.
|
||||
|
||||
**The change (app branch, commit ce8f34a, app and host):**
|
||||
|
||||
| Part | What it does | Where |
|
||||
|---|---|---|
|
||||
| Whole-segment claiming | the work list (lookback 600, the record window) grouped into whole untouched segments: every block present, every shard open (past its 10-DAA exclusive window), unpaid, not in our pool | `app/igneum-app/src/segments.rs` `whole_segments` |
|
||||
| The choice | candidates inside their deadline by a margin (240 DAA, or 1.5x the last segment's wall time), ranked by FNV-1a of (first block, this prover's key hash): deterministic per prover, different between provers, so several provers spread over the candidates with no coordinator; an attempted segment is not retried | `segments::candidates`, `rank`, `need_daa` |
|
||||
| The statement check | for the best three: executed and pending; fresh only when the previous segment is not paid and no verified record of it waits in the pool (the chain rule would refuse a fresh record once that one pays); chained (`--prev`) when the previous is paid and its proof is in this node's pool | `prover.rs` `pick_segment` |
|
||||
| The work | one export to the segment's last block, one fixture per block, one host run `--mode chain --save-shards [--prev]` that proves every shard and aggregates the segment in one process (one key setup), then every shard record signed and submitted (the shard payouts, 90% of the credit) and the segment record signed and submitted (the aggregator share, 10%) | `prover.rs` `prove_segment` |
|
||||
| The fallback | when no whole segment qualifies, the per-block path as shipped | `prover::choose` |
|
||||
| The host | `--mode chain` takes `--prev <aggregated.bin>` (chain_len continues; a previous proof that is not the parent block's is refused by number and parent hash) and with `--save-shards` writes per-shard records (statement, proof sha256, file, time) into the results | `proving/igneum-prove/host/src/main.rs` `run_chain` |
|
||||
| The tile | "Segments: N proven whole, M paid (x IGN to the aggregator), the last in T s" and the segment path's last line; the state carries `segments_submitted`, `segments_paid`, `segment_paid_wei`, `segment_last_s` | `ui/app.js`, `state.rs` |
|
||||
|
||||
Tests: the grid, the grouping (missing block, paid shard, our shard in the pool, exclusive shard), the margin and the attempted set, the per-key order (deterministic, different between two keys), the margin from the last time: 6 unit tests in `segments.rs`; 120 app tests, 8 core and 9 host tests pass. The host flags were run on the Mac's CPU first (bench-log, 07:12Z to 07:17Z): per-shard records written for a chain of 2, then a chain of 1 continued from it (`base_chain_len` 2, final `chain_len` 3).
|
||||
|
||||
**Item 2, "own pool only", answered from the source:** a relayed proof record carries its proof bytes (`protocol/flows/src/v10/proving.rs`: `IgneumProofRecordMessage { record, proof }`, 8 MB bound, handed to the pool with `local = false`), and so does a segment record (message 75). So "this node's pool" is every record relayed to it, and any aggregator can already fold any 8 proven blocks it has received; no node or consensus change is needed for that. What does limit carriage: a block template carries only entries the node's own verifier marked verified (`template_segment_section`, `template_section`), so a node with the verifier `Off` (node 1) never carries a record and a node with `Trust` carries unverified ones; PC 2's node runs the host and verifies. With one prover the carrier is PC 2's own next block.
|
||||
|
||||
**What one 5090 completes (arithmetic from the 5 October rows, the measurement below replaces it):** a chain of 8 empty blocks took 135.6 s cold beside the miner; one export and one key setup per segment instead of eight; so about one segment every 150 to 200 s, 9 to 12 segments an hour out of 450 (2 to 3%), against none. The aggregator share of a segment is 8 x 0.088 IGN = 0.70 IGN plus the 8 shards' 90% share; the forfeited share of the other 97% stays in the escrow until the fleet grows (47 mining cards or 6 dedicated provers for 100% at 1 block/s).
|
||||
|
||||
**PC 2 measurement (job `segments-pc2-pv1c`, 07:52Z to 08:24Z, `tools/proving-v1/pc2-segments.ps1`; bench-log "the segment-aligned prover beside the miner"):**
|
||||
|
||||
| Figure | Value |
|
||||
|---|---|
|
||||
| Whole segments proven in 30 min, one 5090 beside its miner | 9, one every 210 s (export 1.5 s, cut 45 s, chain 160 s: 8 shards 63 s, 8 aggregations 80 s) |
|
||||
| Shard records accepted and paid | 72 of 72, 0.91 to 2.72 IGN a shard (90% of the credit), carried 180 to 226 blocks after the block |
|
||||
| Segment records accepted | 0 of 9: refused by the chain rule as shipped (below) |
|
||||
| Miner's cost | 117.86 to 104.90 MH/s, 13.0 MH/s = 11.0% (the shipped prover: 5.0 MH/s, 4.0%, for 2.8x fewer shards) |
|
||||
| GPU memory peak | 16.5 to 17.6 GB with the miner resident; 24 GB tier unchanged |
|
||||
|
||||
**The second fault, found by the measurement: the chain rule as shipped.** `check_segment_record` accepts a fresh record (chain_len = N) only when the previous segment is UNPROVEN at the carrier. A segment's own deadline is its last block's DAA plus 600, the previous segment's deadline is 8 DAA earlier, so a fresh record is valid for 8 DAA (8 s on devnet) and must be proven before and carried inside them. The refusal on the live node, 9 times: "segment 114470..114477 does not chain to segment 114462..114469 (chain_len 8), which is pending until DAA 169681". The fast-time harness passed on 5 October because its chain continued from proven segments (case 2) and its fresh case ran exactly in that window (case 3, 4 DAA at N = 4). No prover-side move escapes it: the record must be proven, submitted and carried inside the window, which the 210-s proof cannot meet.
|
||||
|
||||
**The fix (fork branch 0f0dda95, behind a switch, never by default):** `proving_v1_fresh_rule_daa`; from it a fresh record is valid whenever the previous segment is not proven (pending or unproven) at the carrier; after a proven one a record must still chain. Two records that do not chain each attest their own blocks against the native statement, so nothing is lost but the longer proof chain, which restarts. In the consensus digest only once set (the v1 pattern), so a 0.3.12 node on the live devnet keeps the 0.3.11 digest until the override file sets it; `igneum_getProvingStatus.v1.freshRuleDaa/freshRuleActive` and `igneum_getSegmentStatement.freshAdmissible` report it. Tests: the digest moves once set; fresh after a pending segment passes from the switch, is refused before it, never after a proven one; the harness `--fresh-rule 0` inverts case 3. This is a rule change in the execution layer, not a parameter tuning, and the code proved it unavoidable: the project lead's "no consensus parameter change unless the code proves it is unavoidable" is met by the 8-DAA window above and the nine refusals. The 0.3.12 coordinator sets the height (the same tip + 14,400 rule) in the override object with the release.
|
||||
|
||||
**Until the switch:** the app (272b025) holds a refused segment record and offers it again every pass until the segment's deadline, so on the shipped rule it lands only if a carrier falls inside the 8-DAA window, and from the switch it lands on the first retry; the shard records (90% of the credit) land either way, 8 per segment.
|
||||
|
||||
**Per tier, with this change and the switch:**
|
||||
|
||||
| Card | What it does | Per 30 min, empty blocks (measured on the 5090, approximate elsewhere) |
|
||||
|---|---|---|
|
||||
| 32 GB mining and proving (5090) | 9 whole segments, 72 shards paid, 9 aggregator shares once the switch is set | miner 11.0% down; 72 x 0.9 IGN = 65 IGN of shard payouts measured, plus 9 x 0.70 IGN aggregator share from the switch |
|
||||
| 24 GB mining and proving | the same path at the 16.5 to 17.6 GB peak measured; the fee-switch shard not yet measured on a 24 GB card | approximate: the 5090's numbers |
|
||||
| 16 GB | mines and proves on the patched server only (prover-floor rows); segment path untested there | pending the prover-floor agent's build |
|
||||
| 12 GB prove-only | proves alone on the patched server (10.3 GB); a dedicated prover takes 4 s a block alone (agg-cost rows), so about one segment every 40 s | approximate: 45 segments per 30 min, 6 such cards for 100% |
|
||||
| A rig (several cards) | one prover loop per machine today; the segment path claims one segment at a time on the aggregation card | the per-card loop is the next item |
|
||||
|
||||
## v1 live on devnet (6 October 2026, C47)
|
||||
|
||||
v1 active at 154,800 (crossed at DAA 154,814, 03:51:42Z); first segment record: none, because on a one-prover devnet no segment can be proven. The app's aggregator (`aggregate_once`, 0.3.11) needs a shard proof of every shard of every block of the segment in its node's pool, and PC 2 alone proves 13 shards per 10 minutes of about 600 blocks (2.2% coverage), so a run of 8 consecutive proven blocks never occurs: node 1 at 04:16Z reads segmentsInWindow pending 55, proven 0, unproven 20, paidSegments 0, and PC 2's app log (run win-1ccfe586-20261005-235130, 04:11Z to 04:16Z) reads every 42 s "aggregator: segment N..N+7: waiting for shard proofs N/0 ... N+7/0 in this node's pool", all 8 missing, each segment then past its 600-DAA deadline unproven. No fault in the node, the app or the record path; the fast-time harness passed because its shards ran at 90% coverage. Meanwhile the aggregator share (a tenth of every block's pool credit) stays in the escrow; shard payouts continue; miners and block watchers see nothing. What ends it: coverage at 8 consecutive blocks, 47 mining 5090-class cards with the shard loop as shipped in 0.3.11 (13 shards per 10 minutes a card), 18 mining cards through the chain mode, or 6 (approximate) proving-only cards, at 1 block/s on empty blocks (the fleet table above), or the segment length lowered on a small devnet (`proving_v1_segment_blocks`, a consensus param, so a digest change). No PC 2 job and no 0.3.12 item follow from this; the open item is the fleet, not the code.
|
||||
|
||||
## The empty `/api/state` reply (6 October 2026)
|
||||
|
||||
The aggregation-cost agent's jobs read the two bytes `{}` from `/api/state` on PC 2 at 22:22Z, 22:41Z and 00:18Z (0.3.10 and 0.3.11); the 21:01Z reply was full. Cause, from the app source and node 1's RPC: `ProvingState.paid_wei` is a `u128`, and serde_json's `to_value` refuses a u128 over u64::MAX (18,446,744,073,709,551,615 wei, 18.45 IGN); `state_json()` turned that refusal into `json!({})` with no log line. A paid shard is 1.23 IGN on average (node 1, `igneum_getProvingStatus`: 814.64 IGN over 663 shards at 00:3xZ), so the fifteenth paid shard after an app start empties the reply. PC 2's prover was blind to the root-owned socket from 20:00:56Z to 22:01Z (paid_wei stayed 0, hence the full reply at 21:01Z), proved from 22:02:13Z, and crossed 18.45 IGN inside its first 15 paid shards, before 22:22Z. Every app restart resets the counter, so the reply comes back for about 15 shards and goes again.
|
||||
|
||||
What it means: the dashboard on a proving machine shows nothing within about 12 minutes of its prover's first payout; every PC playbook that reads a card from `/api/state` fails the same way (the agent's job 5 reads settings.json instead). Mining, proving and payouts are untouched; it is the status page only.
|
||||
|
||||
Fix on the app branch: `paid_wei` serialises as a decimal string (the dashboard already reads it with `Number()`), `state_json` logs `[error] state_json: ...` once instead of answering `{}`, and the reply on any future serialisation error carries `error` and `version` rather than nothing; unit test `a_paid_total_over_u64_max_still_serialises_the_whole_state`. Ships with 0.3.11 if the shipper takes the new code tip, otherwise 0.3.12; until then the workaround is settings.json for the card keys.
|
||||
Fix on the app branch: `paid_wei` serialises as a decimal string (the dashboard already reads it with `Number()`), `state_json` logs `[error] state_json: ...` once instead of answering `{}`, and the reply on any future serialisation error carries `error` and `version` rather than nothing; unit test `a_paid_total_over_u64_max_still_serialises_the_whole_state`. Not in 0.3.11 (that tree closed at 22c2363, master 630da6b, published); 6714a45 heads 0.3.12, the morning's first cut, app only, before PC 1's relaunch (coordinator, counter-asic-2-rollout.md 8a); until then the workaround is settings.json for the card keys.
|
||||
|
||||
## Aggregation cost (5 October, night)
|
||||
|
||||
the project lead, 5 October 2026: "fix everything else in the numbers tonight". Branch `agg-cost`; every number in `docs/bench-log.md`, "aggregation cost on the RTX 5090", with its job id. The proof statement and the pinned guests are unchanged: every existing fixture proof still verifies (`verify-segment` 0.027 s on the Mac, 0.036 to 0.041 s on PC 2).
|
||||
|
||||
| What | Before (5 October evening, `chain-pc2-pv1c`) | After (5 October night) |
|
||||
|---|---|---|
|
||||
| Chained aggregation, the card mining | 9.6 to 9.7 s a block | 9.6 to 9.8 s a block, the same (job `agg-cost-pc2-1`, phase A); the miner's presence is the whole cost: 2.1 to 2.2 s a block with the card to itself, 1.7 s unchained |
|
||||
| Shard proof (empty shard), the card mining | 7.3 to 7.7 s | 7.4 to 7.8 s; 1.9 to 2.2 s with the card to itself |
|
||||
| The miner's slowdown of the prover | 3 to 4x (against 4 October) | measured on the same fixtures 2 min apart: shards 3.6x, chained aggregation 4.5x, a block 4.2x |
|
||||
| Where the time goes | not profiled | the host's share 0.000 s (the prove call is everything); the GPU server prints no timings; the second deferred proof (the chain rule) costs 0.4 to 0.5 s alone and 1.7 to 1.9 s under the miner; the prover alone keeps the card busy 15.8% of the time, the miner 93.9% |
|
||||
| SP1 knobs (`SP1_WORKER_VERIFY_INTERMEDIATES=false`) | not tried | no gain: 7.8 s against 8.1 s over four aggregations, inside the spread; the shape knobs would change the recursion keys the pinned verifier accepts |
|
||||
| Batch fold (K blocks in one aggregator call) | not estimated | estimate from the measured step costs: 0.9 s a block alone and 3.8 s mining at K = 4, 0.7 and 2.8 s at K = 8 (0.25 s per further deferred proof alone, 1.8 s mining); a tree fold gains nothing. A new pinned guest and program id either way, so not tonight |
|
||||
| Two prover processes on one card | not tried | closed on SP1 6.8.1: both share one GPU server socket, run slower together (6.9 s a block against 4.1) and the second dies with the first (`early eof`) |
|
||||
| The miner's kernel length (`--batch-log2` of the CUDA worker, 2^B nonces a launch; job `agg-cost-pc2-6`, the 5090 alone with the job's own miner) | not tried | 2^22 (the default) and 2^20: 18.1 and 18.0 s a block, 10.0 to 10.4 s a chained aggregation, 104 MH/s; 2^18: 15.6 s, 8.8 s, 99 MH/s (minus 4%); 2^16: 11.1 s, 6.1 to 6.2 s, 84 MH/s (minus 19%), reproduced |
|
||||
| The GPU time-slice policy (`nvidia-smi compute-policy --set-timeslice`) | not tried | "Not Supported" on PC 2 (driver 13.3, Windows): closed |
|
||||
| The chosen combination | the defaults | the defaults stay: batch-log2 22 and SP1's default knobs. The one knob that moves the prover (2^16) costs a fifth of the hash rate all the time for a prover that is busy a few seconds a minute on the devnet; it is the project lead's trade, not a default (below) |
|
||||
|
||||
Reading. The per-block aggregation is 2.1 s and a block 4.1 s on a 5090 that only proves, 9.7 and 17.5 s on one that also mines; no knob, fold or stream on tonight's SP1 changes the first pair, and only the miner's kernel length changes the second, at 1 MH/s per 0.37 s of block time. So "under 3 s a block" and "under 1.5x" are met on a card that is not mining and are not reachable on one that is. What that means per tier: a 5090 that mines and proves delivers a proven empty block every 17.5 s (6 cards for 1 block/s), the same card proving only every 4.1 s (2 cards, plus the shard work of full blocks: the fleet table above), and a batch fold of the aggregator (a new pinned guest) would bring the proving-only card to about 2.7 s a block and the mining one to about 12 s. What is being done: the app and host defaults are left as measured; the plan's open decision for the project lead is whether a card that holds a shard assignment should drop to 2^16 for the proof's minute (1.6x faster proof, 19% of its hash rate for that minute) or whether proving-only cards carry the aggregation (the clean 2.1 s), and the batch fold goes on the next pin's list. The state class found on the way (`/api/state` answering `{}` once `paid_wei` passes u64::MAX, fixed on the app branch at 6714a45) is in the bench log with the rest.
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@
|
|||
"proving_v1_activation_daa": 18446744073709551615,
|
||||
"proving_v1_segment_blocks": 8,
|
||||
"proving_v1_unproven_daa": 10,
|
||||
"proving_v1_fresh_rule_daa": 0,
|
||||
"proving_v1_aggregator_share_bps": 1000,
|
||||
"fees": {"pgas": {"version": 1, "cycles_per_pgas": 1000, "intrinsic_pgas_per_tx": 300, "modexp_base": 10, "modexp_per_byte_numer": 1, "modexp_per_byte_denom": 10}, "block_proving_gas_limit": 120000, "shard_proving_gas_budget": 30000, "min_execution_base_fee_wei": 100000000000, "min_proving_base_fee_wei": 10000000000000, "initial_execution_base_fee_wei": 100000000000, "initial_proving_base_fee_wei": 10000000000000, "base_fee_change_denominator": 8}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -93,9 +93,16 @@ fn run() -> Result<()> {
|
|||
// proof (the chain rule of design 5.3), the measurement of docs/plans/proving-v1.md step 2
|
||||
let list = arg("--chain").or_else(|| args.get(1).filter(|a| !a.starts_with("--")).cloned()).context("--chain <f1.json,f2.json,...> (consecutive fixtures)")?;
|
||||
let fixtures: Vec<String> = list.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect();
|
||||
return run_chain(&pinned, &fixtures, prover, out_path.as_deref());
|
||||
// --save-shards writes every shard's compressed proof next to the results (block-N-shard-i-compressed.bin),
|
||||
// so `--mode aggregate` can re-run the aggregation of the same proofs under other settings
|
||||
let save_shards = args.iter().any(|a| a == "--save-shards");
|
||||
// --prev <file>: the previous segment's aggregated proof; the chain continues from it (chain_len grows past
|
||||
// the segment length, the chain rule of spec 7.8) instead of starting fresh. The app's segment path (6 October
|
||||
// 2026) passes it when the node reports the previous segment paid and its proof in the pool.
|
||||
let prev = arg("--prev");
|
||||
return run_chain(&pinned, &fixtures, prover, out_path.as_deref(), save_shards, prev.as_deref());
|
||||
}
|
||||
let path = args.get(1).filter(|a| !a.starts_with("--")).context("usage: igneum-prove-host <fixture.json> [--mode native|execute|shard|compressed|block|all] [--shard N] [--budget <test pgas>] [--prover 0x..] [--out results.json]; --mode chain --chain <f1,f2,...> [--prover 0x..] [--out results.json]; --mode aggregate --proofs <a.bin,...> --parent 0x.. [--prev prev.bin] [--out results.json]; --mode verify --proof <file> --statement 0x..; --mode verify-segment --proof <file> --statement 0x..; --mode id")?;
|
||||
let path = args.get(1).filter(|a| !a.starts_with("--")).context("usage: igneum-prove-host <fixture.json> [--mode native|execute|shard|compressed|block|all] [--shard N] [--budget <test pgas>] [--prover 0x..] [--out results.json]; --mode chain --chain <f1,f2,...> [--prover 0x..] [--out results.json] [--save-shards] [--prev prev.bin]; --mode aggregate --proofs <a.bin,...> --parent 0x.. [--prev prev.bin] [--out results.json]; --mode verify --proof <file> --statement 0x..; --mode verify-segment --proof <file> --statement 0x..; --mode id")?;
|
||||
let shard_index: usize = arg("--shard").map(|s| s.parse()).transpose()?.unwrap_or(0);
|
||||
// `--budget <pgas>`: re-plan the fixture's block at a TEST budget (the S_p curve of 5 October 2026); the fixture's
|
||||
// own per-shard plan is then not compared (the chain and the sums still are), and `--out` records the cut
|
||||
|
|
@ -577,6 +584,8 @@ fn setup_sp1(pinned: &pinned::Pinned, results: &mut serde_json::Map<String, serd
|
|||
let sp1 = Sp1ProofSystem::from_env(pinned.shard_elf(), pinned.agg_elf())?;
|
||||
let setup_s = t.elapsed().as_secs_f64();
|
||||
println!("RESULT setup: {:.2} s, ProofSystem v{} shard program id {} aggregator id {} at {}", setup_s, Sp1ProofSystem::VERSION, sp1.program_id(), sp1.aggregator_id(), now());
|
||||
println!("RESULT sp1 knobs: {}", Sp1ProofSystem::env_knobs());
|
||||
results.insert("sp1_env_knobs".into(), Sp1ProofSystem::env_knobs().into());
|
||||
if sp1.program_id() != pinned.shard_id || sp1.aggregator_id() != pinned.agg_id {
|
||||
bail!("SP1's key setup derived shard program id {} and aggregator id {} from the embedded guests, the pinned manifest says {} and {}: this host would make proofs no other node accepts (re-pin with proving/igneum-prove/pin-guests.sh)", sp1.program_id(), sp1.aggregator_id(), pinned.shard_id, pinned.agg_id);
|
||||
}
|
||||
|
|
@ -616,7 +625,7 @@ fn out_dir_of(out_path: Option<&str>) -> std::path::PathBuf {
|
|||
/// `--mode chain`: every fixture in order, consecutive on the chain (number and parent hash), each block's shards
|
||||
/// proven compressed and aggregated with the previous block's aggregated proof (`AggInput.prev`, the chain rule),
|
||||
/// every proof verified. One RESULT line per shard, per block (with the running totals) and for the chain.
|
||||
fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_path: Option<&str>) -> Result<()> {
|
||||
fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_path: Option<&str>, save_shards: bool, prev_path: Option<&str>) -> Result<()> {
|
||||
if fixtures.is_empty() {
|
||||
bail!("--chain needs at least one fixture");
|
||||
}
|
||||
|
|
@ -654,16 +663,34 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_
|
|||
println!("RESULT chain native block {}: {} shard(s), pgas {}, gas {}, pre {} post {}", f.block.env.number, shards.len(), outcome.pgas_used, outcome.gas_used, pre_root, outcome.state_root);
|
||||
built.push((f.block.env.number, f.block.env.hash, shards, pre_root));
|
||||
}
|
||||
// the previous segment's proof: the chain continues from it (its block must be the parent of the first fixture)
|
||||
let mut prev: Option<proof_system::Sp1SegmentProof> = match prev_path {
|
||||
None => None,
|
||||
Some(p) => {
|
||||
let bytes = std::fs::read(p).with_context(|| format!("read {p}"))?;
|
||||
let proof: sp1_sdk::SP1ProofWithPublicValues = bincode::deserialize(&bytes).with_context(|| format!("{p} is not a bincode SP1 proof"))?;
|
||||
let output = BlockOutput::from_bytes(proof.public_values.as_slice()).with_context(|| format!("{p}: public values are not a block statement"))?;
|
||||
if output.number + 1 != first || output.block_hash != loaded[0].1.block.env.parent_hash {
|
||||
bail!("--prev attests block {} ({}), the chain starts at block {first} with parent {}: the previous proof must be the parent block's", output.number, output.block_hash, loaded[0].1.block.env.parent_hash);
|
||||
}
|
||||
println!("RESULT chain prev: block {} chain_len {} (the chain continues from it)", output.number, output.chain_len);
|
||||
Some(proof_system::Sp1SegmentProof { proof, output })
|
||||
}
|
||||
};
|
||||
let base_len = prev.as_ref().map(|p| p.output.chain_len).unwrap_or(0);
|
||||
let sp1 = setup_sp1(pinned, &mut results)?;
|
||||
let chain_t = Instant::now();
|
||||
let mut prev: Option<proof_system::Sp1SegmentProof> = None;
|
||||
let mut blocks_json = Vec::with_capacity(built.len());
|
||||
let (mut shard_total, mut agg_total, mut shards_total) = (0.0f64, 0.0f64, 0usize);
|
||||
let out_dir = out_dir_of(out_path);
|
||||
let mut shard_files: Vec<String> = Vec::new();
|
||||
for (number, _hash, shards, _) in &built {
|
||||
let block_t = Instant::now();
|
||||
let mut proofs: Vec<Sp1ShardProof> = Vec::with_capacity(shards.len());
|
||||
let mut shard_secs = Vec::new();
|
||||
// with --save-shards: what a shard's proof record carries (the statement, the proof's sha256, the file), so
|
||||
// the app's segment path signs and submits every shard of the chain from one run
|
||||
let mut shard_records: Vec<serde_json::Value> = Vec::new();
|
||||
for s in shards {
|
||||
let i = s.output.shard_index;
|
||||
stage(&format!("chain block {number} compressed shard {i}"));
|
||||
|
|
@ -676,12 +703,24 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_
|
|||
}
|
||||
shard_secs.push(dt);
|
||||
shard_total += dt;
|
||||
if save_shards {
|
||||
let file = out_dir.join(format!("block-{number}-shard-{i}-compressed.bin"));
|
||||
let bytes = bincode::serialize(&p.proof)?;
|
||||
std::fs::write(&file, &bytes).with_context(|| format!("write {}", file.display()))?;
|
||||
let proof_hash: [u8; 32] = sha2::Sha256::digest(&bytes).into();
|
||||
shard_records.push(serde_json::json!({
|
||||
"number": number, "block_hash": s.input.env.hash.to_string(), "shard": i, "statement": alloy_primitives::keccak256(s.output.to_bytes()).to_string(),
|
||||
"proof_sha256": format!("0x{}", hex::encode(proof_hash)), "proof_bytes": bytes.len(), "proof_file": file.display().to_string(), "prove_seconds": dt,
|
||||
}));
|
||||
shard_files.push(file.display().to_string());
|
||||
}
|
||||
proofs.push(p);
|
||||
}
|
||||
shards_total += proofs.len();
|
||||
stage(&format!("chain block {number} aggregate {} shards{}", proofs.len(), if prev.is_some() { " with the previous block proof" } else { "" }));
|
||||
let seg = sp1.aggregate(prev.as_ref(), &proofs)?;
|
||||
let adt = sp1.last_timing("aggregate").unwrap_or_default().as_secs_f64();
|
||||
let sdt = sp1.last_timing("aggregate-stdin").unwrap_or_default().as_secs_f64();
|
||||
agg_total += adt;
|
||||
let claim = SegmentClaim::from_block(&seg.output);
|
||||
let ok = sp1.verify_segment(&seg, &claim);
|
||||
|
|
@ -690,9 +729,10 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_
|
|||
let block_s = block_t.elapsed().as_secs_f64();
|
||||
let cumulative = chain_t.elapsed().as_secs_f64();
|
||||
println!(
|
||||
"RESULT chain block {number}: {} shards ({:.1} s of shard proofs), aggregate prove {adt:.1} s, proof {bytes} bytes, verify {vdt:.3} s, {}; chain_len {}, agg_vk {}; this block {block_s:.1} s, cumulative {cumulative:.1} s over {} block(s) at {}",
|
||||
"RESULT chain block {number}: {} shards ({:.1} s of shard proofs), aggregate prove {adt:.1} s (stdin {sdt:.3} s, {} deferred proofs), proof {bytes} bytes, verify {vdt:.3} s, {}; chain_len {}, agg_vk {}; this block {block_s:.1} s, cumulative {cumulative:.1} s over {} block(s) at {}",
|
||||
seg.output.shard_count,
|
||||
shard_secs.iter().sum::<f64>(),
|
||||
proofs.len() + usize::from(prev.is_some()),
|
||||
if ok { "VERIFIED (shard program id, aggregator id and claim checked)" } else { "VERIFY FAILED" },
|
||||
seg.output.chain_len,
|
||||
seg.output.agg_vk,
|
||||
|
|
@ -702,19 +742,21 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_
|
|||
if !ok {
|
||||
bail!("the aggregated proof of block {number} did not verify");
|
||||
}
|
||||
let expected_len = blocks_json.len() as u64 + 1;
|
||||
let expected_len = base_len + blocks_json.len() as u64 + 1;
|
||||
if seg.output.chain_len != expected_len {
|
||||
bail!("block {number}: chain_len {} is not {expected_len}", seg.output.chain_len);
|
||||
}
|
||||
blocks_json.push(serde_json::json!({
|
||||
"number": number, "shards": seg.output.shard_count, "shard_prove_seconds": shard_secs, "aggregate_prove_seconds": adt,
|
||||
"number": number, "shards": seg.output.shard_count, "shard_prove_seconds": shard_secs, "aggregate_prove_seconds": adt, "aggregate_stdin_seconds": sdt,
|
||||
"aggregate_verify_seconds": vdt, "proof_bytes": bytes, "chain_len": seg.output.chain_len, "block_seconds": block_s, "cumulative_seconds": cumulative,
|
||||
"post_root": seg.output.post_root.to_string(), "statement": alloy_primitives::keccak256(seg.output.to_bytes()).to_string(),
|
||||
"shard_records": shard_records,
|
||||
}));
|
||||
prev = Some(seg);
|
||||
}
|
||||
let seg = prev.unwrap();
|
||||
let total = chain_t.elapsed().as_secs_f64();
|
||||
results.insert("base_chain_len".into(), base_len.into());
|
||||
let (statement, bytes) = segment_results(&seg, &out_dir, &mut results)?;
|
||||
println!(
|
||||
"RESULT chain: {} blocks {first}..={last}, {shards_total} shards, shard proofs {shard_total:.1} s, aggregation {agg_total:.1} s, end to end {total:.1} s; final proof {bytes} bytes attests chain_len {} (statement {statement}), pre {} post {} provers {} at {}",
|
||||
|
|
@ -732,6 +774,7 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_
|
|||
results.insert("shard_prove_seconds_total".into(), shard_total.into());
|
||||
results.insert("aggregate_prove_seconds_total".into(), agg_total.into());
|
||||
results.insert("chain_seconds".into(), total.into());
|
||||
results.insert("shard_proof_files".into(), shard_files.into_iter().map(serde_json::Value::from).collect::<Vec<_>>().into());
|
||||
drop(sp1);
|
||||
finish(results, out_path.map(|s| s.to_string()))
|
||||
}
|
||||
|
|
@ -794,16 +837,18 @@ fn run_aggregate(pinned: &pinned::Pinned, proofs: &str, parent: &str, prev_path:
|
|||
for shards in &blocks {
|
||||
let number = shards[0].output.number;
|
||||
stage(&format!("aggregate block {number}, {} shards", shards.len()));
|
||||
let deferred = shards.len() + usize::from(prev.is_some());
|
||||
let seg = sp1.aggregate(prev.as_ref(), shards)?;
|
||||
let adt = sp1.last_timing("aggregate").unwrap_or_default().as_secs_f64();
|
||||
let sdt = sp1.last_timing("aggregate-stdin").unwrap_or_default().as_secs_f64();
|
||||
let claim = SegmentClaim::from_block(&seg.output);
|
||||
let ok = sp1.verify_segment(&seg, &claim);
|
||||
let vdt = sp1.last_timing("verify-block").unwrap_or_default().as_secs_f64();
|
||||
println!("RESULT aggregate block {number}: {} shards, prove {adt:.1} s, proof {} bytes, verify {vdt:.3} s, {}; chain_len {}, post {} at {}", seg.output.shard_count, bincode::serialize(&seg.proof)?.len(), if ok { "VERIFIED" } else { "VERIFY FAILED" }, seg.output.chain_len, seg.output.post_root, now());
|
||||
println!("RESULT aggregate block {number}: {} shards, prove {adt:.1} s (stdin {sdt:.3} s, {deferred} deferred proofs), proof {} bytes, verify {vdt:.3} s, {}; chain_len {}, post {} at {}", seg.output.shard_count, bincode::serialize(&seg.proof)?.len(), if ok { "VERIFIED" } else { "VERIFY FAILED" }, seg.output.chain_len, seg.output.post_root, now());
|
||||
if !ok {
|
||||
bail!("the aggregated proof of block {number} did not verify");
|
||||
}
|
||||
per_block.push(serde_json::json!({ "number": number, "shards": seg.output.shard_count, "aggregate_prove_seconds": adt, "aggregate_verify_seconds": vdt, "chain_len": seg.output.chain_len }));
|
||||
per_block.push(serde_json::json!({ "number": number, "shards": seg.output.shard_count, "aggregate_prove_seconds": adt, "aggregate_stdin_seconds": sdt, "deferred_proofs": deferred, "aggregate_verify_seconds": vdt, "chain_len": seg.output.chain_len }));
|
||||
prev = Some(seg);
|
||||
}
|
||||
let seg = prev.unwrap();
|
||||
|
|
|
|||
|
|
@ -259,6 +259,16 @@ impl Sp1ProofSystem {
|
|||
self.timings.lock().unwrap().push((what.to_string(), dt));
|
||||
}
|
||||
|
||||
/// The SP1 prover knobs set in this process's environment (the GPU server inherits them; the names from
|
||||
/// sp1-core-executor 6.8.1 `opts.rs` and sp1-prover 6.8.1 `worker/config.rs`), for the RESULT lines, so a
|
||||
/// measurement names the settings it ran under. "none" when the defaults apply.
|
||||
pub fn env_knobs() -> String {
|
||||
let fixed = ["SHARD_SIZE", "ELEMENT_THRESHOLD", "HEIGHT_THRESHOLD", "FULL_SIZE_SHARDS", "MINIMAL_TRACE_CHUNK_THRESHOLD", "TRACE_CHUNK_SLOTS", "MEMORY_LIMIT", "WITHOUT_VK_VERIFICATION", "RUST_LOG"];
|
||||
let mut out: Vec<String> = std::env::vars().filter(|(k, _)| k.starts_with("SP1_WORKER_") || fixed.contains(&k.as_str())).map(|(k, v)| format!("{k}={v}")).collect();
|
||||
out.sort();
|
||||
if out.is_empty() { "none".into() } else { out.join(" ") }
|
||||
}
|
||||
|
||||
pub fn last_timing(&self, what: &str) -> Option<Duration> {
|
||||
self.timings.lock().unwrap().iter().rev().find(|(k, _)| k == what).map(|(_, d)| *d)
|
||||
}
|
||||
|
|
@ -286,6 +296,9 @@ impl ProofSystem for Sp1ProofSystem {
|
|||
/// The aggregator guest over the shard proofs (and the previous segment's proof when given), by recursion.
|
||||
fn aggregate(&self, prev: Option<&Sp1SegmentProof>, shards: &[Sp1ShardProof]) -> Result<Sp1SegmentProof> {
|
||||
let first = shards.first().ok_or_else(|| anyhow!("no shards"))?;
|
||||
// 5 October 2026 (aggregation cost): the stdin build (the proof clones into the request) is timed apart
|
||||
// from the prove call, so the host's own share of an aggregation is visible next to the GPU's.
|
||||
let t_stdin = Instant::now();
|
||||
let mut stdin = SP1Stdin::new();
|
||||
let input = AggInput {
|
||||
shard_vk: self.shard_vk_hash(),
|
||||
|
|
@ -302,6 +315,7 @@ impl ProofSystem for Sp1ProofSystem {
|
|||
let SP1Proof::Compressed(proof) = p.proof.proof.clone() else { return Err(anyhow!("the previous block proof is not a compressed proof")) };
|
||||
stdin.write_proof(*proof, self.agg_vk.vk.clone());
|
||||
}
|
||||
self.record("aggregate-stdin", t_stdin.elapsed());
|
||||
let t = Instant::now();
|
||||
let proof = self.client.prove(&self.agg_pk, stdin).compressed().run()?;
|
||||
self.record("aggregate", t.elapsed());
|
||||
|
|
|
|||
|
|
@ -34,9 +34,13 @@ if [ "${SKIP_GATE:-0}" != "1" ]; then
|
|||
fi
|
||||
H="$ROOT/proving/igneum-prove/target/release/igneum-prove-host"
|
||||
for f in "$ROOT"/proving/fixtures/block-*.json; do
|
||||
# the exporter's side files (block-N.json.node-plan.json, 5 October 2026) are not fixtures
|
||||
case "$f" in *.node-plan.json) continue ;; esac
|
||||
if ! "$H" "$f" --mode native >>"$GATE_LOG" 2>&1; then echo "GATE FAILED: native run of $(basename "$f"); see $GATE_LOG"; exit 1; fi
|
||||
done
|
||||
if ! "$ROOT/tools/lock/with-lock.sh" measure "$H" "$ROOT/proving/fixtures/block-338-shard1.json" --mode execute --shard 0 >>"$GATE_LOG" 2>&1; then
|
||||
# the execute step reports a cycle count, not a time: the `run` lock (tools/lock/with-lock.sh: counts, not ms), so the
|
||||
# gate does not queue behind every build and measurement on the Mac (5 October 2026: 25 min behind a packbench run)
|
||||
if ! "$ROOT/tools/lock/with-lock.sh" run "$H" "$ROOT/proving/fixtures/block-338-shard1.json" --mode execute --shard 0 >>"$GATE_LOG" 2>&1; then
|
||||
echo "GATE FAILED: the guest did not execute the shard fixture (the exact failure the PC hit on 4 October); see $GATE_LOG"; exit 1
|
||||
fi
|
||||
"$H" --mode id | tee -a "$GATE_LOG" # the pinned program ids this package carries (the PC's build embeds the same elf/ files)
|
||||
|
|
|
|||
50
tools/proving-v1/agg-cost-table.mjs
Normal file
50
tools/proving-v1/agg-cost-table.mjs
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
#!/usr/bin/env node
|
||||
// The aggregation-cost curve from one or more pc2-agg-cost.ps1 jobs (5 October 2026 night): per phase, the shard and
|
||||
// aggregation seconds per block (the chain and aggregate-only RESULT lines), the GPU utilisation and memory peak of
|
||||
// the phase, the own miner's rate (MH/s wall) and its batch-log2. Reads the job uploads from the log intake like
|
||||
// tools/jobs.mjs (DATABASE_URL in ~/.config/igneum/env). Prints a markdown table.
|
||||
// node tools/proving-v1/agg-cost-table.mjs agg-cost-pc2-1 agg-cost-pc2-2 ...
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
const ids = process.argv.slice(2);
|
||||
if (!ids.length) { console.error('usage: agg-cost-table.mjs <job id> [...]'); process.exit(2); }
|
||||
const m = /^DATABASE_URL=(.*)$/m.exec(readFileSync(`${homedir()}/.config/igneum/env`, 'utf8'));
|
||||
const url = m[1].trim().replace(/^['"]|['"]$/g, '');
|
||||
const host = new URL(url).hostname.replace('-pooler', '');
|
||||
const sql = async (query, params = []) => {
|
||||
const r = await fetch(`https://${host}/sql`, { method: 'POST', headers: { 'Neon-Connection-String': url, 'Content-Type': 'application/json' }, body: JSON.stringify({ query, params }) });
|
||||
const j = await r.json();
|
||||
if (!r.ok) throw new Error(j.message || JSON.stringify(j));
|
||||
return j.rows;
|
||||
};
|
||||
const f1 = x => (Math.round(x * 10) / 10).toFixed(1);
|
||||
const rows = [];
|
||||
for (const id of ids) {
|
||||
const ups = await sql('SELECT lines FROM miner_logs WHERE run_id LIKE $1 ORDER BY received_at DESC LIMIT 1', [`job-${id}-%`]);
|
||||
if (!ups.length) { console.error(`${id}: no upload`); continue; }
|
||||
const lines = ups[0].lines.split('\n');
|
||||
const phases = new Map();
|
||||
const ph = l => { if (!phases.has(l)) phases.set(l, { job: id, label: l, shard: [], agg: [], deferred: [], util: '', mem: '', rate: '', batch: '', note: '' }); return phases.get(l); };
|
||||
for (const raw of lines) {
|
||||
const l = raw.replace(/^\d+(\.\d+)? job \S+: /, '');
|
||||
let x;
|
||||
if ((x = /^([A-Z0-9]+): RESULT chain block \d+ shard \d+: compressed prove ([\d.]+) s/.exec(l))) ph(x[1]).shard.push(Number(x[2]));
|
||||
else if ((x = /^([A-Z0-9]+): RESULT (?:chain|aggregate) block \d+: .*?(?:aggregate )?prove ([\d.]+) s \(stdin [\d.]+ s, (\d+) deferred/.exec(l))) { ph(x[1]).agg.push(Number(x[2])); ph(x[1]).deferred.push(Number(x[3])); }
|
||||
else if ((x = /^RESULT phase_gpu ([A-Z0-9]+) samples=\d+ memory_used_max_mib=(\d+) util_mean_pct=([\d.]+)/.exec(l))) { ph(x[1]).mem = x[2]; ph(x[1]).util = x[3]; }
|
||||
else if ((x = /^RESULT ([A-Z0-9]+) miner rate (n=\d+ mean=[\d.]+)/.exec(l))) ph(x[1]).rate = x[2].replace('n=', 'n ').replace(' mean=', ', mean ');
|
||||
else if ((x = /^RESULT ([A-Z0-9]+) own miner started .* batch_log2=(\d+)/.exec(l))) ph(x[1]).batch = x[2];
|
||||
else if ((x = /^RESULT ([A-Z0-9]+) own miner (not hashing|EXITED|: no app miner)/.exec(l))) ph(x[1]).note = 'own miner ' + x[2];
|
||||
else if ((x = /^RESULT phase ([A-Z0-9]+) end .* exit (\d+) wall ([\d.]+) s/.exec(l))) { const p = ph(x[1]); p.exit = x[2]; p.wall = x[3]; }
|
||||
else if ((x = /^RESULT phase ([A-Z0-9]+) skipped/.exec(l))) ph(x[1]).note = 'skipped';
|
||||
else if ((x = /^RESULT (H (?:choice|knobs)): (.*)$/.exec(l))) console.error(`${id}: ${x[1]}: ${x[2]}`);
|
||||
}
|
||||
for (const p of phases.values()) rows.push(p);
|
||||
}
|
||||
const mean = a => a.length ? a.reduce((s, v) => s + v, 0) / a.length : NaN;
|
||||
console.log('| Job | Phase | batch-log2 | Shard s (each) | Aggregation s (each, deferred proofs) | Block s (shard + aggregation, mean) | GPU util % | GPU peak MiB | Miner MH/s | Note |');
|
||||
console.log('|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const p of rows) {
|
||||
const chained = p.agg.filter((_, i) => p.deferred[i] >= 2);
|
||||
const block = p.shard.length && p.agg.length ? f1(mean(p.shard) + mean(chained.length ? chained : p.agg)) : '';
|
||||
console.log(`| ${p.job} | ${p.label} | ${p.batch || (p.label.startsWith('E') && /^E\d+$/.test(p.label) ? p.label.slice(1) : '')} | ${p.shard.map(f1).join(', ')} | ${p.agg.map((a, i) => `${f1(a)} (${p.deferred[i]})`).join(', ')} | ${block} | ${p.util} | ${p.mem} | ${p.rate} | ${[p.note, p.exit && p.exit !== '0' ? `exit ${p.exit}` : ''].filter(Boolean).join('; ')} |`);
|
||||
}
|
||||
35
tools/proving-v1/miner-rate.mjs
Normal file
35
tools/proving-v1/miner-rate.mjs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
#!/usr/bin/env node
|
||||
// The hash rate of one miner over a UTC window, from its STATUS lines in the log intake (the app uploads the miner's
|
||||
// log every minute; each STATUS line starts with a unix timestamp and carries `now=<MH/s> MH/s wall`, the last
|
||||
// interval's rate). For the aggregation-cost phases that keep the app's own miner running (docs/bench-log.md,
|
||||
// 5 October 2026 night), where the job cannot read the app's state from PowerShell 5.1.
|
||||
// node tools/proving-v1/miner-rate.mjs <label> <start ISO> <end ISO> e.g. miner-nvidia-1ccfe586-1 2026-10-05T21:00:00Z 2026-10-05T21:05:00Z
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
const [label, start, end] = process.argv.slice(2);
|
||||
if (!label || !start || !end) { console.error('usage: miner-rate.mjs <label> <start ISO> <end ISO>'); process.exit(2); }
|
||||
const m = /^DATABASE_URL=(.*)$/m.exec(readFileSync(`${homedir()}/.config/igneum/env`, 'utf8'));
|
||||
const url = m[1].trim().replace(/^['"]|['"]$/g, '');
|
||||
const host = new URL(url).hostname.replace('-pooler', '');
|
||||
const sql = async (query, params = []) => {
|
||||
const r = await fetch(`https://${host}/sql`, { method: 'POST', headers: { 'Neon-Connection-String': url, 'Content-Type': 'application/json' }, body: JSON.stringify({ query, params }) });
|
||||
const j = await r.json();
|
||||
if (!r.ok) throw new Error(j.message || JSON.stringify(j));
|
||||
return j.rows;
|
||||
};
|
||||
const t0 = Date.parse(start) / 1000, t1 = Date.parse(end) / 1000;
|
||||
// every upload received up to 15 min after the window (the log is re-uploaded whole, so the newest covering upload holds every line)
|
||||
const rows = await sql(`SELECT received_at, lines FROM miner_logs WHERE label = $1 AND received_at > to_timestamp($2) AND received_at < to_timestamp($3) ORDER BY received_at DESC LIMIT 20`, [label, t0, t1 + 900]);
|
||||
const seen = new Map();
|
||||
for (const r of rows) {
|
||||
for (const l of r.lines.split('\n')) {
|
||||
const mm = /^(\d+(?:\.\d+)?) STATUS .* now=([\d.]+) MH\/s wall/.exec(l);
|
||||
if (!mm) continue;
|
||||
const t = Number(mm[1]);
|
||||
if (t >= t0 && t <= t1) seen.set(mm[1], Number(mm[2]));
|
||||
}
|
||||
}
|
||||
const h = [...seen.values()].sort((a, b) => a - b);
|
||||
if (!h.length) { console.log(`${label} ${start}..${end}: no STATUS lines in the window (${rows.length} uploads read)`); process.exit(1); }
|
||||
const mean = h.reduce((a, b) => a + b, 0) / h.length;
|
||||
console.log(`${label} ${start}..${end}: n=${h.length} mean=${mean.toFixed(2)} p50=${h[Math.floor((h.length - 1) / 2)].toFixed(2)} min=${h[0].toFixed(2)} max=${h[h.length - 1].toFixed(2)} MH/s wall (now= of the STATUS lines, ${rows.length} uploads read)`);
|
||||
|
|
@ -37,6 +37,10 @@ const args = process.argv.slice(2);
|
|||
const flag = (name, dflt) => { const i = args.indexOf(name); return i >= 0 && args[i + 1] !== undefined ? +args[i + 1] : dflt; };
|
||||
const SECS = flag('--secs', 1200);
|
||||
const V0 = 60, V1 = flag('--v1', 120), SEG = flag('--segment', 4), UNPROVEN = flag('--unproven', 60), SHARE_BPS = 1000;
|
||||
// --fresh-rule <daa>: the fresh-record rule switch (6 October 2026); default never (the 0.3.11 rule), so case 3 keeps
|
||||
// its known-failed line; with --fresh-rule 0 a fresh record after a PENDING segment is accepted (the new rule) and
|
||||
// case 2's refusal after a PROVEN one still stands
|
||||
const FRESH_RULE = process.argv.includes('--fresh-rule') ? Number(process.argv[process.argv.indexOf('--fresh-rule') + 1]) : null;
|
||||
const started = [];
|
||||
const t0 = Date.now();
|
||||
const since = () => ((Date.now() - t0) / 1000).toFixed(1);
|
||||
|
|
@ -56,6 +60,7 @@ let overrideText = readFileSync(FILE, 'utf8')
|
|||
.replace(/"proving_v1_segment_blocks":\s*\d+/, `"proving_v1_segment_blocks": ${SEG}`)
|
||||
.replace(/"proving_v1_unproven_daa":\s*\d+/, `"proving_v1_unproven_daa": ${UNPROVEN}`)
|
||||
.replace(/"proving_v1_aggregator_share_bps":\s*\d+/, `"proving_v1_aggregator_share_bps": ${SHARE_BPS}`)
|
||||
.replace(/"proving_v1_fresh_rule_daa":\s*\d+/, FRESH_RULE === null ? '"proving_v1_fresh_rule_daa": 18446744073709551615' : `"proving_v1_fresh_rule_daa": ${FRESH_RULE}`)
|
||||
.replace(/"skip_proof_of_work":\s*(true|false)/, '"skip_proof_of_work": true');
|
||||
for (const re of [/"skip_proof_of_work": true/, new RegExp(`"proving_v1_activation_daa": ${V1}`), new RegExp(`"proving_v1_segment_blocks": ${SEG}`), new RegExp(`"proving_v1_unproven_daa": ${UNPROVEN}`)]) if (!re.test(overrideText)) throw new Error(`override edit failed: ${re}`);
|
||||
writeFileSync(override, overrideText);
|
||||
|
|
@ -211,7 +216,13 @@ try {
|
|||
const lastHash3 = stmt3early.blocks[stmt3early.blocks.length - 1].hash;
|
||||
const fresh3 = signSegment(seg3[0], seg3[1], lastHash3, stmt3early.publicValuesFresh, 'seg3-fresh');
|
||||
const subEarly = await n0.eth('igneum_submitSegmentRecord', [{ record: fresh3.record, proof: fresh3.proof }]);
|
||||
check('known-failed: segment 3 cannot start a fresh chain while segment 2 is pending', subEarly.accepted === false && /pending until DAA/.test(subEarly.reason), subEarly);
|
||||
if (FRESH_RULE === null) {
|
||||
check('known-failed: segment 3 cannot start a fresh chain while segment 2 is pending', subEarly.accepted === false && /pending until DAA/.test(subEarly.reason), subEarly);
|
||||
} else {
|
||||
check('fresh-record rule: segment 3 starts a fresh chain while segment 2 is pending', subEarly.accepted === true, subEarly);
|
||||
const stmt3now = await n0.eth('igneum_getSegmentStatement', ['0x' + seg3[0].toString(16)]);
|
||||
check('fresh-record rule: the statement reports the fresh record admissible', stmt3now.freshAdmissible === true, { freshAdmissible: stmt3now.freshAdmissible });
|
||||
}
|
||||
await waitDaa(n0, deadline2 + 2, 'past segment 2 deadline');
|
||||
const stmt2late = await n0.eth('igneum_getSegmentStatement', ['0x' + seg2[0].toString(16)]);
|
||||
check('known-failed: segment 2 is unproven after its deadline', stmt2late.status.status === 'unproven', stmt2late.status);
|
||||
|
|
@ -220,7 +231,13 @@ try {
|
|||
const subLate = await n0.eth('igneum_submitSegmentRecord', [{ record: late2.record, proof: late2.proof }]);
|
||||
check('known-failed: a late record for segment 2 pays nothing (refused as unproven)', subLate.accepted === false && /unproven/.test(subLate.reason), subLate);
|
||||
const subRestart = await n1.eth('igneum_submitSegmentRecord', [{ record: fresh3.record, proof: fresh3.proof }]);
|
||||
check('segment 3 restarts the chain with a fresh-chain record after the unproven segment', subRestart.accepted === true, subRestart);
|
||||
if (FRESH_RULE === null) {
|
||||
check('segment 3 restarts the chain with a fresh-chain record after the unproven segment', subRestart.accepted === true, subRestart);
|
||||
} else {
|
||||
// under the fresh-record rule the record was accepted while segment 2 was still pending (above) and is carried
|
||||
// by now: the second offer is a duplicate or a paid segment, never a chain-rule refusal
|
||||
check('fresh-record rule: the second offer of segment 3\'s record is a duplicate, not a chain-rule refusal', subRestart.accepted === true || !/does not chain/.test(subRestart.reason || ''), subRestart);
|
||||
}
|
||||
const paid3 = await waitSegmentPaid(n0, seg3[0]);
|
||||
check('segment 3 paid with chain_len 4', hexn(paid3.paid.chainLen) === SEG, paid3.paid);
|
||||
st = await n0.eth('igneum_getProvingStatus');
|
||||
|
|
|
|||
57
tools/proving-v1/pc2-agg-cost-restore.ps1
Normal file
57
tools/proving-v1/pc2-agg-cost-restore.ps1
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
# Aggregation cost, the restore job (5 October 2026, night): what pc2-agg-cost.ps1's finally block does, for the case
|
||||
# where the app restarted under the job (the 0.3.10 rollout; a running job dies with the app and its finally block
|
||||
# never runs): stops any miner or CUDA worker the job started itself (a process whose parent is not igneum-app.exe),
|
||||
# resumes the app's miners, switches the prover back on, resets the GPU compute-policy timeslice. Idempotent.
|
||||
$ErrorActionPreference = 'Continue'
|
||||
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
||||
$urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
||||
$base = (Get-Content $urlFile -Raw).Trim().TrimEnd('/')
|
||||
# the state as text parsed by regex (app 0.3.10's response comes back empty to PowerShell 5.1's JSON reader)
|
||||
function State {
|
||||
try {
|
||||
$raw = (Invoke-WebRequest -UseBasicParsing -Uri "$base/api/state" -TimeoutSec 20).Content
|
||||
$cards = @()
|
||||
# one card = the text from its "index" key to the next card's (the fields come in any order inside it; the first
|
||||
# version took the first "identities" AFTER "vendor", which belongs to the NEXT card: 23:16:53Z, the 5090 set to the
|
||||
# iGPU's 2 identities)
|
||||
$idx = @([regex]::Matches($raw, '\{"index":\d+,"key":"') | ForEach-Object { $_.Index })
|
||||
for ($i = 0; $i -lt $idx.Count; $i++) {
|
||||
$end = if ($i + 1 -lt $idx.Count) { $idx[$i + 1] } else { $raw.Length }
|
||||
$seg = $raw.Substring($idx[$i], $end - $idx[$i])
|
||||
$f = { param($re, $def) $x = [regex]::Match($seg, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
||||
$v = (& $f '"vendor":"([^"]*)"' '')
|
||||
if (-not $v) { continue }
|
||||
$cards += [pscustomobject]@{ key = (& $f '"key":"([^"]*)"' ''); name = (& $f '"name":"([^"]*)"' ''); vendor = $v; enabled = ((& $f '"enabled":(true|false)' 'true') -eq 'true'); identities = [int](& $f '"identities":(\d+)' '1'); state = (& $f '"state":"([^"]*)"' ''); hash_now = [double](& $f '"hash_now":([\d.eE+-]+)' '0') }
|
||||
}
|
||||
$g = { param($re, $def) $x = [regex]::Match($raw, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
||||
return [pscustomobject]@{ version = (& $g '"version":"([^"]*)"' ''); settings = [pscustomobject]@{ prove = ((& $g '"settings":\{[^}]*?"prove":(true|false)' 'false') -eq 'true') }; mining = [pscustomobject]@{ paused = ((& $g '"mining":\{[^}]*?"paused":(true|false)' 'false') -eq 'true'); cards = $cards }; proving = [pscustomobject]@{ status = (& $g '"proving":\{[^}]*?"status":"([^"]*)"' '') } }
|
||||
} catch { $null }
|
||||
}
|
||||
function Post($path, $body) { try { (Invoke-RestMethod -Method Post -Uri "$base$path" -ContentType 'application/json' -Body ($body | ConvertTo-Json -Compress -Depth 5) -TimeoutSec 15) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
||||
$st = State
|
||||
"RESULT before $(Stamp) app $($st.version) paused=$($st.mining.paused) prove=$($st.settings.prove) proving=$($st.proving.status)"
|
||||
$apps = @(Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-app*' } | ForEach-Object { $_.Id })
|
||||
"RESULT app_pids $(Stamp) $($apps -join ',')"
|
||||
$killed = 0
|
||||
foreach ($p in (Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -or $_.Name -like 'igneum-worker*' })) {
|
||||
$parent = Get-CimInstance Win32_Process -Filter "ProcessId = $($p.ParentProcessId)" -ErrorAction SilentlyContinue
|
||||
$stray = (-not $parent) -or (($parent.Name -notlike 'igneum-app*') -and ($parent.Name -notlike 'igneum-miner*'))
|
||||
if ($stray) { "RESULT stray $(Stamp) pid $($p.ProcessId) $($p.Name) parent=$($p.ParentProcessId) ($(if ($parent) { $parent.Name } else { 'gone' })): stopped"; Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue; $killed++ }
|
||||
}
|
||||
"RESULT strays_stopped $(Stamp) $killed"
|
||||
& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
||||
function Card($st) { if ($null -eq $st) { return $null }; $st.mining.cards | Where-Object { $_.vendor -eq 'nvidia' } | Select-Object -First 1 }
|
||||
$c = Card (State)
|
||||
if ($c) {
|
||||
$stc = State
|
||||
"RESULT cards_before $(Stamp) $(($stc.mining.cards | ForEach-Object { "$($_.name):enabled=$($_.enabled):identities=$($_.identities):$($_.state)" }) -join ',')"
|
||||
$list = @(); foreach ($k in $stc.mining.cards) { $list += @{ key = $k.key; enabled = $(if ($k.key -eq $c.key) { $true } else { [bool]$k.enabled }); identities = [int]$k.identities } }
|
||||
"RESULT card_on $(Stamp) $(Post '/api/cards' @{ cards = $list })"
|
||||
}
|
||||
"RESULT timeslice $(Stamp) $((& nvidia-smi compute-policy --set-timeslice=0 2>&1) -join ' ')"
|
||||
"RESULT resume $(Stamp) $(Post '/api/resume' @{})"
|
||||
"RESULT start $(Stamp) $(Post '/api/start' @{})"
|
||||
"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})"
|
||||
Start-Sleep -Seconds 20
|
||||
$st = State
|
||||
"RESULT after $(Stamp) app $($st.version) paused=$($st.mining.paused) prove=$($st.settings.prove) proving=$($st.proving.status) cards=$(($st.mining.cards | ForEach-Object { "$($_.name):$($_.state):$([math]::Round($_.hash_now,1))" }) -join ',')"
|
||||
433
tools/proving-v1/pc2-agg-cost.ps1
Normal file
433
tools/proving-v1/pc2-agg-cost.ps1
Normal file
|
|
@ -0,0 +1,433 @@
|
|||
# Aggregation cost (5 October 2026, night; the project lead: "fix everything else in the numbers tonight"): what a per-block
|
||||
# aggregation on PC 2's RTX 5090 spends and what each lever gives. A signed `run` job (shell powershell, not elevated).
|
||||
# The live prover is switched OFF for the run (its sp1-gpu-server would otherwise be shared, socket /tmp/sp1-cuda-0.sock)
|
||||
# and ON again at the end; the app's miners are paused for the idle and own-miner phases and resumed at the end
|
||||
# (try/finally). The live /opt/igneum host is untouched: this build lands in /opt/igneum-aggcost.
|
||||
# Phases (the set at the top; every number is a RESULT line prefixed with the phase):
|
||||
# A0 app miner mining, chain of 1 with RUST_LOG=info: the SP1 GPU server's own log lines (the profile)
|
||||
# A app miner mining, chain of 4 (--save-shards), default knobs: tonight's baseline, shard proofs kept
|
||||
# B0 app miner mining, aggregate-only over A's shard proofs, default knobs
|
||||
# B app miner mining, aggregate-only, SP1_WORKER_VERIFY_INTERMEDIATES=false
|
||||
# C miners paused: chain of 4, default knobs (the idle baseline tonight)
|
||||
# C1 miners paused: aggregate-only over A's shard proofs
|
||||
# C2 miners paused: block-344-shards4 (four full shards, one block, --save-shards): a 4-deferred aggregation idle
|
||||
# D own miner, the app's command line, batch-log2 22 (the worker's default): chain of 4; MH/s from its STATUS lines
|
||||
# E own miner at batch-log2 20, 18, 16: chain of 4 each; MH/s each
|
||||
# E0 own miner 22: aggregate-only over C2's four shard proofs (a 4-deferred aggregation under mining)
|
||||
# F own miner 22 with nvidia-smi compute-policy --set-timeslice=1 (SHORT), chain of 4; the policy restored after
|
||||
# G0 miners paused: two host processes at once (chain of 2 each, disjoint blocks) on the one card
|
||||
# G own miner 22: two host processes at once (chain of 2 each, disjoint blocks) on the one GPU server
|
||||
# H own miner at $BestBatch with the best knobs: chain of 4 (the combination the plan adopts)
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$PhaseSet = @('A0','A','B0','B','C','C1','C2','D','E')
|
||||
$ShardSourceDir = '' # job 2: the WSL directory holding job 1's block-N-shard-i-compressed.bin and block-344 proofs ('' = this job's own)
|
||||
$ShardSourceFirst = 0 # job 2: job 1's first block number (its four shard proofs are consecutive from it)
|
||||
$ShardSourceParent = '' # job 2: the parent chain block hash of that first block
|
||||
$BestBatch = 20
|
||||
$BestKnobs = ''
|
||||
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
||||
$urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
||||
if (-not (Test-Path $urlFile)) { $urlFile = Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
||||
$base = (Get-Content $urlFile -Raw).Trim().TrimEnd('/')
|
||||
$app = if ($env:IGNEUM_APP_DIR) { $env:IGNEUM_APP_DIR } else { Join-Path $env:LOCALAPPDATA 'igneum\app' }
|
||||
# The app's state as plain text parsed by regex (job 3, 22:41Z, app 0.3.10: Invoke-RestMethod's JSON conversion came back
|
||||
# empty to PowerShell 5.1, so the card switch found no card and the own miner ran beside the app's). The card objects are
|
||||
# serialised in struct order (src/state.rs CardState: key ... vendor ... enabled, state, hash_now), so the fields after a
|
||||
# "vendor" belong to that card and the last "key" before it is its key.
|
||||
$script:stateErr = ''
|
||||
$script:nvidiaSeg = ''
|
||||
function State {
|
||||
try {
|
||||
$raw = (Invoke-WebRequest -UseBasicParsing -Uri "$base/api/state" -TimeoutSec 20).Content
|
||||
if (-not $raw) { $script:stateErr = 'empty body'; return $null }
|
||||
$cards = @()
|
||||
# one card = the text from its "index" key to the next card's (the fields come in any order inside it; the first
|
||||
# version took the first "identities" AFTER "vendor", which belongs to the NEXT card: 23:16:53Z, the 5090 set to the
|
||||
# iGPU's 2 identities)
|
||||
$idx = @([regex]::Matches($raw, '\{"index":\d+,"key":"') | ForEach-Object { $_.Index })
|
||||
for ($i = 0; $i -lt $idx.Count; $i++) {
|
||||
$end = if ($i + 1 -lt $idx.Count) { $idx[$i + 1] } else { $raw.Length }
|
||||
$seg = $raw.Substring($idx[$i], $end - $idx[$i])
|
||||
$f = { param($re, $def) $x = [regex]::Match($seg, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
||||
$v = (& $f '"vendor":"([^"]*)"' '')
|
||||
if (-not $v) { continue }
|
||||
if ($v -eq 'nvidia' -and -not $script:nvidiaSeg) { $script:nvidiaSeg = $seg.Substring(0, [Math]::Min(700, $seg.Length)) }
|
||||
$cards += [pscustomobject]@{ key = (& $f '"key":"([^"]*)"' ''); name = (& $f '"name":"([^"]*)"' ''); vendor = $v; enabled = ((& $f '"enabled":(true|false)' 'true') -eq 'true'); identities = [int](& $f '"identities":(\d+)' '1'); state = (& $f '"state":"([^"]*)"' ''); hash_now = [double](& $f '"hash_now":([\d.eE+-]+)' '0') }
|
||||
}
|
||||
$g = { param($re, $def) $x = [regex]::Match($raw, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
||||
$script:stateErr = ''
|
||||
return [pscustomobject]@{
|
||||
version = (& $g '"version":"([^"]*)"' ''); machine_id = (& $g '"machine_id":"([^"]*)"' '')
|
||||
settings = [pscustomobject]@{ prove = ((& $g '"settings":\{[^}]*?"prove":(true|false)' 'false') -eq 'true') }
|
||||
mining = [pscustomobject]@{ paused = ((& $g '"mining":\{[^}]*?"paused":(true|false)' 'false') -eq 'true'); cards = $cards }
|
||||
proving = [pscustomobject]@{ status = (& $g '"proving":\{[^}]*?"status":"([^"]*)"' '') }
|
||||
raw_len = $raw.Length
|
||||
}
|
||||
} catch { $script:stateErr = "$_"; $null }
|
||||
}
|
||||
function Post($path, $body) { try { (Invoke-RestMethod -Method Post -Uri "$base$path" -ContentType 'application/json' -Body ($body | ConvertTo-Json -Compress -Depth 5) -TimeoutSec 15) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
||||
function Rpc($port, $method, $params) {
|
||||
try { (Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$port" -ContentType 'application/json' -Body (@{jsonrpc='2.0'; id=1; method=$method; params=$params} | ConvertTo-Json -Compress -Depth 6) -TimeoutSec 180).result } catch { $null }
|
||||
}
|
||||
function Card($st) { if ($null -eq $st) { return $null }; $st.mining.cards | Where-Object { $_.vendor -eq 'nvidia' } | Select-Object -First 1 }
|
||||
function Smi { try { (& nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu,power.draw --format=csv,noheader,nounits 2>$null) -join ' | ' } catch { 'nvidia-smi failed' } }
|
||||
# The card keys come from the app's own settings.json (`cards`: key -> {enabled, identities, ...}), never from /api/state:
|
||||
# that reply is "{}" whenever the state fails to serialise (job 4, 00:18Z: raw_len=2; the app's state_json falls back
|
||||
# to an empty object), which no parser can read. The state is still read for the hash rate when it answers.
|
||||
function CardPrefs {
|
||||
$f = Join-Path $app 'settings.json'
|
||||
if (-not (Test-Path $f)) { return @{} }
|
||||
$raw = Get-Content $f -Raw
|
||||
$prefs = @{}
|
||||
$cm = [regex]::Match($raw, '"cards":\s*\{')
|
||||
if (-not $cm.Success) { return $prefs }
|
||||
$i = $cm.Index + $cm.Length; $depth = 1; $start = $i
|
||||
while ($i -lt $raw.Length -and $depth -gt 0) { $ch = $raw[$i]; if ($ch -eq '{') { $depth++ } elseif ($ch -eq '}') { $depth-- }; $i++ }
|
||||
$body = $raw.Substring($start, $i - $start - 1)
|
||||
foreach ($m in [regex]::Matches($body, '"([^"]+)":\s*\{([^}]*)\}')) {
|
||||
$k = $m.Groups[1].Value; $v = $m.Groups[2].Value
|
||||
$en = [regex]::Match($v, '"enabled":\s*(true|false)'); $id = [regex]::Match($v, '"identities":\s*(\d+)')
|
||||
$prefs[$k] = @{ enabled = $(if ($en.Success) { $en.Groups[1].Value -eq 'true' } else { $true }); identities = $(if ($id.Success) { [int]$id.Groups[1].Value } else { 1 }) }
|
||||
}
|
||||
return $prefs
|
||||
}
|
||||
function NvidiaKey { $p = CardPrefs; ($p.Keys | Where-Object { $_ -like 'nvidia:*' } | Select-Object -First 1) }
|
||||
function CardSwitch($on, $identities) {
|
||||
$p = CardPrefs; $nk = NvidiaKey
|
||||
if (-not $nk) { return "no nvidia card key in settings.json (keys: $($p.Keys -join ','))" }
|
||||
$list = @(); foreach ($k in $p.Keys) { $list += @{ key = $k; enabled = $(if ($k -eq $nk) { [bool]$on } else { [bool]$p[$k].enabled }); identities = $(if ($k -eq $nk -and $identities) { [int]$identities } else { [int]$p[$k].identities }) } }
|
||||
"prefs_before=$(($p.Keys | ForEach-Object { "$($_):enabled=$($p[$_].enabled):identities=$($p[$_].identities)" }) -join ',') " + (Post '/api/cards' @{ cards = $list })
|
||||
}
|
||||
function CudaWorkers { (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' } | Measure-Object).Count }
|
||||
$job = $env:IGNEUM_JOB_DIR
|
||||
if (-not $job) { $job = Join-Path $env:TEMP 'igneum-agg-cost' }
|
||||
New-Item -ItemType Directory -Force -Path $job | Out-Null
|
||||
$jobStart = Get-Date
|
||||
$st0 = State
|
||||
# the 5090 back to its 8 identities (the restore job of 23:16:53Z set it to 2 through the parser fault above)
|
||||
$RestoreIdentities = 8
|
||||
$p0 = CardPrefs; $nk0 = NvidiaKey
|
||||
"RESULT card_prefs $(Stamp) settings.json cards: $(($p0.Keys | ForEach-Object { "$($_):enabled=$($p0[$_].enabled):identities=$($p0[$_].identities)" }) -join ',') nvidia_key='$nk0'"
|
||||
if (-not $nk0) { "RESULT refused $(Stamp) no nvidia card key in settings.json: nothing is paused and nothing runs"; exit 1 }
|
||||
if ($p0[$nk0].identities -ne $RestoreIdentities) {
|
||||
"RESULT identities_set $(Stamp) $nk0 $($p0[$nk0].identities) -> ${RestoreIdentities}: $(CardSwitch ([bool]$p0[$nk0].enabled) $RestoreIdentities)"
|
||||
Start-Sleep -Seconds 5; $p1 = CardPrefs
|
||||
"RESULT identities_after $(Stamp) settings.json says identities=$($p1[$nk0].identities) enabled=$($p1[$nk0].enabled)"
|
||||
}
|
||||
"RESULT state_read $(Stamp) ok=$($null -ne $st0) raw_len=$($st0.raw_len) cards=$(($st0.mining.cards | Measure-Object).Count) err=$script:stateErr"
|
||||
"RESULT nvidia_card_json $(Stamp) $script:nvidiaSeg"
|
||||
if ($st0 -and ($st0.mining.cards | Measure-Object).Count -eq 0) { "RESULT state_head $(Stamp) no card parsed; the body starts: $((Invoke-WebRequest -UseBasicParsing -Uri "$base/api/state" -TimeoutSec 20).Content.Substring(0, 1500))" }
|
||||
"RESULT start $(Stamp) app $($st0.version) machine $($st0.machine_id) prove_setting=$($st0.settings.prove) proving=$($st0.proving.status) paused=$($st0.mining.paused) phases=$($PhaseSet -join ',') shard_source='$ShardSourceDir' $ShardSourceFirst $ShardSourceParent"
|
||||
"RESULT gpus $(Stamp) $(Smi)"
|
||||
$evmPort = $null
|
||||
foreach ($p in 26790, 26800, 26810) { if (Rpc $p 'igneum_getProvingStatus' @()) { $evmPort = $p; break } }
|
||||
# the NVIDIA worker must be hashing (the app's own miner) before the mining phases mean anything
|
||||
$waited = 0
|
||||
while ($waited -lt 180) {
|
||||
$c = Card (State)
|
||||
if ($c -and $c.hash_now -gt 10) { break }
|
||||
if ($waited % 60 -eq 0) { "WAIT $(Stamp) nvidia worker state=$(if ($c) { $c.state } else { 'na' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' })" }
|
||||
Start-Sleep -Seconds 15; $waited += 15
|
||||
}
|
||||
$c = Card (State)
|
||||
"RESULT miner $(Stamp) nvidia worker $(if ($c -and $c.hash_now -gt 10) { 'hashing' } else { 'NOT hashing: the app-miner phases are void' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' }) after $waited s"
|
||||
# the app's miner command line, for the own-miner phases (the same binary, args, cwd and tuning file)
|
||||
$mp = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' -and $_.CommandLine -like '*igneum-worker-cuda*' } | Select-Object -First 1
|
||||
$minerExe = $null; $minerArgs = $null
|
||||
if ($mp) {
|
||||
$minerExe = $mp.ExecutablePath
|
||||
$cl = $mp.CommandLine
|
||||
if ($cl -match '^\s*"[^"]*"\s*(.*)$') { $minerArgs = $Matches[1] } elseif ($cl -match '^\s*\S+\s+(.*)$') { $minerArgs = $Matches[1] }
|
||||
$minerArgs = $minerArgs -replace '--status-secs \d+', '--status-secs 10'
|
||||
}
|
||||
if (-not $mp) {
|
||||
# the 5090 miner is not running (job 2, 21:34Z: "off" after job 1's resume): the same command line as the app builds for
|
||||
# it (engine.rs miner_args), synthesised from the iGPU miner's line when that one runs, else from the app's install
|
||||
$amd = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' } | Select-Object -First 1
|
||||
$prog = Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner'
|
||||
$cuda = Join-Path $prog 'igneum-worker-cuda.exe'
|
||||
if ($amd -and $amd.CommandLine -match '--evm-address (0x[0-9a-fA-F]{40})') {
|
||||
$evm = $Matches[1]; $minerExe = $amd.ExecutablePath
|
||||
$rpc = if ($amd.CommandLine -match ' mine (\S+) ') { $Matches[1] } else { 'grpc://127.0.0.1:26610' }
|
||||
$minerArgs = "mine $rpc 1 100000000 nvidia-$($st0.machine_id.Substring(0,8))-1 --worker `"$cuda`" --status-secs 10 --exit-on-seed-change --evm-address $evm --identities 8 --payout-label nvidia-$($st0.machine_id.Substring(0,8))-1 --prepare-packs packs\prepare --worker-args `"--device 0 --pack packs\devnet`""
|
||||
"RESULT miner_cmdline_synthesised $(Stamp) from the iGPU miner's line; exe=$minerExe cuda_worker_exists=$(Test-Path $cuda)"
|
||||
} else { "RESULT miner_cmdline_synthesised $(Stamp) no running miner to copy from: the own-miner phases are void" }
|
||||
}
|
||||
$tuning = Join-Path $app 'tuning.json'
|
||||
if (Test-Path $tuning) { $env:IGNEUM_TUNING_FILE = $tuning }
|
||||
"RESULT miner_cmdline $(Stamp) exe=$minerExe tuning_file=$(Test-Path $tuning) args=$minerArgs"
|
||||
# strays from an earlier run that died with an app restart (job 3, 23:03Z): a miner or worker whose parent is not the
|
||||
# app (nor a miner) is ours and is stopped before anything is measured
|
||||
$killed = 0
|
||||
foreach ($p in (Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -or $_.Name -like 'igneum-worker*' })) {
|
||||
$parent = Get-CimInstance Win32_Process -Filter "ProcessId = $($p.ParentProcessId)" -ErrorAction SilentlyContinue
|
||||
if ((-not $parent) -or (($parent.Name -notlike 'igneum-app*') -and ($parent.Name -notlike 'igneum-miner*'))) { "RESULT stray $(Stamp) pid $($p.ProcessId) $($p.Name) parent=$($p.ParentProcessId): stopped"; Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue; $killed++ }
|
||||
}
|
||||
"RESULT strays_stopped $(Stamp) $killed"
|
||||
# the live prover off for the whole run: its sp1-gpu-server would be shared with ours (same socket, its environment)
|
||||
"RESULT prove_off $(Stamp) $(Post '/api/prove' @{on=$false})"
|
||||
# WSL side: the package, the export, the fixtures, the phase runner
|
||||
$pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-aggcost\igneum-prove-wsl2'
|
||||
if (-not (Test-Path $pkg)) { $pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-aggcost' }
|
||||
"RESULT package $(Stamp) $pkg exists=$(Test-Path (Join-Path $pkg 'package'))"
|
||||
function WslPath($p) { $w = (& wsl.exe -d Ubuntu-24.04 -u root -- wslpath -a ($p -replace '\\', '/') 2>$null); if ($w) { ($w -replace "`0", '').Trim() } else { '/mnt/c' + ($p.Substring(2) -replace '\\', '/') } }
|
||||
$pkgW = WslPath $pkg; $jobW = WslPath $job
|
||||
$srcW = if ($ShardSourceDir) { $ShardSourceDir.TrimEnd('/') } else { $jobW }
|
||||
$tipHex = Rpc $evmPort 'eth_blockNumber' @()
|
||||
$tip = [Convert]::ToInt64($tipHex, 16)
|
||||
$first = $tip - 30; $last = $first + 3
|
||||
"RESULT chain $(Stamp) node_evm_port=$evmPort tip=$tip blocks $first..$last"
|
||||
$t = Get-Date
|
||||
$body = (@{jsonrpc='2.0'; id=1; method='igneum_exportSegments'; params=@('0x0', ('0x{0:x}' -f $last))} | ConvertTo-Json -Compress)
|
||||
$seqFile = Join-Path $job 'seq.json'; $bodyFile = Join-Path $job 'export-request.json'
|
||||
[IO.File]::WriteAllText($bodyFile, $body, (New-Object System.Text.UTF8Encoding $false))
|
||||
& curl.exe -s -S -m 600 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bodyFile" -o $seqFile 2>&1 | ForEach-Object { "curl: $_" }
|
||||
if (-not (Test-Path $seqFile) -or (Get-Item $seqFile).Length -lt 1000) { "RESULT export FAILED: no reply file"; exit 1 }
|
||||
"RESULT export $(Stamp) $((Get-Item $seqFile).Length) bytes in $([math]::Round(((Get-Date) - $t).TotalSeconds,1)) s"
|
||||
$vars = @"
|
||||
export PKG='$pkgW'
|
||||
export JOB='$jobW'
|
||||
export SRC='$srcW'
|
||||
export FIRST=$first
|
||||
export LAST=$last
|
||||
"@
|
||||
[IO.File]::WriteAllText((Join-Path $job 'vars.sh'), ($vars -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
||||
$prep = @'
|
||||
set -uo pipefail
|
||||
. "$(dirname "$0")/vars.sh"
|
||||
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
||||
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}"
|
||||
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
DEST="$HOME/igneum-prove-aggcost"; LIVE_TARGET="$HOME/igneum-prove/proving/igneum-prove/target"
|
||||
mkdir -p "$DEST"
|
||||
rsync -a --delete --exclude target "$PKG/package/" "$DEST/"
|
||||
find "$DEST" -name target -prune -o -type f -exec touch {} + 2>/dev/null
|
||||
grep -o '"program_id": "0x[0-9a-f]*"' "$DEST/proving/igneum-prove/elf/manifest.json" | sed 's/^/RESULT manifest /'
|
||||
cd "$DEST/proving/igneum-prove"
|
||||
t0=$(date +%s)
|
||||
if ! CARGO_TARGET_DIR="$LIVE_TARGET" cargo build --release -p igneum-prove-export -p igneum-prove-host --features igneum-prove-host/cuda 2>&1 | tail -3; then echo "RESULT build FAILED"; exit 1; fi
|
||||
echo "RESULT build $(stamp) exit 0 in $(( $(date +%s) - t0 )) s"
|
||||
mkdir -p /opt/igneum-aggcost && cp "$LIVE_TARGET/release/igneum-prove-host" "$LIVE_TARGET/release/igneum-prove-export" /opt/igneum-aggcost/
|
||||
H=/opt/igneum-aggcost/igneum-prove-host; X=/opt/igneum-aggcost/igneum-prove-export
|
||||
echo "RESULT installed $(sha256sum $H | cut -c1-16) host, $(sha256sum $X | cut -c1-16) export; live /opt/igneum untouched: $(sha256sum /opt/igneum/igneum-prove-host | cut -c1-16)"
|
||||
$H --mode id | sed 's/^/RESULT aggcost-host /'
|
||||
python3 -c "import json,sys; d=json.load(open('$JOB/seq.json')); json.dump(d['result'], open('$JOB/export.json','w'))"
|
||||
LIST=""
|
||||
for n in $(seq $FIRST $LAST); do
|
||||
if ! $X "$JOB/export.json" $n "$JOB/block-$n.json" --source "PC 2 live devnet export, aggregation cost run" 2>&1 | tail -1 | sed "s/^/export $n: /"; then echo "RESULT cut $n FAILED"; exit 1; fi
|
||||
$H "$JOB/block-$n.json" --mode native 2>&1 | grep -E "^RESULT (native|plan)" | sed "s/^/block $n /"
|
||||
LIST="$LIST${LIST:+,}$JOB/block-$n.json"
|
||||
done
|
||||
echo "$LIST" > "$JOB/fixtures.txt"
|
||||
cp "$DEST/proving/fixtures/block-344-shards4.json" "$JOB/"
|
||||
# the parent hash of every fixture, for --mode aggregate
|
||||
python3 - <<PY
|
||||
import json
|
||||
for n in range($FIRST, $LAST + 1):
|
||||
f = json.load(open('$JOB/block-%d.json' % n))
|
||||
print('RESULT parent', n, f['block']['env']['parent_hash'] if 'parent_hash' in f['block']['env'] else f['block']['env'].get('parentHash'))
|
||||
open('$JOB/parent-%d.txt' % n, 'w').write(f['block']['env'].get('parent_hash') or f['block']['env'].get('parentHash'))
|
||||
f = json.load(open('$JOB/block-344-shards4.json'))
|
||||
open('$JOB/parent-344.txt', 'w').write(f['block']['env'].get('parent_hash') or f['block']['env'].get('parentHash'))
|
||||
PY
|
||||
echo "RESULT fixtures $(stamp) $LIST"
|
||||
# the live prover's GPU server must be gone (its environment and socket would be ours otherwise); and this job runs as
|
||||
# root, so the socket it leaves would be root-owned and the app's prover (another WSL user) could not connect (the
|
||||
# 21:25Z fault of job 1): the server is killed and the socket removed here and again at the end (the rule of
|
||||
# tools/ci/prover-socket-check.sh)
|
||||
for i in $(seq 1 36); do pgrep -x sp1-gpu-server >/dev/null || break; sleep 5; done
|
||||
pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock
|
||||
echo "RESULT gpu_server_before $(stamp) running=$(pgrep -x sp1-gpu-server | wc -l) socket=$(ls /tmp/sp1-cuda-0.sock 2>/dev/null || echo none)"
|
||||
'@
|
||||
$phase = @'
|
||||
# bash phase.sh <label> <cmd file>: runs the host command in the file (one line, env assignments first) with a 1-s
|
||||
# nvidia-smi sampler underneath; every RESULT/STAGE line comes out prefixed "<label>:"; a gpu line closes the phase.
|
||||
set -uo pipefail
|
||||
. "$(dirname "$0")/vars.sh"
|
||||
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
||||
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}"
|
||||
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
L="$1"; CMD="$2"
|
||||
H=/opt/igneum-aggcost/igneum-prove-host
|
||||
export H
|
||||
nvidia-smi --query-gpu=timestamp,index,memory.used,utilization.gpu,power.draw --format=csv,noheader,nounits -l 1 > "$JOB/smi-$L.csv" 2>/dev/null &
|
||||
SMI=$!
|
||||
t0=$(date +%s.%N); START=$(stamp)
|
||||
echo "RESULT phase $L start $START cmd: $(cat "$CMD" | cut -c1-400)"
|
||||
bash "$CMD" > "$JOB/host-$L.log" 2> "$JOB/host-$L.err"
|
||||
RC=$?
|
||||
WALL=$(python3 -c "import time; print(round(time.time() - $t0, 1))")
|
||||
kill $SMI 2>/dev/null; sleep 1
|
||||
grep -E "^(RESULT|STAGE|igneum-prove-host sources|segment proof written)" "$JOB/host-$L.log" | sed "s/^/$L: /"
|
||||
awk -F', *' '{ if ($3+0 > max) max=$3+0; u+=$4; n++ } END { if (n) printf "RESULT phase_gpu LABEL samples=%d memory_used_max_mib=%d util_mean_pct=%.1f\n", n, max, u/n }' "$JOB/smi-$L.csv" | sed "s/LABEL/$L/"
|
||||
echo "RESULT phase $L end $(stamp) exit $RC wall $WALL s (started $START); stderr $(wc -l < "$JOB/host-$L.err") lines"
|
||||
if [ $RC -ne 0 ]; then tail -5 "$JOB/host-$L.err" | sed "s/^/$L stderr: /"; fi
|
||||
'@
|
||||
foreach ($pair in @(@('prep.sh', $prep), @('phase.sh', $phase))) {
|
||||
[IO.File]::WriteAllText((Join-Path $job $pair[0]), ($pair[1] -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
||||
}
|
||||
function Wsl($file) { & wsl.exe -d Ubuntu-24.04 -u root -- bash (WslPath $file) 2>&1 | ForEach-Object { ($_ -replace "`0", '') } }
|
||||
function Cmd($label, $line) {
|
||||
$f = Join-Path $job "cmd-$label.sh"
|
||||
[IO.File]::WriteAllText($f, ($line + "`n"), (New-Object System.Text.UTF8Encoding $false))
|
||||
return (WslPath $f)
|
||||
}
|
||||
function RunPhase($label, $line) {
|
||||
$el = ((Get-Date) - $jobStart).TotalMinutes
|
||||
if ($el -gt 16.5) { "RESULT phase $label skipped: $([math]::Round($el,1)) min into the job (the 20-min window)"; return }
|
||||
$cmdW = Cmd $label $line
|
||||
& wsl.exe -d Ubuntu-24.04 -u root -- bash "$jobW/phase.sh" $label $cmdW 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
||||
}
|
||||
function Fixtures { (Get-Content (Join-Path $job 'fixtures.txt') -Raw).Trim() }
|
||||
function Parent($n) { (Get-Content (Join-Path $job "parent-$n.txt") -Raw).Trim() }
|
||||
# proof groups for --mode aggregate: the four blocks' shard 0 proofs from the source job (one shard a block tonight)
|
||||
function SrcFirst { if ($ShardSourceFirst -gt 0) { $ShardSourceFirst } else { $first } }
|
||||
function SrcParent { if ($ShardSourceParent) { $ShardSourceParent } else { Parent $first } }
|
||||
function AggGroups($src) { $f = SrcFirst; (($f..($f + 3)) | ForEach-Object { "$src/block-$_-shard-0-compressed.bin" }) -join ';' }
|
||||
# --- own miner ---
|
||||
$script:miner = $null
|
||||
# (job 1, 21:10Z: `if (StartMiner ...)` was always true because the function's RESULT strings are part of its output; the
|
||||
# outcome is the script-scope flag $script:minerOk now)
|
||||
function StartMiner($label, $batchLog2) {
|
||||
$script:minerOk = $false
|
||||
if (-not $minerExe) { "RESULT $label own miner: no app miner command line was captured; phase void"; return }
|
||||
$alive = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' } | Measure-Object).Count
|
||||
if ($alive -gt 0) { "RESULT $label own miner: the app's CUDA worker is still running ($alive); the phase would mine twice on the card, so it is void (job 3, 22:48Z)"; return }
|
||||
$a = $minerArgs
|
||||
$extra = "--race off"
|
||||
if ($batchLog2 -and $batchLog2 -ne 22) { $extra = "$extra --batch-log2 $batchLog2" }
|
||||
if ($a -match '--worker-args "([^"]*)"') { $a = $a -replace '--worker-args "([^"]*)"', ('--worker-args "$1 ' + $extra + '"') }
|
||||
elseif ($a -match '--worker-args (\S+)') { $a = $a -replace '--worker-args (\S+)', ('--worker-args "$1 ' + $extra + '"') }
|
||||
else { $a = $a + ' --worker-args "' + $extra + '"' }
|
||||
$out = Join-Path $job "miner-$label.out"; $err = Join-Path $job "miner-$label.err"
|
||||
$script:miner = Start-Process -FilePath $minerExe -ArgumentList $a -WorkingDirectory $app -RedirectStandardOutput $out -RedirectStandardError $err -NoNewWindow -PassThru
|
||||
"RESULT $label own miner started $(Stamp) pid $($script:miner.Id) batch_log2=$batchLog2 args=$a"
|
||||
$w = 0
|
||||
while ($w -lt 150) {
|
||||
Start-Sleep -Seconds 10; $w += 10
|
||||
$s = Get-Content $out -ErrorAction SilentlyContinue | Where-Object { $_ -match 'STATUS' } | Select-Object -Last 1
|
||||
if ($s -and $s -match 'hash=([\d.]+) MH/s' -and [double]$Matches[1] -gt 10) { "RESULT $label own miner hashing after $w s: $s"; $script:minerOk = $true; return }
|
||||
if ($script:miner.HasExited) { "RESULT $label own miner EXITED (code $($script:miner.ExitCode)); err: $((Get-Content $err -ErrorAction SilentlyContinue | Select-Object -Last 3) -join ' / ')"; return }
|
||||
}
|
||||
"RESULT $label own miner not hashing after $w s; last lines: $((Get-Content $out -ErrorAction SilentlyContinue | Select-Object -Last 3) -join ' / ')"
|
||||
}
|
||||
function StopMiner($label) {
|
||||
if (-not $script:miner) { return }
|
||||
$pid0 = $script:miner.Id
|
||||
Get-CimInstance Win32_Process | Where-Object { $_.ParentProcessId -eq $pid0 } | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }
|
||||
Stop-Process -Id $pid0 -Force -ErrorAction SilentlyContinue
|
||||
Start-Sleep -Seconds 3
|
||||
$left = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker*' -or $_.ProcessName -like 'igneum-miner*' } | Measure-Object).Count
|
||||
"RESULT $label own miner stopped $(Stamp); miner/worker processes left: $left"
|
||||
$script:miner = $null
|
||||
}
|
||||
function MinerRate($label, $skip) {
|
||||
$out = Join-Path $job "miner-$label.out"
|
||||
$h = @(); foreach ($l in (Get-Content $out -ErrorAction SilentlyContinue | Where-Object { $_ -match 'STATUS' })) { if ($l -match 'now=([\d.]+) MH/s wall') { $h += [double]$Matches[1] } }
|
||||
if ($h.Count -gt $skip) { $h = $h[$skip..($h.Count - 1)] }
|
||||
if ($h.Count -eq 0) { return "n=0" }
|
||||
$m = $h | Measure-Object -Average -Minimum -Maximum
|
||||
"n=$($h.Count) mean=$([math]::Round($m.Average,2)) min=$([math]::Round($m.Minimum,2)) max=$([math]::Round($m.Maximum,2)) MH/s wall (the now= field of the STATUS lines every 10 s, the first $skip skipped)"
|
||||
}
|
||||
$paused = $false
|
||||
try {
|
||||
Wsl (Join-Path $job 'prep.sh')
|
||||
if (-not (Test-Path (Join-Path $job 'fixtures.txt'))) { "RESULT prep FAILED: no fixtures"; exit 1 }
|
||||
$list = Fixtures
|
||||
$first2 = ($list -split ',')[0..1] -join ','; $last2 = ($list -split ',')[2..3] -join ','
|
||||
if ($PhaseSet -contains 'A0') {
|
||||
RunPhase 'A0' "SP1_PROVER=cuda RUST_LOG=info `$H --mode chain --chain $(($list -split ',')[0]) --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-A0.json"
|
||||
$errf = Join-Path $job 'host-A0.err'
|
||||
"RESULT A0 profile: $((Get-Content $errf -ErrorAction SilentlyContinue | Measure-Object).Count) stderr lines; the first 160 with a level or a duration follow"
|
||||
Get-Content $errf -ErrorAction SilentlyContinue | Where-Object { $_ -match 'INFO|WARN|DEBUG|ERROR|time\.|elapsed|took|ms\b|\bs\b' } | Select-Object -First 160 | ForEach-Object { "PROFILE $_" }
|
||||
}
|
||||
if ($PhaseSet -contains 'A') { RunPhase 'A' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --save-shards --out `$JOB/results-A.json" }
|
||||
if ($PhaseSet -contains 'B0') { RunPhase 'B0' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B0.json" }
|
||||
if ($PhaseSet -contains 'B') { RunPhase 'B' "SP1_WORKER_VERIFY_INTERMEDIATES=false SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B.json" }
|
||||
$needIdle = @('C','C1','C2','G0','D','E','E0','F','G','H') | Where-Object { $PhaseSet -contains $_ }
|
||||
if ($needIdle) {
|
||||
# the 5090 switched off through /api/cards (and on again in the finally block): /api/pause then /api/resume left
|
||||
# the worker off on 0.3.9 (21:25Z, an app defect in the resume path, fixed for 0.3.11)
|
||||
"RESULT card_off $(Stamp) $(CardSwitch $false $null)"; $paused = $true
|
||||
$w = 0; $left = 1
|
||||
while ($w -lt 120 -and $left -gt 0) { Start-Sleep -Seconds 5; $w += 5; $left = CudaWorkers }
|
||||
if ($left -gt 0) { "RESULT card_off_failed $(Stamp) the app's CUDA worker is still running after $w s ($left); /api/pause as the fallback: $(Post '/api/pause' @{})"; $w2 = 0; while ($w2 -lt 60 -and (CudaWorkers) -gt 0) { Start-Sleep -Seconds 5; $w2 += 5 }; $left = CudaWorkers }
|
||||
"RESULT paused $(Stamp) after $w s; CUDA workers left: $left; $(Smi)"
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
if ($PhaseSet -contains 'C') { RunPhase 'C' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-C.json" }
|
||||
if ($PhaseSet -contains 'C1') { RunPhase 'C1' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-C1.json" }
|
||||
if ($PhaseSet -contains 'C2') { RunPhase 'C2' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain `$JOB/block-344-shards4.json --prover 0xCAfc6e74000000000000000000000000000000c2 --save-shards --out `$JOB/results-C2.json" }
|
||||
if ($PhaseSet -contains 'G0') {
|
||||
$c1 = Cmd 'G01' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $first2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G01.json"
|
||||
$c2 = Cmd 'G02' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $last2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G02.json"
|
||||
$p1 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G01 $c1" -RedirectStandardOutput (Join-Path $job 'G01.out') -NoNewWindow -PassThru
|
||||
Start-Sleep -Seconds 2
|
||||
$p2 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G02 $c2" -RedirectStandardOutput (Join-Path $job 'G02.out') -NoNewWindow -PassThru
|
||||
$p1.WaitForExit(); $p2.WaitForExit()
|
||||
foreach ($f in 'G01.out', 'G02.out') { ((Get-Content (Join-Path $job $f) -Raw -ErrorAction SilentlyContinue) -replace "`0", '') -split "`n" | ForEach-Object { $_.TrimEnd("`r") } | Where-Object { $_ } }
|
||||
}
|
||||
if ($PhaseSet -contains 'D') {
|
||||
StartMiner 'D' 22; if ($script:minerOk) { RunPhase 'D' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-D.json"; "RESULT D miner rate $(MinerRate 'D' 2)" }
|
||||
StopMiner 'D'
|
||||
}
|
||||
if ($PhaseSet -contains 'E') {
|
||||
foreach ($b in 20, 18, 16) {
|
||||
$lab = "E$b"
|
||||
StartMiner $lab $b; if ($script:minerOk) { RunPhase $lab "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-$lab.json"; "RESULT $lab miner rate $(MinerRate $lab 2)" }
|
||||
StopMiner $lab
|
||||
}
|
||||
}
|
||||
if ($PhaseSet -contains 'E0') {
|
||||
$g = (0..3 | ForEach-Object { "$srcW/block-344-shard-$_-compressed.bin" }) -join ','
|
||||
StartMiner 'E0' 22; if ($script:minerOk) { RunPhase 'E0' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$g' --parent $(Parent 344) --out `$JOB/results-E0.json"; "RESULT E0 miner rate $(MinerRate 'E0' 2)" }
|
||||
StopMiner 'E0'
|
||||
}
|
||||
if ($PhaseSet -contains 'F') {
|
||||
$pol = (& nvidia-smi compute-policy --set-timeslice=1 2>&1) -join ' '
|
||||
"RESULT F timeslice set: $pol"
|
||||
if ($pol -notmatch 'rror|nsufficient|not supported|Unknown') {
|
||||
StartMiner 'F' 22; if ($script:minerOk) { RunPhase 'F' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-F.json"; "RESULT F miner rate $(MinerRate 'F' 2)" }
|
||||
StopMiner 'F'
|
||||
}
|
||||
"RESULT F timeslice restored: $((& nvidia-smi compute-policy --set-timeslice=0 2>&1) -join ' ')"
|
||||
}
|
||||
if ($PhaseSet -contains 'G') {
|
||||
StartMiner 'G' 22; if ($script:minerOk) {
|
||||
$c1 = Cmd 'G1' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $first2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G1.json"
|
||||
$c2 = Cmd 'G2' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $last2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G2.json"
|
||||
$p1 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G1 $c1" -RedirectStandardOutput (Join-Path $job 'G1.out') -NoNewWindow -PassThru
|
||||
Start-Sleep -Seconds 2
|
||||
$p2 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G2 $c2" -RedirectStandardOutput (Join-Path $job 'G2.out') -NoNewWindow -PassThru
|
||||
$p1.WaitForExit(); $p2.WaitForExit()
|
||||
foreach ($f in 'G1.out', 'G2.out') { ((Get-Content (Join-Path $job $f) -Raw -ErrorAction SilentlyContinue) -replace "`0", '') -split "`n" | ForEach-Object { $_.TrimEnd("`r") } | Where-Object { $_ } }
|
||||
"RESULT G miner rate $(MinerRate 'G' 2)"
|
||||
}
|
||||
StopMiner 'G'
|
||||
}
|
||||
if ($PhaseSet -contains 'H') {
|
||||
# the combination from this job's own phases: the batch-log2 with the shortest chain among D and E (the miner's
|
||||
# rate alongside), the knob when phase B beat B0 by 5% or more; skipped when the job is past 15 min (the window)
|
||||
try {
|
||||
$cands = @(); foreach ($lab in 'D', 'E20', 'E18', 'E16') { $f = Join-Path $job "results-$lab.json"; if (Test-Path $f) { $r = (Get-Content $f -Raw | ConvertFrom-Json); $cands += [pscustomobject]@{ lab = $lab; batch = $(if ($lab -eq 'D') { 22 } else { [int]$lab.Substring(1) }); secs = [double]$r.chain_seconds } } }
|
||||
if ($cands.Count -gt 0) { $bestC = $cands | Sort-Object secs | Select-Object -First 1; $BestBatch = $bestC.batch; "RESULT H choice: $(($cands | ForEach-Object { "$($_.lab)=$([math]::Round($_.secs,1))s" }) -join ' ') -> batch_log2 $BestBatch" }
|
||||
$b0 = Join-Path $job 'results-B0.json'; $b1 = Join-Path $job 'results-B.json'
|
||||
if ((Test-Path $b0) -and (Test-Path $b1)) { $s0 = [double](Get-Content $b0 -Raw | ConvertFrom-Json).aggregate_seconds; $s1 = [double](Get-Content $b1 -Raw | ConvertFrom-Json).aggregate_seconds; if ($s1 -lt 0.95 * $s0) { $BestKnobs = 'SP1_WORKER_VERIFY_INTERMEDIATES=false' }; "RESULT H knobs: B0 $([math]::Round($s0,1)) s, B $([math]::Round($s1,1)) s -> '$BestKnobs'" }
|
||||
} catch { "RESULT H choice error: $_" }
|
||||
$elapsedMin = ((Get-Date) - $jobStart).TotalMinutes
|
||||
if ($elapsedMin -gt 15) { "RESULT H skipped: $([math]::Round($elapsedMin,1)) min into the job (the 20-min window)" }
|
||||
else { StartMiner 'H' $BestBatch }
|
||||
if ($elapsedMin -le 15 -and $script:minerOk) { RunPhase 'H' "$BestKnobs SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-H.json"; "RESULT H miner rate $(MinerRate 'H' 2)" }
|
||||
StopMiner 'H'
|
||||
}
|
||||
} finally {
|
||||
StopMiner 'final'
|
||||
& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
||||
if ($paused) {
|
||||
"RESULT card_on $(Stamp) $(CardSwitch $true $null)"; "RESULT resume $(Stamp) $(Post '/api/resume' @{})"
|
||||
$w3 = 0; while ($w3 -lt 90 -and (CudaWorkers) -eq 0) { Start-Sleep -Seconds 5; $w3 += 5 }
|
||||
if ((CudaWorkers) -eq 0) { "RESULT miner_back_failed $(Stamp) no CUDA worker after $w3 s; /api/start: $(Post '/api/start' @{})" } else { "RESULT miner_back $(Stamp) CUDA worker running after $w3 s" }
|
||||
}
|
||||
"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})"
|
||||
"RESULT end $(Stamp) $(Smi)"
|
||||
}
|
||||
342
tools/proving-v1/pc2-segments.ps1
Normal file
342
tools/proving-v1/pc2-segments.ps1
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
# Proving v1, the segment-aligned prover on PC 2 (6 October 2026, the project lead: "proving v1 is active but has proven zero
|
||||
# segments, and cannot with one prover; find a way to solve this"). A signed `run` job (shell powershell, not
|
||||
# elevated; the app keeps mining; the app's own prover is switched OFF for the run through /api/prove and ON again
|
||||
# at the end; the live /opt/igneum host is untouched: this build lands in /opt/igneum-segal).
|
||||
#
|
||||
# What it does, for RUN_MINUTES, exactly what the app's segment path (app/igneum-app/src/prover.rs, pick_segment and
|
||||
# prove_segment) does:
|
||||
# 1. the node's v1 status and the work list (igneum_getAssignedShards, lookback 600) grouped into whole untouched
|
||||
# segments (every block present, every shard open, unpaid, not in our pool); the newest one inside its deadline by
|
||||
# the margin (240 DAA, or 1.5x the last segment's time) is claimed; the app ranks by FNV of (first, key) for the
|
||||
# multi-prover spread, which with one prover changes nothing but the order
|
||||
# 2. the segment statement: executed, pending; fresh when the previous segment is not paid and no verified record of
|
||||
# it waits in the pool, else chained to the previous proof (--prev) when the pool holds it
|
||||
# 3. one export (igneum_exportSegments 0..last), one fixture per block, one host run (--mode chain --save-shards
|
||||
# [--prev]) with a 1-s nvidia-smi sampler underneath
|
||||
# 4. every shard record signed (igneum-miner.exe sign-record) and submitted (igneum_submitProofRecord), then the
|
||||
# segment record (sign-segment-record, igneum_submitSegmentRecord)
|
||||
# 5. the paid state of every submitted segment polled each pass (igneum_getSegmentRecords: carried, paid)
|
||||
# Every number is a RESULT line. The miner's hash rate comes from the app log's "status: ... MH/s" lines (every 30 s),
|
||||
# the 30 minutes before the job against the run. Card keys, when needed, come from settings.json, never /api/state
|
||||
# (that reply is "{}" on 0.3.11 once paid_wei passes u64::MAX; fixed in 6714a45).
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$RunMinutes = 30
|
||||
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
||||
$app = if ($env:IGNEUM_APP_DIR) { $env:IGNEUM_APP_DIR } else { Join-Path $env:LOCALAPPDATA 'igneum\app' }
|
||||
$urlFile = Join-Path $app 'app.url'
|
||||
$base = if (Test-Path $urlFile) { (Get-Content $urlFile -Raw).Trim().TrimEnd('/') } else { '' }
|
||||
function Post($path, $obj) { try { (Invoke-RestMethod -Method Post -Uri "$base$path" -ContentType 'application/json' -Body ($obj | ConvertTo-Json -Compress) -TimeoutSec 10) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
||||
function Rpc($method, $params) {
|
||||
try { (Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$evmPort" -ContentType 'application/json' -Body (@{jsonrpc='2.0'; id=1; method=$method; params=$params} | ConvertTo-Json -Compress -Depth 8) -TimeoutSec 120).result } catch { $script:rpcErr = "$_"; $null }
|
||||
}
|
||||
function RpcRaw($method, $params) {
|
||||
try { Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$evmPort" -ContentType 'application/json' -Body (@{jsonrpc='2.0'; id=1; method=$method; params=$params} | ConvertTo-Json -Compress -Depth 8) -TimeoutSec 120 } catch { $script:rpcErr = "$_"; $null }
|
||||
}
|
||||
function Hex($v) { if ($null -eq $v) { return 0 }; if ($v -is [string] -and $v.StartsWith('0x')) { return [Convert]::ToInt64($v.Substring(2), 16) }; return [int64]$v }
|
||||
$job = $env:IGNEUM_JOB_DIR
|
||||
if (-not $job) { $job = Join-Path $env:TEMP 'igneum-segments' }
|
||||
New-Item -ItemType Directory -Force -Path $job | Out-Null
|
||||
$jobStart = Get-Date
|
||||
"RESULT start $(Stamp) app_dir=$app base_len=$($base.Length) job=$job run_minutes=$RunMinutes"
|
||||
$evmPort = $null
|
||||
foreach ($p in 26790, 26800, 26810) { $evmPort = $p; if (Rpc 'igneum_getProvingStatus' @()) { break }; $evmPort = $null }
|
||||
if (-not $evmPort) { "RESULT refused $(Stamp) no node EVM RPC on 26790/26800/26810: $script:rpcErr"; exit 1 }
|
||||
# the app's miner (the signer) and its label, the payout address
|
||||
$mp = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' } | Select-Object -First 1
|
||||
$minerExe = if ($mp) { $mp.ExecutablePath } else { $null }
|
||||
if (-not $minerExe -or -not (Test-Path $minerExe)) {
|
||||
$cand = Get-ChildItem -Path (Split-Path $app -Parent) -Recurse -Filter 'igneum-miner.exe' -ErrorAction SilentlyContinue | Select-Object -First 1
|
||||
if ($cand) { $minerExe = $cand.FullName }
|
||||
}
|
||||
if (-not $minerExe) { "RESULT refused $(Stamp) no igneum-miner.exe found (the signer)"; exit 1 }
|
||||
$settingsRaw = Get-Content (Join-Path $app 'settings.json') -Raw
|
||||
$payout = [regex]::Match($settingsRaw, '"address":\s*"(0x[0-9a-fA-F]{40})"').Groups[1].Value
|
||||
if ($payout.Length -ne 42) { "RESULT refused $(Stamp) no payout address in settings.json"; exit 1 }
|
||||
$identities = 1; $im = [regex]::Match($settingsRaw, '"nvidia:[^"]*":\s*\{[^}]*"identities":\s*(\d+)'); if ($im.Success) { $identities = [int]$im.Groups[1].Value }
|
||||
$machineId = ''; $mm = [regex]::Match($settingsRaw, '"machine_id":\s*"([0-9a-f]+)"'); if ($mm.Success) { $machineId = $mm.Groups[1].Value }
|
||||
if (-not $machineId) { $mf = Join-Path $app 'machine-id'; if (Test-Path $mf) { $machineId = (Get-Content $mf -Raw).Trim() } }
|
||||
if (-not $machineId -and $job -match '([0-9a-f]{8})') { $machineId = $Matches[1] }
|
||||
$id8 = if ($machineId.Length -ge 8) { $machineId.Substring(0, 8) } else { '1ccfe586' }
|
||||
$label = if ($identities -gt 1) { "win-$id8-1-1" } else { "win-$id8-1" }
|
||||
$chain = 'igneum-devnet' # the app's chain_name with no devnet suffix (IGNEUM_APP_DEVNET_SUFFIX unset on the fleet)
|
||||
$keyHash = (& $minerExe key-hash $label 2>$null | Select-Object -Last 1).Trim()
|
||||
if ($keyHash -and -not $keyHash.StartsWith('0x')) { $keyHash = "0x$keyHash" }
|
||||
"RESULT signer $(Stamp) miner=$minerExe label=$label key=$keyHash chain=$chain payout=$payout identities=$identities"
|
||||
if (-not $keyHash -or $keyHash.Length -lt 64) { "RESULT refused $(Stamp) key-hash gave nothing"; exit 1 }
|
||||
# the live prover off for the run: its sp1-gpu-server would be shared with ours (same socket, its environment)
|
||||
"RESULT prove_off $(Stamp) $(Post '/api/prove' @{on=$false})"
|
||||
Start-Sleep -Seconds 5
|
||||
# --- WSL side: the package built into /opt/igneum-segal (root, the warm target of the live build), the segment runner
|
||||
$pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-segal\igneum-prove-wsl2'
|
||||
if (-not (Test-Path $pkg)) { $pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-segal' }
|
||||
"RESULT package $(Stamp) $pkg exists=$(Test-Path (Join-Path $pkg 'package'))"
|
||||
function WslPath($p) { $w = (& wsl.exe -d Ubuntu-24.04 -u root -- wslpath -a ($p -replace '\\', '/') 2>$null); if ($w) { ($w -replace "`0", '').Trim() } else { '/mnt/c' + ($p.Substring(2) -replace '\\', '/') } }
|
||||
$pkgW = WslPath $pkg; $jobW = WslPath $job
|
||||
$prep = @'
|
||||
set -uo pipefail
|
||||
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
||||
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}"
|
||||
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
PKG="$1"
|
||||
DEST="$HOME/igneum-prove-segal"; LIVE_TARGET="$HOME/igneum-prove/proving/igneum-prove/target"
|
||||
mkdir -p "$DEST"
|
||||
rsync -a --delete --exclude target "$PKG/package/" "$DEST/"
|
||||
find "$DEST" -name target -prune -o -type f -exec touch {} + 2>/dev/null
|
||||
grep -o '"program_id": "0x[0-9a-f]*"' "$DEST/proving/igneum-prove/elf/manifest.json" | sed 's/^/RESULT manifest /'
|
||||
cd "$DEST/proving/igneum-prove"
|
||||
t0=$(date +%s)
|
||||
if ! CARGO_TARGET_DIR="$LIVE_TARGET" cargo build --release -p igneum-prove-export -p igneum-prove-host --features igneum-prove-host/cuda 2>&1 | tail -3; then echo "RESULT build FAILED"; exit 1; fi
|
||||
echo "RESULT build $(stamp) exit 0 in $(( $(date +%s) - t0 )) s"
|
||||
mkdir -p /opt/igneum-segal && cp "$LIVE_TARGET/release/igneum-prove-host" "$LIVE_TARGET/release/igneum-prove-export" /opt/igneum-segal/
|
||||
H=/opt/igneum-segal/igneum-prove-host; X=/opt/igneum-segal/igneum-prove-export
|
||||
echo "RESULT installed $(sha256sum $H | cut -c1-16) host, $(sha256sum $X | cut -c1-16) export; live /opt/igneum untouched: $(sha256sum /opt/igneum/igneum-prove-host | cut -c1-16)"
|
||||
$H --mode id | sed 's/^/RESULT segal-host /'
|
||||
echo "RESULT gpu_server_before $(stamp) running=$(pgrep -x sp1-gpu-server | wc -l) socket=$(ls -la /tmp/sp1-cuda-0.sock 2>/dev/null || echo none)"
|
||||
# the root-socket rule (tools/ci/prover-socket-check.sh): this run is root; the app's prover is off, so its server goes too
|
||||
pkill -f sp1-gpu-server; sleep 1; rm -f /tmp/sp1-cuda-*.sock
|
||||
echo "RESULT socket_start $(stamp) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"
|
||||
'@
|
||||
# bash seg.sh <job dir> <first> <last> <payout> [prev file]: unwrap the export, cut the blocks, run the chain with the sampler
|
||||
$seg = @'
|
||||
set -uo pipefail
|
||||
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
||||
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}"
|
||||
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
JOB="$1"; FIRST="$2"; LAST="$3"; PAYOUT="$4"; PREV="${5:-}"
|
||||
H=/opt/igneum-segal/igneum-prove-host; X=/opt/igneum-segal/igneum-prove-export
|
||||
D="$JOB/seg-$FIRST"; mkdir -p "$D"
|
||||
t0=$(date +%s.%N)
|
||||
python3 -c "import json; d=json.load(open('$D/seq.json')); json.dump(d['result'], open('$D/export.json','w'))" || { echo "RESULT seg $FIRST unwrap FAILED"; exit 1; }
|
||||
LIST=""
|
||||
for n in $(seq $FIRST $LAST); do
|
||||
if ! $X "$D/export.json" $n "$D/block-$n.json" --source "PC 2 live devnet, segment-aligned prover" >"$D/export-$n.log" 2>&1; then echo "RESULT seg $FIRST cut $n FAILED: $(tail -1 "$D/export-$n.log")"; exit 1; fi
|
||||
LIST="$LIST${LIST:+,}$D/block-$n.json"
|
||||
done
|
||||
CUT=$(python3 -c "import time; print(round(time.time() - $t0, 1))")
|
||||
echo "RESULT seg $FIRST cut $(stamp) $LAST blocks in $CUT s"
|
||||
nvidia-smi --query-gpu=timestamp,index,memory.used,utilization.gpu,power.draw --format=csv,noheader,nounits -l 1 > "$D/smi.csv" 2>/dev/null &
|
||||
SMI=$!
|
||||
t1=$(date +%s.%N)
|
||||
PREVARG=""; [ -n "$PREV" ] && PREVARG="--prev $PREV"
|
||||
SP1_PROVER=cuda RUST_LOG=off $H --mode chain --chain "$LIST" --prover "$PAYOUT" --save-shards $PREVARG --out "$D/chain-results.json" >"$D/chain.log" 2>"$D/chain.err"
|
||||
RC=$?
|
||||
WALL=$(python3 -c "import time; print(round(time.time() - $t1, 1))")
|
||||
kill $SMI 2>/dev/null; sleep 1
|
||||
grep -E "^RESULT (chain block|chain:|chain prev|setup)" "$D/chain.log" | sed "s/^/seg $FIRST: /"
|
||||
awk -F', *' '{ if ($3+0 > max) max=$3+0; u+=$4; n++ } END { if (n) printf "RESULT seg FIRST gpu samples=%d memory_used_max_mib=%d util_mean_pct=%.1f\n", n, max, u/n }' "$D/smi.csv" | sed "s/FIRST/$FIRST/"
|
||||
echo "RESULT seg $FIRST chain $(stamp) exit $RC wall $WALL s"
|
||||
if [ $RC -ne 0 ]; then tail -4 "$D/chain.err" | sed "s/^/seg $FIRST stderr: /"; exit 1; fi
|
||||
# the submit bodies: one JSON-RPC request per shard record and one for the segment, the proof bytes as hex
|
||||
python3 - "$D" <<'PY'
|
||||
import json, sys, os
|
||||
d = sys.argv[1]
|
||||
r = json.load(open(os.path.join(d, 'chain-results.json')))
|
||||
recs = []
|
||||
for b in r['blocks']:
|
||||
for s in b.get('shard_records', []):
|
||||
recs.append(s)
|
||||
json.dump(recs, open(os.path.join(d, 'shard-records.json'), 'w'))
|
||||
print('RESULT seg %s records %d shard records, segment chain_len %s, proof %s bytes' % (r['first'], len(recs), r['segment_chain_len'], r['segment_proof_bytes']))
|
||||
PY
|
||||
'@
|
||||
# bash body.sh <proof file> <record hex> <method> <out file>: the JSON-RPC body with the proof as hex
|
||||
$body = @'
|
||||
set -u
|
||||
python3 - "$1" "$2" "$3" "$4" <<'PY'
|
||||
import json, sys, binascii
|
||||
proof = '0x' + binascii.hexlify(open(sys.argv[1], 'rb').read()).decode()
|
||||
json.dump({'jsonrpc': '2.0', 'id': 1, 'method': sys.argv[3], 'params': [{'record': sys.argv[2], 'proof': proof}]}, open(sys.argv[4], 'w'))
|
||||
PY
|
||||
'@
|
||||
# bash unhex.sh <hex file> <out file>: a hex string (0x...) to bytes
|
||||
$unhex = @'
|
||||
set -u
|
||||
python3 -c "import sys,binascii; h=open(sys.argv[1]).read().strip(); h=h[2:] if h.startswith('0x') else h; open(sys.argv[2],'wb').write(binascii.unhexlify(h))" "$1" "$2"
|
||||
'@
|
||||
foreach ($pair in @(@('prep.sh', $prep), @('seg.sh', $seg), @('body.sh', $body), @('unhex.sh', $unhex))) {
|
||||
[IO.File]::WriteAllText((Join-Path $job $pair[0]), ($pair[1] -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
||||
}
|
||||
function Wsl { param([string[]]$a) & wsl.exe -d Ubuntu-24.04 -u root -- bash @a 2>&1 | ForEach-Object { ($_ -replace "`0", '') } }
|
||||
Wsl @("$jobW/prep.sh", $pkgW)
|
||||
if (-not (Test-Path (Join-Path $job 'prep.sh'))) { "RESULT refused $(Stamp) prep.sh missing"; exit 1 }
|
||||
# --- the app log's miner rate before the run (baseline): "status: ... <x> MH/s" every 30 s
|
||||
$logDir = Join-Path $app 'logs'
|
||||
function RateLines($since, $until) {
|
||||
$out = @()
|
||||
$files = @(Get-ChildItem -Path $logDir, (Join-Path (Split-Path $app -Parent) 'logs'), $app -Filter '*.log' -ErrorAction SilentlyContinue | Where-Object { $_.LastWriteTime -gt (Get-Date).AddHours(-3) } | Sort-Object LastWriteTime -Descending | Select-Object -First 4)
|
||||
if ($files.Count -eq 0) { "RESULT rate_files none under $logDir, $(Join-Path (Split-Path $app -Parent) 'logs'), $app" }
|
||||
foreach ($f in $files) {
|
||||
foreach ($l in (Get-Content $f.FullName -ErrorAction SilentlyContinue | Where-Object { $_ -match '^(\d{10}) status: .* ([\d.]+) MH/s' })) {
|
||||
$t = [int64]$Matches[1]; if ($t -ge $since -and $t -le $until) { $out += [double]$Matches[2] }
|
||||
}
|
||||
}
|
||||
return $out
|
||||
}
|
||||
function Epoch($d) { [int64](($d.ToUniversalTime()) - (Get-Date '1970-01-01')).TotalSeconds }
|
||||
$t0 = Epoch $jobStart
|
||||
$before = RateLines ($t0 - 1800) $t0
|
||||
if ($before.Count) { $m = $before | Measure-Object -Average -Minimum -Maximum; "RESULT rate_before $(Stamp) n=$($before.Count) mean=$([math]::Round($m.Average,2)) min=$($m.Minimum) max=$($m.Maximum) MH/s (the app log's status lines, the 30 min before the job)" } else { "RESULT rate_before $(Stamp) no status lines found in $logDir" }
|
||||
# --- the loop
|
||||
$submittedSegments = @{} # first -> @{last; aggWei; at}
|
||||
$attempted = @{}
|
||||
$lastSegSecs = 0
|
||||
$n = 1; $start = 0; $unproven = 600
|
||||
$passes = 0; $claimed = 0; $submitted = 0; $shardsAccepted = 0; $shardsRefused = 0
|
||||
$runStart = Get-Date
|
||||
function Candidates {
|
||||
$st = Rpc 'igneum_getProvingStatus' @()
|
||||
if (-not $st -or -not $st.v1 -or -not $st.v1.active -or -not $st.v1.start) { return @() }
|
||||
$script:start = Hex $st.v1.start; $script:n = [math]::Max(1, (Hex $st.v1.segmentBlocks)); $script:unproven = Hex $st.v1.unprovenDaa
|
||||
$tipDaa = Hex $st.tipDaa
|
||||
$work = Rpc 'igneum_getAssignedShards' @(@($keyHash), 600)
|
||||
if (-not $work) { return @() }
|
||||
$by = @{}
|
||||
foreach ($w in $work) { [int64]$num = Hex $w.number; if ($num -lt $script:start) { continue }; if (-not $by.ContainsKey($num)) { $by[$num] = @() }; $by[$num] += $w }
|
||||
"RESULT worklist $(Stamp) entries=$(@($work).Count) blocks=$($by.Count) start=$($script:start) n=$($script:n) tipDaa=$tipDaa"
|
||||
$need = [math]::Max(240, [math]::Ceiling($lastSegSecs * 1.5))
|
||||
$segs = @()
|
||||
$seen = @{}
|
||||
foreach ($num in ($by.Keys | Sort-Object)) {
|
||||
# [int64] throughout: [math]::Floor gives a double, and a double never matches an int64 hashtable key (job b,
|
||||
# 07:20Z to 07:50Z: every segment "not whole", 88 passes, nothing claimed)
|
||||
[int64]$k = [math]::Floor(($num - $script:start) / $script:n); [int64]$first = $script:start + $k * $script:n; [int64]$last = $first + $script:n - 1
|
||||
if ($seen.ContainsKey($first)) { continue }; $seen[$first] = $true
|
||||
if ($attempted.ContainsKey($first)) { continue }
|
||||
$whole = $true; $shards = @(); $lastDaa = 0; $wei = 0
|
||||
for ([int64]$b = $first; $b -le $last; $b++) {
|
||||
if (-not $by.ContainsKey($b)) { $whole = $false; break }
|
||||
$es = $by[$b] | Sort-Object { $_.shard } -Unique
|
||||
foreach ($e in $es) {
|
||||
$inPool = ($e.pool -and (@($e.pool)).Count -gt 0)
|
||||
if (-not $e.open -or $null -ne $e.paid -or $inPool) { $whole = $false; break }
|
||||
$shards += @{ number = $b; hash = $e.hash; shard = [int]$e.shard }
|
||||
if ($b -eq $last) { $lastDaa = Hex $e.daaScore }
|
||||
}
|
||||
if (-not $whole) { break }
|
||||
}
|
||||
if ($whole -and $shards.Count -gt 0) {
|
||||
$deadline = $lastDaa + $script:unproven
|
||||
if ($deadline -ge ($tipDaa + 1 + $need)) { $segs += @{ first = $first; last = $last; lastDaa = $lastDaa; deadline = $deadline; shards = $shards; margin = [int64]($deadline - $tipDaa - 1) } }
|
||||
}
|
||||
}
|
||||
# newest first (the most time before the deadline); the app ranks by FNV(first, key) for the spread
|
||||
return @($segs | Sort-Object { $_.first } -Descending)
|
||||
}
|
||||
while (((Get-Date) - $runStart).TotalMinutes -lt $RunMinutes) {
|
||||
$passes++
|
||||
# paid segments
|
||||
foreach ($f in @($submittedSegments.Keys)) {
|
||||
$rec = Rpc 'igneum_getSegmentRecords' @(('0x{0:x}' -f $f))
|
||||
if ($rec) {
|
||||
$carried = (@($rec.carried)).Count
|
||||
if ($rec.paid) { "RESULT paid $(Stamp) segment $f..$($submittedSegments[$f].last) paid wei=$($rec.paid.wei) ($([math]::Round((Hex $rec.paid.wei) / 1e18, 4)) IGN) carrier=$(Hex $rec.paid.carrierNumber) payout=$($rec.paid.payout) after $([math]::Round(((Get-Date) - $submittedSegments[$f].at).TotalSeconds)) s"; $submittedSegments.Remove($f) }
|
||||
elseif ($carried -gt 0) { "RESULT carried $(Stamp) segment $f carried=$carried valid=$(@($rec.carried | ForEach-Object { $_.valid }) -join ',') rejected=$(@($rec.carried | ForEach-Object { $_.rejected }) -join ';')" }
|
||||
}
|
||||
}
|
||||
$cands = Candidates
|
||||
if (-not $cands -or $cands.Count -eq 0) { "RESULT pass $passes $(Stamp) no whole segment inside the margin (need=$([math]::Max(240, [math]::Ceiling($lastSegSecs * 1.5))) DAA); waiting 20 s"; Start-Sleep -Seconds 20; continue }
|
||||
$picked = $null; $prevFile = $null; $expected = ''
|
||||
$tries = 0
|
||||
foreach ($c in $cands) {
|
||||
if ($tries -ge 3) { break }; $tries++
|
||||
$stmt = Rpc 'igneum_getSegmentStatement' @(('0x{0:x}' -f $c.first))
|
||||
if (-not $stmt -or -not $stmt.executed -or $stmt.status.status -ne 'pending') { $attempted[$c.first] = $true; "RESULT skip $(Stamp) segment $($c.first): executed=$($stmt.executed) status=$($stmt.status.status)"; continue }
|
||||
if ($null -eq $stmt.previous) {
|
||||
if ($c.first -ge ($start + $n)) {
|
||||
$pr = Rpc 'igneum_getSegmentRecords' @(('0x{0:x}' -f ($c.first - $n)))
|
||||
$waiting = $false
|
||||
if ($pr -and $pr.pool) { foreach ($e in @($pr.pool)) { if ($e.verified -eq $true -and $null -eq $e.includedIn) { $waiting = $true } } }
|
||||
if ($waiting -or ($pr -and $pr.paid)) { "RESULT skip $(Stamp) segment $($c.first): the previous segment has a record waiting or paid; a fresh chain would be refused"; continue }
|
||||
}
|
||||
$picked = $c; $expected = $stmt.publicValuesFresh; break
|
||||
}
|
||||
if ($stmt.previous.proofInPool -ne $true) { "RESULT skip $(Stamp) segment $($c.first): previous paid, its proof not in this pool"; continue }
|
||||
$got = Rpc 'igneum_getSegmentProofBytes' @($stmt.previous.first, $stmt.previous.keyHash)
|
||||
if (-not $got -or -not $got.proof) { continue }
|
||||
$hexFile = Join-Path $job "prev-$($c.first).hex"; $binFile = Join-Path $job "prev-$($c.first).bin"
|
||||
[IO.File]::WriteAllText($hexFile, $got.proof)
|
||||
Wsl @("$jobW/unhex.sh", (WslPath $hexFile), (WslPath $binFile))
|
||||
if (-not (Test-Path $binFile)) { continue }
|
||||
$picked = $c; $prevFile = WslPath $binFile; $expected = $stmt.publicValuesContinuing; break
|
||||
}
|
||||
if (-not $picked) { "RESULT pass $passes $(Stamp) $($cands.Count) candidates, none usable this pass; waiting 20 s"; Start-Sleep -Seconds 20; continue }
|
||||
$first = $picked.first; $last = $picked.last
|
||||
$attempted[$first] = $true; $claimed++
|
||||
$tipNow = Hex (Rpc 'eth_blockNumber' @())
|
||||
"RESULT claim $(Stamp) segment $first..$last ($($picked.shards.Count) shards, $(if ($prevFile) { 'continuing' } else { 'fresh' })), margin=$($picked.margin) DAA, tip=$tipNow, candidates=$($cands.Count)"
|
||||
$segStart = Get-Date
|
||||
$d = Join-Path $job "seg-$first"; New-Item -ItemType Directory -Force -Path $d | Out-Null
|
||||
# 3. the export (curl.exe streams the reply to a file; Invoke-WebRequest's Content is a string there)
|
||||
$t = Get-Date
|
||||
$bodyTxt = (@{jsonrpc='2.0'; id=1; method='igneum_exportSegments'; params=@('0x0', ('0x{0:x}' -f $last))} | ConvertTo-Json -Compress)
|
||||
$bodyFile = Join-Path $d 'export-request.json'; $seqFile = Join-Path $d 'seq.json'
|
||||
[IO.File]::WriteAllText($bodyFile, $bodyTxt, (New-Object System.Text.UTF8Encoding $false))
|
||||
& curl.exe -s -S -m 600 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bodyFile" -o $seqFile 2>&1 | ForEach-Object { "curl: $_" }
|
||||
if (-not (Test-Path $seqFile) -or (Get-Item $seqFile).Length -lt 1000) { "RESULT seg $first export FAILED"; continue }
|
||||
"RESULT seg $first export $(Stamp) $((Get-Item $seqFile).Length) bytes in $([math]::Round(((Get-Date) - $t).TotalSeconds,1)) s"
|
||||
$segArgs = @("$jobW/seg.sh", $jobW, "$first", "$last", $payout); if ($prevFile) { $segArgs += $prevFile }
|
||||
Wsl $segArgs
|
||||
Remove-Item $seqFile -ErrorAction SilentlyContinue
|
||||
$resFile = Join-Path $d 'chain-results.json'
|
||||
if (-not (Test-Path $resFile)) { "RESULT seg $first FAILED: no chain results"; continue }
|
||||
$res = Get-Content $resFile -Raw | ConvertFrom-Json
|
||||
$recs = Get-Content (Join-Path $d 'shard-records.json') -Raw | ConvertFrom-Json
|
||||
# 4. the shard records
|
||||
$okShards = 0
|
||||
foreach ($r in $recs) {
|
||||
$sg = (& $minerExe sign-record $label $chain $r.block_hash "$($r.number)" "$($r.shard)" $payout $r.statement $r.proof_sha256 2>&1 | Select-Object -Last 1)
|
||||
$signed = $null; try { $signed = $sg | ConvertFrom-Json } catch {}
|
||||
if (-not $signed -or -not $signed.record) { "RESULT seg $first shard $($r.number)/$($r.shard) sign FAILED: $sg"; $shardsRefused++; continue }
|
||||
$proofWin = $r.proof_file -replace '^/mnt/c/', 'C:/' -replace '/', '\'
|
||||
$bf = Join-Path $d "body-$($r.number)-$($r.shard).json"
|
||||
Wsl @("$jobW/body.sh", $r.proof_file, $signed.record, 'igneum_submitProofRecord', (WslPath $bf))
|
||||
$reply = (& curl.exe -s -S -m 120 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bf" 2>&1)
|
||||
$rr = $null; try { $rr = ($reply | ConvertFrom-Json).result } catch {}
|
||||
if ($rr -and $rr.accepted) { $okShards++; $shardsAccepted++ } else { $shardsRefused++; "RESULT seg $first shard $($r.number)/$($r.shard) refused: $(if ($rr) { $rr.reason } else { "$reply".Substring(0, [math]::Min(300, "$reply".Length)) })" }
|
||||
Remove-Item $bf -ErrorAction SilentlyContinue
|
||||
}
|
||||
"RESULT seg $first shards $(Stamp) accepted $okShards of $($recs.Count) (prove $([math]::Round($res.shard_prove_seconds_total,1)) s, aggregation $([math]::Round($res.aggregate_prove_seconds_total,1)) s, chain $([math]::Round($res.chain_seconds,1)) s)"
|
||||
if ($okShards -ne $recs.Count) { "RESULT seg $first FAILED: not every shard record accepted; the segment record is not submitted"; continue }
|
||||
# 5. the segment record: the statement against the node's (every field but provers, bytes 236..268 of the 340)
|
||||
$pv = $res.segment_public_values
|
||||
function Strip($h) { $h = $h -replace '^0x', ''; if ($h.Length -eq 680) { $h.Substring(0, 472) + $h.Substring(536) } else { $h } }
|
||||
if ((Strip $pv) -ne (Strip $expected)) { "RESULT seg $first FAILED: statement differs from the node's native one; ours $($pv.Substring(0,66)) node $($expected.Substring(0, [math]::Min(66, $expected.Length)))"; continue }
|
||||
$lastHash = ($picked.shards | Where-Object { $_.number -eq $last } | Select-Object -First 1).hash
|
||||
$sg = (& $minerExe sign-segment-record $label $chain "$first" "$last" $lastHash $payout $pv $res.segment_proof_sha256 2>&1 | Select-Object -Last 1)
|
||||
$signed = $null; try { $signed = $sg | ConvertFrom-Json } catch {}
|
||||
if (-not $signed -or -not $signed.record) { "RESULT seg $first segment sign FAILED: $sg"; continue }
|
||||
$bf = Join-Path $d 'body-segment.json'
|
||||
Wsl @("$jobW/body.sh", $res.segment_proof_file, $signed.record, 'igneum_submitSegmentRecord', (WslPath $bf))
|
||||
$reply = (& curl.exe -s -S -m 120 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bf" 2>&1)
|
||||
$rr = $null; try { $rr = ($reply | ConvertFrom-Json).result } catch {}
|
||||
Remove-Item $bf -ErrorAction SilentlyContinue
|
||||
$segSecs = [math]::Round(((Get-Date) - $segStart).TotalSeconds, 1)
|
||||
if ($rr -and $rr.accepted) {
|
||||
$submitted++; $lastSegSecs = $segSecs
|
||||
$stmt2 = Rpc 'igneum_getSegmentStatement' @(('0x{0:x}' -f $first))
|
||||
$aggWei = if ($stmt2) { Hex $stmt2.aggregatorWei } else { 0 }
|
||||
$submittedSegments[$first] = @{ last = $last; aggWei = $aggWei; at = (Get-Date) }
|
||||
"RESULT seg $first submitted $(Stamp) segment $first..$last record accepted (new=$($rr.new), chain_len $($res.segment_chain_len)), aggregator share $([math]::Round($aggWei / 1e18, 4)) IGN, end to end $segSecs s (export+cut+chain+sign+submit)"
|
||||
} else {
|
||||
"RESULT seg $first segment refused $(Stamp): $(if ($rr) { $rr.reason } else { "$reply".Substring(0, [math]::Min(300, "$reply".Length)) }); end to end $segSecs s"
|
||||
}
|
||||
foreach ($b in $first..$last) { Remove-Item (Join-Path $d "block-$b.json") -ErrorAction SilentlyContinue }
|
||||
Remove-Item (Join-Path $d 'export.json') -ErrorAction SilentlyContinue
|
||||
}
|
||||
# --- the end: paid state once more, the rate during the run, the sockets, the prover back on
|
||||
Start-Sleep -Seconds 30
|
||||
$paidTotal = 0; $paidCount = 0
|
||||
foreach ($f in @($submittedSegments.Keys)) {
|
||||
$rec = Rpc 'igneum_getSegmentRecords' @(('0x{0:x}' -f $f))
|
||||
if ($rec -and $rec.paid) { $paidCount++; $paidTotal += (Hex $rec.paid.wei); "RESULT paid $(Stamp) segment $f..$($submittedSegments[$f].last) paid wei=$($rec.paid.wei) carrier=$(Hex $rec.paid.carrierNumber)"; $submittedSegments.Remove($f) }
|
||||
else { "RESULT unpaid $(Stamp) segment $f..$($submittedSegments[$f].last) carried=$(if ($rec) { (@($rec.carried)).Count } else { '?' }) pool=$(if ($rec) { (@($rec.pool)).Count } else { '?' }) $(if ($rec -and $rec.pool) { ($rec.pool | ForEach-Object { "verified=$($_.verified) included=$($_.includedIn) note=$($_.note)" }) -join '; ' })" }
|
||||
}
|
||||
$st = Rpc 'igneum_getProvingStatus' @()
|
||||
if ($st) { "RESULT node_v1 $(Stamp) paidSegments=$($st.v1.paidSegments) paidSegmentWei=$($st.v1.paidSegmentWei) window pending=$($st.v1.segmentsInWindow.pending) proven=$($st.v1.segmentsInWindow.proven) unproven=$($st.v1.segmentsInWindow.unproven) pool entries=$($st.v1.pool.entries) verified=$($st.v1.pool.verified) failed=$($st.v1.pool.failed)" }
|
||||
$t1 = Epoch (Get-Date)
|
||||
$during = RateLines (Epoch $runStart) $t1
|
||||
if ($during.Count) { $m = $during | Measure-Object -Average -Minimum -Maximum; "RESULT rate_during $(Stamp) n=$($during.Count) mean=$([math]::Round($m.Average,2)) min=$($m.Minimum) max=$($m.Maximum) MH/s (the app log's status lines over the run)" } else { "RESULT rate_during $(Stamp) no status lines found" }
|
||||
"RESULT summary $(Stamp) passes=$passes claimed=$claimed submitted=$submitted paid=$paidCount paid_wei=$paidTotal shards_accepted=$shardsAccepted shards_refused=$shardsRefused last_segment_s=$lastSegSecs run_min=$([math]::Round(((Get-Date) - $runStart).TotalMinutes,1))"
|
||||
& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -f sp1-gpu-server; sleep 1; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
||||
"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})"
|
||||
"RESULT end $(Stamp)"
|
||||
Loading…
Reference in a new issue