Merge master 494c9c7 into ember-tune (0.3.12's version files and the proving-v1 app merge; my own 7f6c4e6 came back as ebea8b6)
This commit is contained in:
igneum-josh
commit
fd1917c363
28 changed files with 2060 additions and 29 deletions
2
app/igneum-app/Cargo.lock
generated
2
app/igneum-app/Cargo.lock
generated
|
|
@ -219,7 +219,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "igneum-app"
|
||||
version = "0.3.11"
|
||||
version = "0.3.12"
|
||||
dependencies = [
|
||||
"ed25519-dalek",
|
||||
"getrandom",
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "igneum-app"
|
||||
version = "0.3.11"
|
||||
version = "0.3.12"
|
||||
edition = "2021"
|
||||
description = "Igneum Miner engine: supervises the node, the miner and the GPU workers, and serves the dashboard on 127.0.0.1"
|
||||
license = "MIT"
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
1 ICON "igneum.ico"
|
||||
|
||||
1 VERSIONINFO
|
||||
FILEVERSION 0,3,11,0
|
||||
PRODUCTVERSION 0,3,11,0
|
||||
FILEVERSION 0,3,12,0
|
||||
PRODUCTVERSION 0,3,12,0
|
||||
FILEFLAGSMASK 0x3fL
|
||||
FILEFLAGS 0x0L
|
||||
FILEOS VOS_NT_WINDOWS32
|
||||
|
|
@ -20,12 +20,12 @@ BEGIN
|
|||
BEGIN
|
||||
VALUE "CompanyName", "Igneum"
|
||||
VALUE "FileDescription", "Igneum Miner engine"
|
||||
VALUE "FileVersion", "0.3.11"
|
||||
VALUE "FileVersion", "0.3.12"
|
||||
VALUE "InternalName", "igneum-app"
|
||||
VALUE "LegalCopyright", "Igneum contributors"
|
||||
VALUE "OriginalFilename", "igneum-app.exe"
|
||||
VALUE "ProductName", "Igneum Miner"
|
||||
VALUE "ProductVersion", "0.3.11"
|
||||
VALUE "ProductVersion", "0.3.12"
|
||||
END
|
||||
END
|
||||
BLOCK "VarFileInfo"
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ mod jobrun;
|
|||
mod jobbuild;
|
||||
mod prover;
|
||||
mod provedefault;
|
||||
mod segments;
|
||||
mod verifier;
|
||||
mod wslhost;
|
||||
mod sweep;
|
||||
|
|
|
|||
|
|
@ -47,6 +47,8 @@ pub struct Work {
|
|||
pub shard_wei: u128,
|
||||
/// The first of this machine's keys that is assigned (the label that signs).
|
||||
pub key_hash: String,
|
||||
/// The chain block's DAA score (the deadline clock of spec 7.8).
|
||||
pub daa: u64,
|
||||
}
|
||||
|
||||
/// Parses the node's work list. Newest first, as the node returns it.
|
||||
|
|
@ -67,6 +69,7 @@ pub fn parse_work(v: &Value) -> Vec<Work> {
|
|||
in_pool: w["pool"].as_array().map(|p| !p.is_empty()).unwrap_or(false),
|
||||
shard_wei: hexu(&w["shardWei"]),
|
||||
key_hash: w["assignedKeys"].as_array().and_then(|k| k.first()).and_then(|k| k.as_str()).unwrap_or("").to_string(),
|
||||
daa: hexu(&w["daaScore"]) as u64,
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
|
|
@ -344,6 +347,13 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
let ram_mb = crate::detect::total_ram_mb();
|
||||
let mut last_probe = Instant::now() - Duration::from_secs(600);
|
||||
let mut submitted: Vec<(u64, String, u32, u128)> = Vec::new();
|
||||
// proving v1 segment path: (first, last, aggregator wei) of the segment records this machine submitted
|
||||
let mut submitted_segments: Vec<(u64, u64, u128)> = Vec::new();
|
||||
let mut last_segment_secs: Option<f64> = None;
|
||||
// segment records the node refused by the chain rule ("does not chain to ... pending"): held and offered again
|
||||
// every pass until the segment's deadline (the fresh-record window of spec 7.8 is the segment length in DAA on
|
||||
// the rule as shipped, 6 October 2026; from the fresh-rule switch the first retry lands)
|
||||
let mut held_segments: Vec<HeldSegment> = Vec::new();
|
||||
let mut last_verifier_read = Instant::now() - Duration::from_secs(600);
|
||||
let mut asked_restart = false;
|
||||
// macOS and Linux: the host sits next to the engine, so its pinned ids are read at once, proving on or off
|
||||
|
|
@ -450,7 +460,7 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
});
|
||||
continue;
|
||||
}
|
||||
let work = match evm_rpc(&shared, "igneum_getAssignedShards", json!([keys.iter().map(|(_, h)| h.clone()).collect::<Vec<_>>(), 60]), Duration::from_secs(10)) {
|
||||
let work = match evm_rpc(&shared, "igneum_getAssignedShards", json!([keys.iter().map(|(_, h)| h.clone()).collect::<Vec<_>>(), crate::segments::WORK_LOOKBACK]), Duration::from_secs(10)) {
|
||||
Ok(v) => parse_work(&v),
|
||||
Err(e) => {
|
||||
set(&shared, |p| {
|
||||
|
|
@ -477,6 +487,52 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
});
|
||||
}
|
||||
}
|
||||
// held segment records: offered again, dropped past the deadline
|
||||
if !held_segments.is_empty() {
|
||||
let tip_daa = evm_rpc(&shared, "igneum_getProvingStatus", json!([]), Duration::from_secs(10)).ok().and_then(|st| st["tipDaa"].as_str().and_then(|x| u64::from_str_radix(x.trim_start_matches("0x"), 16).ok())).unwrap_or(0);
|
||||
let mut keep = Vec::new();
|
||||
for h in held_segments.drain(..) {
|
||||
match retry_held(&shared, &h, tip_daa) {
|
||||
Retry::Accepted => {
|
||||
shared.event("proving", &format!("segment {}..{} record accepted on retry {} (held {} s)", h.first, h.last, h.tries + 1, h.since.elapsed().as_secs()));
|
||||
submitted_segments.push((h.first, h.last, h.agg_wei));
|
||||
set(&shared, |p| {
|
||||
p.segments_submitted += 1;
|
||||
p.aggregated += 1;
|
||||
p.segment_note = format!("segment {}..{} accepted on retry", h.first, h.last);
|
||||
});
|
||||
}
|
||||
Retry::Expired(why) => {
|
||||
shared.log(&format!("prover: segment {}..{} record dropped after {} tries: {why}", h.first, h.last, h.tries));
|
||||
}
|
||||
Retry::Again(why) => {
|
||||
let mut h = h;
|
||||
h.tries += 1;
|
||||
if h.tries % 30 == 1 {
|
||||
shared.log(&format!("prover: segment {}..{} record held (try {}): {why}", h.first, h.last, h.tries));
|
||||
}
|
||||
keep.push(h);
|
||||
}
|
||||
}
|
||||
}
|
||||
held_segments = keep;
|
||||
set(&shared, |p| p.segments_held = held_segments.len() as u32);
|
||||
}
|
||||
// paid segments among what we submitted
|
||||
for (first, last, wei) in submitted_segments.clone() {
|
||||
let paid = evm_rpc(&shared, "igneum_getSegmentRecords", json!([format!("{first:#x}")]), Duration::from_secs(10))
|
||||
.ok()
|
||||
.map(|r| !r["paid"].is_null() && r["paid"]["payout"].as_str().map(|a| a.eq_ignore_ascii_case(&payout_address(&shared))).unwrap_or(false))
|
||||
.unwrap_or(false);
|
||||
if paid {
|
||||
submitted_segments.retain(|x| x.0 != first);
|
||||
shared.event("proving", &format!("segment {first}..{last} paid {} IGN to the aggregator", wei as f64 / 1e18));
|
||||
set(&shared, |p| {
|
||||
p.segments_paid += 1;
|
||||
p.segment_paid_wei += wei;
|
||||
});
|
||||
}
|
||||
}
|
||||
let assigned = work.iter().filter(|w| w.assigned).count() as u32;
|
||||
set(&shared, |p| {
|
||||
p.assigned = assigned;
|
||||
|
|
@ -496,10 +552,60 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
}
|
||||
}
|
||||
}
|
||||
// proving v1 segment path (src/segments.rs, 6 October 2026): a whole segment first, the newest shard only
|
||||
// when no whole segment qualifies
|
||||
let payout = shared.settings.lock().unwrap().address.clone();
|
||||
if payout.len() == 42 {
|
||||
if let Some((seg, prev_file, expected_pv)) = pick_segment(&shared, &work, &keys[0].1, &mut attempted_segments, last_segment_secs) {
|
||||
attempted_segments.insert(seg.first);
|
||||
let started = Instant::now();
|
||||
match prove_segment(&shared, t, &seg, &keys[0].0, &payout, prev_file.as_deref(), &expected_pv, &mut submitted) {
|
||||
Ok(SegmentOutcome::Held(h)) => {
|
||||
let secs = started.elapsed().as_secs_f64();
|
||||
last_segment_secs = Some(secs);
|
||||
shared.event("proving", &format!("segment {}..{}: {} shards proven and submitted in {secs:.0} s; the segment record is held ({})", seg.first, seg.last, seg.shards.len(), h.why));
|
||||
set(&shared, |p| {
|
||||
p.segment_last_s = secs;
|
||||
p.status = "submitted".into();
|
||||
p.message = format!("segment {}..{}: shards submitted, the segment record waits for the chain rule", seg.first, seg.last);
|
||||
p.segment_note = format!("segment {}..{} proven whole in {secs:.0} s; its record is held: {}", seg.first, seg.last, h.why);
|
||||
p.current = String::new();
|
||||
});
|
||||
held_segments.push(h);
|
||||
set(&shared, |p| p.segments_held = held_segments.len() as u32);
|
||||
}
|
||||
Ok(SegmentOutcome::Submitted(agg_wei)) => {
|
||||
let secs = started.elapsed().as_secs_f64();
|
||||
last_segment_secs = Some(secs);
|
||||
submitted_segments.push((seg.first, seg.last, agg_wei));
|
||||
shared.event("proving", &format!("segment {}..{}: {} shards proven, aggregated and submitted in {secs:.0} s", seg.first, seg.last, seg.shards.len()));
|
||||
set(&shared, |p| {
|
||||
p.segments_submitted += 1;
|
||||
p.segment_last_s = secs;
|
||||
p.status = "submitted".into();
|
||||
p.message = format!("segment {}..{} submitted; paid when a block carries it", seg.first, seg.last);
|
||||
p.segment_note = format!("segment {}..{} proven whole in {secs:.0} s", seg.first, seg.last);
|
||||
p.current = String::new();
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
shared.log(&format!("prover: segment {}..{}: {e}", seg.first, seg.last));
|
||||
set(&shared, |p| {
|
||||
p.failed += 1;
|
||||
p.status = "idle".into();
|
||||
p.message = e.clone();
|
||||
p.segment_note = e;
|
||||
p.current = String::new();
|
||||
});
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let Some(w) = choose(&work, &attempted) else {
|
||||
set(&shared, |p| {
|
||||
p.status = if submitted.is_empty() { "idle".into() } else { "submitted".into() };
|
||||
p.message = if assigned == 0 { "no shard assigned to this machine and none open in the last 60 blocks".into() } else { "every assigned and open shard is proven or paid".into() };
|
||||
p.message = if assigned == 0 { format!("no shard assigned to this machine and none open in the last {} blocks", crate::segments::WORK_LOOKBACK) } else { "every assigned and open shard is proven or paid".into() };
|
||||
});
|
||||
continue;
|
||||
};
|
||||
|
|
@ -592,6 +698,218 @@ fn loop_forever(shared: Arc<Shared>, bin_dir: PathBuf) {
|
|||
}
|
||||
}
|
||||
|
||||
/// A segment record the node refused by the chain rule, kept with its proof for another offer.
|
||||
pub struct HeldSegment {
|
||||
pub first: u64,
|
||||
pub last: u64,
|
||||
pub deadline_daa: u64,
|
||||
pub record: String,
|
||||
pub proof_path: PathBuf,
|
||||
pub agg_wei: u128,
|
||||
pub why: String,
|
||||
pub tries: u32,
|
||||
pub since: Instant,
|
||||
}
|
||||
|
||||
pub enum SegmentOutcome {
|
||||
Submitted(u128),
|
||||
Held(HeldSegment),
|
||||
}
|
||||
|
||||
pub enum Retry {
|
||||
Accepted,
|
||||
Again(String),
|
||||
Expired(String),
|
||||
}
|
||||
|
||||
/// Offers a held segment record again: accepted, held for another pass, or dropped past the segment's deadline.
|
||||
fn retry_held(shared: &Shared, h: &HeldSegment, tip_daa: u64) -> Retry {
|
||||
if tip_daa > 0 && tip_daa + 1 > h.deadline_daa {
|
||||
return Retry::Expired(format!("past the deadline DAA {} at tip DAA {tip_daa}", h.deadline_daa));
|
||||
}
|
||||
let Ok(proof) = std::fs::read(&h.proof_path) else { return Retry::Expired(format!("proof file {} gone", h.proof_path.display())) };
|
||||
let proof_hex = format!("0x{}", proof.iter().map(|b| format!("{b:02x}")).collect::<String>());
|
||||
match evm_rpc(shared, "igneum_submitSegmentRecord", json!([{ "record": h.record, "proof": proof_hex }]), Duration::from_secs(60)) {
|
||||
Ok(r) if r["accepted"].as_bool().unwrap_or(false) => Retry::Accepted,
|
||||
Ok(r) => Retry::Again(r["reason"].as_str().unwrap_or("?").to_string()),
|
||||
Err(e) => Retry::Again(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Proving v1 segment path, the choice: the node's v1 status (active, the grid start, the segment length, the
|
||||
/// deadline clock), the work list grouped into whole untouched segments (`segments::whole_segments`), the
|
||||
/// candidates inside the deadline ranked for this key, then for the best three the node's segment statement:
|
||||
/// executed and pending; the previous segment either paid with its proof in this node's pool (the chain continues,
|
||||
/// `--prev`) or not paid and with no verified record of it waiting in the pool (fresh). Returns the segment, the
|
||||
/// previous proof's host path when the chain continues, and the public values the node expects.
|
||||
fn pick_segment(shared: &Shared, work: &[Work], key_hash: &str, attempted: &mut HashSet<u64>, last_secs: Option<f64>) -> Option<(crate::segments::SegmentWork, Option<String>, String)> {
|
||||
let hexu = |x: &Value| x.as_str().and_then(|s| u64::from_str_radix(s.trim_start_matches("0x"), 16).ok()).unwrap_or(0);
|
||||
let st = evm_rpc(shared, "igneum_getProvingStatus", json!([]), Duration::from_secs(10)).ok()?;
|
||||
let v1 = &st["v1"];
|
||||
if !v1["active"].as_bool().unwrap_or(false) || v1["start"].is_null() {
|
||||
return None;
|
||||
}
|
||||
let (start, n, unproven, tip_daa) = (hexu(&v1["start"]), hexu(&v1["segmentBlocks"]).max(1), hexu(&v1["unprovenDaa"]), hexu(&st["tipDaa"]));
|
||||
let segs = crate::segments::whole_segments(start, n, unproven, work);
|
||||
let need = crate::segments::need_daa(last_secs);
|
||||
let cands = crate::segments::candidates(&segs, tip_daa, need, key_hash, attempted);
|
||||
if cands.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let dir = shared.runtime.app_dir.join("proving");
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
let wsl = cfg!(windows);
|
||||
let as_host_path = |p: &Path| if wsl { wsl_path(p) } else { p.display().to_string() };
|
||||
for seg in cands.into_iter().take(3) {
|
||||
let Ok(stmt) = evm_rpc(shared, "igneum_getSegmentStatement", json!([format!("{:#x}", seg.first)]), Duration::from_secs(10)) else { continue };
|
||||
if !stmt["executed"].as_bool().unwrap_or(false) || stmt["status"]["status"].as_str() != Some("pending") {
|
||||
attempted.insert(seg.first);
|
||||
continue;
|
||||
}
|
||||
let prev = &stmt["previous"];
|
||||
if prev.is_null() {
|
||||
// fresh only when no record of the previous segment is waiting to be carried (the chain rule would
|
||||
// refuse a fresh record once that one pays)
|
||||
if seg.first >= start + n {
|
||||
let p = evm_rpc(shared, "igneum_getSegmentRecords", json!([format!("{:#x}", seg.first - n)]), Duration::from_secs(10)).unwrap_or(Value::Null);
|
||||
let waiting = p["pool"].as_array().map(|a| a.iter().any(|e| e["verified"] == json!(true) && e["includedIn"].is_null())).unwrap_or(false);
|
||||
if waiting || !p["paid"].is_null() {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return Some((seg, None, stmt["publicValuesFresh"].as_str().unwrap_or("").to_string()));
|
||||
}
|
||||
if prev["proofInPool"] != json!(true) {
|
||||
continue;
|
||||
}
|
||||
let Ok(got) = evm_rpc(shared, "igneum_getSegmentProofBytes", json!([prev["first"], prev["keyHash"]]), Duration::from_secs(60)) else { continue };
|
||||
let hex = got["proof"].as_str().unwrap_or("").trim_start_matches("0x").to_string();
|
||||
if hex.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let bytes: Vec<u8> = (0..hex.len() / 2).map(|k| u8::from_str_radix(&hex[2 * k..2 * k + 2], 16).unwrap_or(0)).collect();
|
||||
let f = dir.join(format!("prev-{}.bin", seg.first));
|
||||
if std::fs::write(&f, bytes).is_err() {
|
||||
continue;
|
||||
}
|
||||
return Some((seg, Some(as_host_path(&f)), stmt["publicValuesContinuing"].as_str().unwrap_or("").to_string()));
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Proving v1 segment path, the work: one export of the chain to the segment's last block, one fixture per block,
|
||||
/// one host run (`--mode chain --save-shards`, `--prev` when the chain continues) that proves every shard and
|
||||
/// aggregates the segment, then every shard record signed and submitted (the shard payouts) and the segment
|
||||
/// record signed and submitted (the aggregator share). Returns the segment's aggregator wei.
|
||||
fn prove_segment(shared: &Shared, t: &Tools, seg: &crate::segments::SegmentWork, label: &str, payout: &str, prev_file: Option<&str>, expected_pv: &str, submitted: &mut Vec<(u64, String, u32, u128)>) -> Result<SegmentOutcome, String> {
|
||||
let (first, last) = (seg.first, seg.last);
|
||||
let dir = shared.runtime.app_dir.join("proving").join(format!("seg-{first}"));
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
let as_host_path = |p: &Path| if t.wsl { wsl_path(p) } else { p.display().to_string() };
|
||||
set(shared, |p| {
|
||||
p.status = "proving".into();
|
||||
p.current = format!("segment {first}..{last} ({} shards)", seg.shards.len());
|
||||
p.started_at = crate::platform::unix_now_f();
|
||||
p.message = "exporting the chain and cutting the segment's blocks".into();
|
||||
});
|
||||
shared.log(&format!("prover: segment {first}..{last} claimed ({} shards{}): export, cut, chain ({}), sign, submit", seg.shards.len(), if prev_file.is_some() { ", continuing the previous segment's proof" } else { ", fresh" }, if t.cuda { "CUDA" } else { "CPU" }));
|
||||
// 1. export once, cut every block
|
||||
let seq = dir.join("seq.json");
|
||||
let export = evm_rpc(shared, "igneum_exportSegments", json!(["0x0", format!("{last:#x}")]), Duration::from_secs(300))?;
|
||||
std::fs::write(&seq, export.to_string()).map_err(|e| e.to_string())?;
|
||||
let mut fixtures: Vec<String> = Vec::new();
|
||||
for b in first..=last {
|
||||
let fixture = dir.join(format!("block-{b}.json"));
|
||||
let (ok, out) = run_tool(shared, t, &t.export, &[as_host_path(&seq), b.to_string(), as_host_path(&fixture)], &[], Duration::from_secs(600), &dir.join(format!("export-{b}.log")));
|
||||
if !ok || !fixture.exists() {
|
||||
return Err(format!("exporter, block {b}: {}", out.lines().rev().find(|l| !l.trim().is_empty()).unwrap_or("failed")));
|
||||
}
|
||||
fixtures.push(as_host_path(&fixture));
|
||||
}
|
||||
let _ = std::fs::remove_file(&seq);
|
||||
// 2. the chain: every shard proven, every block aggregated with the previous, in one process
|
||||
set(shared, |p| p.message = format!("proving {} shards and aggregating segment {first}..{last} ({})", seg.shards.len(), if t.cuda { "GPU" } else { "CPU, slow" }));
|
||||
let results = dir.join("chain-results.json");
|
||||
let mut args: Vec<String> = vec!["--mode".into(), "chain".into(), "--chain".into(), fixtures.join(","), "--prover".into(), payout.to_string(), "--save-shards".into(), "--out".into(), as_host_path(&results)];
|
||||
if let Some(pf) = prev_file {
|
||||
args.push("--prev".into());
|
||||
args.push(pf.to_string());
|
||||
}
|
||||
let (ok, out) = run_tool(shared, t, &t.host, &args, &[("SP1_PROVER", if t.cuda { "cuda" } else { "cpu" }), ("RUST_LOG", "off")], Duration::from_secs(3 * 3600), &dir.join("chain.log"));
|
||||
if !ok || !results.exists() {
|
||||
let last_line = out.lines().rev().find(|l| l.contains("RESULT") || l.contains("rror")).unwrap_or("failed").to_string();
|
||||
let hint = if last_line.contains("PermissionDenied") { " (a GPU-server socket /tmp/sp1-cuda-*.sock owned by another user: the root-socket class)" } else { "" };
|
||||
return Err(format!("chain: {last_line}{hint}"));
|
||||
}
|
||||
let res: Value = serde_json::from_str(&std::fs::read_to_string(&results).map_err(|e| e.to_string())?).map_err(|e| e.to_string())?;
|
||||
let to_win = |f: &str| if t.wsl { PathBuf::from(f.replace("/mnt/c/", "C:/")) } else { PathBuf::from(f) };
|
||||
// 3. the shard records
|
||||
let chain = chain_name(shared);
|
||||
let mut shard_ok = 0usize;
|
||||
for b in res["blocks"].as_array().cloned().unwrap_or_default() {
|
||||
for r in b["shard_records"].as_array().cloned().unwrap_or_default() {
|
||||
let (number, hash, shard) = (r["number"].as_u64().unwrap_or(0), r["block_hash"].as_str().unwrap_or("").to_string(), r["shard"].as_u64().unwrap_or(0) as u32);
|
||||
let statement = r["statement"].as_str().unwrap_or("").to_string();
|
||||
let proof_sha = r["proof_sha256"].as_str().unwrap_or("").to_string();
|
||||
let proof_file = r["proof_file"].as_str().unwrap_or("").to_string();
|
||||
let sg = crate::detect::run_timeout(crate::platform::quiet(&mut Command::new(&t.miner)).args(["sign-record", label, &chain, &hash, &number.to_string(), &shard.to_string(), payout, &statement, &proof_sha]), None, Duration::from_secs(20)).ok_or("sign-record did not run")?;
|
||||
let signed: Value = serde_json::from_str(sg.lines().last().unwrap_or("")).map_err(|_| format!("sign-record: {}", sg.trim()))?;
|
||||
let record = signed["record"].as_str().ok_or("sign-record gave no record")?.to_string();
|
||||
let proof = std::fs::read(to_win(&proof_file)).map_err(|e| format!("proof file {proof_file}: {e}"))?;
|
||||
let proof_hex = format!("0x{}", proof.iter().map(|b| format!("{b:02x}")).collect::<String>());
|
||||
let out = evm_rpc(shared, "igneum_submitProofRecord", json!([{ "record": record, "proof": proof_hex }]), Duration::from_secs(60))?;
|
||||
if out["accepted"].as_bool().unwrap_or(false) {
|
||||
shard_ok += 1;
|
||||
let wei = seg.shards.iter().position(|(n, _, s)| *n == number && *s == shard).map(|_| seg.shard_wei / seg.shards.len().max(1) as u128).unwrap_or(0);
|
||||
submitted.push((number, hash.clone(), shard, wei));
|
||||
} else {
|
||||
shared.log(&format!("prover: segment {first}..{last}: block {number} shard {shard} record refused: {}", out["reason"].as_str().unwrap_or("?")));
|
||||
}
|
||||
}
|
||||
}
|
||||
if shard_ok != seg.shards.len() {
|
||||
return Err(format!("{shard_ok} of {} shard records accepted; the segment record is not submitted", seg.shards.len()));
|
||||
}
|
||||
set(shared, |p| {
|
||||
p.proved += shard_ok as u32;
|
||||
p.submitted += shard_ok as u32;
|
||||
});
|
||||
// 4. the segment record: the aggregated statement against the node's native one (every field but provers)
|
||||
let pv = res["segment_public_values"].as_str().ok_or("no public values in the chain results")?.to_string();
|
||||
let proof_sha = res["segment_proof_sha256"].as_str().ok_or("no segment proof hash in the chain results")?.to_string();
|
||||
let proof_file = res["segment_proof_file"].as_str().ok_or("no segment proof file in the chain results")?.to_string();
|
||||
let strip = |h: &str| { let h = h.trim_start_matches("0x"); if h.len() == 680 { format!("{}{}", &h[..472], &h[536..]) } else { h.to_string() } };
|
||||
if strip(&pv) != strip(expected_pv) {
|
||||
return Err(format!("the aggregated statement differs from the node's native statement (it would be vetoed); ours {} node {}", &pv[..66.min(pv.len())], &expected_pv[..66.min(expected_pv.len())]));
|
||||
}
|
||||
let last_hash = seg.shards.iter().rev().find(|(n, _, _)| *n == last).map(|(_, h, _)| h.clone()).ok_or("no last block hash")?;
|
||||
let sg = crate::detect::run_timeout(crate::platform::quiet(&mut Command::new(&t.miner)).args(["sign-segment-record", label, &chain, &first.to_string(), &last.to_string(), &last_hash, payout, &pv, &proof_sha]), None, Duration::from_secs(20)).ok_or("sign-segment-record did not run")?;
|
||||
let signed: Value = serde_json::from_str(sg.lines().last().unwrap_or("")).map_err(|_| format!("sign-segment-record: {}", sg.trim()))?;
|
||||
let record = signed["record"].as_str().ok_or("sign-segment-record gave no record")?.to_string();
|
||||
let proof = std::fs::read(to_win(&proof_file)).map_err(|e| format!("segment proof file {proof_file}: {e}"))?;
|
||||
let proof_hex = format!("0x{}", proof.iter().map(|b| format!("{b:02x}")).collect::<String>());
|
||||
let r = evm_rpc(shared, "igneum_submitSegmentRecord", json!([{ "record": record, "proof": proof_hex }]), Duration::from_secs(60))?;
|
||||
let hexu = |x: &Value| x.as_str().and_then(|s| u128::from_str_radix(s.trim_start_matches("0x"), 16).ok()).unwrap_or(0);
|
||||
let stmt = evm_rpc(shared, "igneum_getSegmentStatement", json!([format!("{first:#x}")]), Duration::from_secs(10)).unwrap_or(Value::Null);
|
||||
let agg_wei = hexu(&stmt["aggregatorWei"]);
|
||||
// the fixtures and the export go; the proofs stay (the next segment's chain link, and a held record's offer)
|
||||
for b in first..=last {
|
||||
let _ = std::fs::remove_file(dir.join(format!("block-{b}.json")));
|
||||
}
|
||||
if !r["accepted"].as_bool().unwrap_or(false) {
|
||||
let why = r["reason"].as_str().unwrap_or("?").to_string();
|
||||
// the chain rule's refusal ("does not chain to ... pending until DAA ..."): held, not failed; anything else
|
||||
// (a bad statement, a late carrier) is an error
|
||||
if why.contains("does not chain") {
|
||||
let deadline = hexu(&stmt["status"]["deadline_daa"]) as u64;
|
||||
return Ok(SegmentOutcome::Held(HeldSegment { first, last, deadline_daa: if deadline > 0 { deadline } else { u64::MAX }, record, proof_path: to_win(&proof_file), agg_wei, why, tries: 0, since: Instant::now() }));
|
||||
}
|
||||
return Err(format!("segment record refused: {why}"));
|
||||
}
|
||||
set(shared, |p| p.aggregated += 1);
|
||||
Ok(SegmentOutcome::Submitted(agg_wei))
|
||||
}
|
||||
|
||||
/// Proving v1 (spec 7.8): one aggregation attempt. When the node reports v1 active, takes the newest executed
|
||||
/// segment that is still pending and not yet attempted here, needs one shard proof per shard of every block in
|
||||
/// this node's pool (`igneum_getProofBytes`, a verified one when there is one) and, when the previous segment is
|
||||
|
|
|
|||
214
app/igneum-app/src/segments.rs
Normal file
214
app/igneum-app/src/segments.rs
Normal file
|
|
@ -0,0 +1,214 @@
|
|||
//! Proving v1 (spec 7.8): segment-aligned work for the prover loop (6 October 2026).
|
||||
//!
|
||||
//! The shipped loop took the newest open shard each pass, so one prover scattered one block in about 45 across
|
||||
//! the segment grid and no segment ever had all its blocks proven (node 1, 04:16Z: pending 55, proven 0). Here a
|
||||
//! free prover claims a whole segment (`proving_v1_segment_blocks` consecutive chain blocks), proves every shard
|
||||
//! of it in order from one export in one host run (`--mode chain --save-shards`), submits the shard records and the
|
||||
//! aggregated segment record, then takes the next. One card completes whole segments at its own rate instead of
|
||||
//! completing none.
|
||||
//!
|
||||
//! The choice is deterministic per prover: among the untouched whole segments still inside their deadline by a
|
||||
//! margin, the lowest FNV-1a of (first block, this prover's key hash) wins, so several provers spread over the
|
||||
//! candidates without a coordinator; the per-block fallback (`prover::choose`) stays for the passes where no whole
|
||||
//! segment qualifies.
|
||||
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
|
||||
use crate::prover::Work;
|
||||
|
||||
/// The least time a claimed segment is given before its deadline (DAA units, about one a second on devnet): the
|
||||
/// chain of 8 empty blocks took 135.6 s cold beside the miner (bench-log, 5 October 2026), so 240 leaves the
|
||||
/// submission and the carrying block inside the window.
|
||||
pub const SEGMENT_MARGIN_MIN_DAA: u64 = 240;
|
||||
/// The margin grows with what the last segment actually took, times this.
|
||||
pub const SEGMENT_MARGIN_FACTOR: f64 = 1.5;
|
||||
/// How far back the work list reaches (chain blocks): the record window, so every open segment inside the
|
||||
/// deadline is visible.
|
||||
pub const WORK_LOOKBACK: u64 = 600;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct SegmentWork {
|
||||
pub first: u64,
|
||||
pub last: u64,
|
||||
/// the last block's DAA score; the deadline is it plus `proving_v1_unproven_daa`
|
||||
pub last_daa: u64,
|
||||
pub deadline_daa: u64,
|
||||
/// (chain block number, block hash, shard index) in chain order
|
||||
pub shards: Vec<(u64, String, u32)>,
|
||||
/// the shard payouts summed (what the shards earn when carried)
|
||||
pub shard_wei: u128,
|
||||
}
|
||||
|
||||
/// The segment holding chain block `number` on the grid that starts at `start`.
|
||||
pub fn segment_of(start: u64, n: u64, number: u64) -> (u64, u64) {
|
||||
let n = n.max(1);
|
||||
let k = number.saturating_sub(start) / n;
|
||||
(start + k * n, start + k * n + n - 1)
|
||||
}
|
||||
|
||||
/// The DAA margin a segment must have before its deadline: the floor, or 1.5 times the last segment's wall time.
|
||||
pub fn need_daa(last_segment_secs: Option<f64>) -> u64 {
|
||||
let from_last = last_segment_secs.map(|s| (s * SEGMENT_MARGIN_FACTOR).ceil() as u64).unwrap_or(0);
|
||||
from_last.max(SEGMENT_MARGIN_MIN_DAA)
|
||||
}
|
||||
|
||||
/// Groups the node's work list into whole, untouched segments: every block of the segment is in the list, every
|
||||
/// listed shard is open (past its exclusive window), unpaid and not in this node's pool from us. A segment with a
|
||||
/// block missing (inside the exclusive window, or outside the lookback) or a shard already paid is not a candidate.
|
||||
pub fn whole_segments(start: u64, n: u64, unproven_daa: u64, work: &[Work]) -> Vec<SegmentWork> {
|
||||
let n = n.max(1);
|
||||
let mut by_block: BTreeMap<u64, Vec<&Work>> = BTreeMap::new();
|
||||
for w in work.iter().filter(|w| w.number >= start) {
|
||||
by_block.entry(w.number).or_default().push(w);
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
let mut seen = HashSet::new();
|
||||
for number in by_block.keys() {
|
||||
let (first, last) = segment_of(start, n, *number);
|
||||
if !seen.insert(first) {
|
||||
continue;
|
||||
}
|
||||
let mut shards = Vec::new();
|
||||
let mut wei: u128 = 0;
|
||||
let mut last_daa = 0;
|
||||
let mut whole = true;
|
||||
for b in first..=last {
|
||||
let Some(entries) = by_block.get(&b) else {
|
||||
whole = false;
|
||||
break;
|
||||
};
|
||||
let mut e: Vec<&&Work> = entries.iter().collect();
|
||||
e.sort_by_key(|w| w.shard);
|
||||
e.dedup_by_key(|w| w.shard);
|
||||
if e.iter().any(|w| !w.open || w.paid || w.in_pool) {
|
||||
whole = false;
|
||||
break;
|
||||
}
|
||||
for w in e {
|
||||
shards.push((w.number, w.hash.clone(), w.shard));
|
||||
wei = wei.saturating_add(w.shard_wei);
|
||||
if b == last {
|
||||
last_daa = w.daa;
|
||||
}
|
||||
}
|
||||
}
|
||||
if whole && !shards.is_empty() {
|
||||
out.push(SegmentWork { first, last, last_daa, deadline_daa: last_daa.saturating_add(unproven_daa), shards, shard_wei: wei });
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// FNV-1a 64 of the segment's first block and this prover's key hash: the per-prover rank.
|
||||
pub fn rank(first: u64, key_hash: &str) -> u64 {
|
||||
let mut h: u64 = 0xcbf29ce484222325;
|
||||
for b in first.to_be_bytes().iter().chain(key_hash.as_bytes()) {
|
||||
h ^= *b as u64;
|
||||
h = h.wrapping_mul(0x100000001b3);
|
||||
}
|
||||
h
|
||||
}
|
||||
|
||||
/// The segments to try, best first: inside the deadline by `need` DAA at `tip_daa`, not attempted, ranked by
|
||||
/// `rank(first, key)` (ties by the older first block).
|
||||
pub fn candidates(segs: &[SegmentWork], tip_daa: u64, need: u64, key_hash: &str, attempted: &HashSet<u64>) -> Vec<SegmentWork> {
|
||||
let mut c: Vec<SegmentWork> = segs.iter().filter(|s| !attempted.contains(&s.first) && s.deadline_daa >= tip_daa.saturating_add(1).saturating_add(need)).cloned().collect();
|
||||
c.sort_by(|a, b| rank(a.first, key_hash).cmp(&rank(b.first, key_hash)).then(a.first.cmp(&b.first)));
|
||||
c
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn w(number: u64, shard: u32, daa: u64, open: bool, paid: bool, in_pool: bool) -> Work {
|
||||
Work { number, hash: format!("0x{number:064x}"), shard, pgas: 0, tx_count: 0, assigned: false, open, paid, in_pool, shard_wei: 10, key_hash: String::new(), daa }
|
||||
}
|
||||
|
||||
fn grid(start: u64, n: u64, segments: u64, daa0: u64) -> Vec<Work> {
|
||||
(0..segments * n).map(|i| w(start + i, 0, daa0 + i, true, false, false)).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_grid_is_counted_from_the_first_v1_block() {
|
||||
assert_eq!(segment_of(100, 8, 100), (100, 107));
|
||||
assert_eq!(segment_of(100, 8, 107), (100, 107));
|
||||
assert_eq!(segment_of(100, 8, 108), (108, 115));
|
||||
assert_eq!(segment_of(100, 8, 123), (116, 123));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_whole_open_unpaid_untouched_segments_qualify() {
|
||||
let mut work = grid(100, 4, 3, 1000); // 100..111, three segments
|
||||
work.retain(|x| x.number != 105); // 104..107 has a block missing (inside its exclusive window, say)
|
||||
work.iter_mut().find(|x| x.number == 110).unwrap().paid = true; // 108..111 has a paid shard
|
||||
let segs = whole_segments(100, 4, 600, &work);
|
||||
assert_eq!(segs.len(), 1);
|
||||
assert_eq!((segs[0].first, segs[0].last), (100, 103));
|
||||
assert_eq!(segs[0].shards.len(), 4);
|
||||
assert_eq!(segs[0].last_daa, 1003);
|
||||
assert_eq!(segs[0].deadline_daa, 1603);
|
||||
assert_eq!(segs[0].shard_wei, 40);
|
||||
// a shard of ours already in the pool, or one still exclusive, also disqualifies
|
||||
let mut work = grid(100, 4, 1, 1000);
|
||||
work[1].in_pool = true;
|
||||
assert!(whole_segments(100, 4, 600, &work).is_empty());
|
||||
let mut work = grid(100, 4, 1, 1000);
|
||||
work[3].open = false;
|
||||
assert!(whole_segments(100, 4, 600, &work).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_block_with_several_shards_lists_them_in_order() {
|
||||
let mut work = grid(100, 2, 1, 1000);
|
||||
work.push(w(101, 1, 1001, true, false, false));
|
||||
work.push(w(100, 1, 1000, true, false, false));
|
||||
let segs = whole_segments(100, 2, 600, &work);
|
||||
assert_eq!(segs[0].shards.iter().map(|(n, _, s)| (*n, *s)).collect::<Vec<_>>(), vec![(100, 0), (100, 1), (101, 0), (101, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_deadline_margin_and_the_attempted_set_filter_the_candidates() {
|
||||
let work = grid(100, 8, 4, 1000); // 100..131, deadlines 1607, 1615, 1623, 1631
|
||||
let segs = whole_segments(100, 8, 600, &work);
|
||||
assert_eq!(segs.len(), 4);
|
||||
// at tip DAA 1380 with a 240 margin only the segments with a deadline at or past 1621 remain
|
||||
let c = candidates(&segs, 1380, 240, "0xkey", &HashSet::new());
|
||||
let firsts: Vec<u64> = c.iter().map(|s| s.first).collect();
|
||||
assert_eq!(firsts.len(), 2);
|
||||
assert!(firsts.contains(&116) && firsts.contains(&124));
|
||||
let mut attempted = HashSet::new();
|
||||
attempted.insert(firsts[0]);
|
||||
let c2 = candidates(&segs, 1380, 240, "0xkey", &attempted);
|
||||
assert_eq!(c2.len(), 1);
|
||||
assert_eq!(c2[0].first, firsts[1]);
|
||||
// past every deadline: nothing
|
||||
assert!(candidates(&segs, 1700, 240, "0xkey", &HashSet::new()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_order_is_deterministic_per_key_and_differs_between_keys() {
|
||||
let work = grid(100, 8, 6, 1000);
|
||||
let segs = whole_segments(100, 8, 600, &work);
|
||||
let a = candidates(&segs, 1000, 240, "0xaaaa", &HashSet::new());
|
||||
let a2 = candidates(&segs, 1000, 240, "0xaaaa", &HashSet::new());
|
||||
assert_eq!(a, a2);
|
||||
assert_eq!(a.len(), 6);
|
||||
// two provers rank the six candidates differently (the spread); the sets are the same
|
||||
let b = candidates(&segs, 1000, 240, "0xbbbb", &HashSet::new());
|
||||
let (fa, fb): (Vec<u64>, Vec<u64>) = (a.iter().map(|s| s.first).collect(), b.iter().map(|s| s.first).collect());
|
||||
let mut sa = fa.clone();
|
||||
let mut sb = fb.clone();
|
||||
sa.sort();
|
||||
sb.sort();
|
||||
assert_eq!(sa, sb);
|
||||
assert_ne!(fa, fb, "two keys should not rank six segments identically");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_margin_follows_the_last_segment_time() {
|
||||
assert_eq!(need_daa(None), 240);
|
||||
assert_eq!(need_daa(Some(100.0)), 240);
|
||||
assert_eq!(need_daa(Some(190.0)), 285);
|
||||
}
|
||||
}
|
||||
|
|
@ -221,6 +221,15 @@ pub struct ProvingState {
|
|||
/// proving v1: segment records this machine aggregated and submitted, and the aggregator's last line
|
||||
pub aggregated: u32,
|
||||
pub segment_note: String,
|
||||
/// proving v1 segment path (6 October 2026): whole segments this machine proved and submitted, paid, and what
|
||||
/// they paid (wei as a decimal string, see `paid_wei`); the last segment's wall time
|
||||
pub segments_submitted: u32,
|
||||
pub segments_paid: u32,
|
||||
#[serde(serialize_with = "u128_string")]
|
||||
pub segment_paid_wei: u128,
|
||||
pub segment_last_s: f64,
|
||||
/// segment records the node refused by the chain rule and this machine offers again each pass
|
||||
pub segments_held: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Serialize, Default)]
|
||||
|
|
|
|||
|
|
@ -1054,6 +1054,11 @@ if (typeof document !== 'undefined') (function () {
|
|||
setText('pv-submitted', String(pv.submitted || 0));
|
||||
setText('pv-paid', String(pv.paid || 0));
|
||||
setText('pv-paid-sub', pv.paid_wei ? (Number(pv.paid_wei) / 1e18).toFixed(4) + ' IGN earned' : 'shards paid out');
|
||||
// proving v1 segments (6 October 2026): whole segments this machine proved, and the segment path's last line
|
||||
var segLine = '';
|
||||
if (pv.segments_submitted) segLine = 'Segments: ' + pv.segments_submitted + ' proven whole, ' + (pv.segments_paid || 0) + ' paid' + (pv.segment_paid_wei && Number(pv.segment_paid_wei) ? ' (' + (Number(pv.segment_paid_wei) / 1e18).toFixed(4) + ' IGN to the aggregator)' : '') + (pv.segment_last_s ? ', the last in ' + Math.round(pv.segment_last_s) + ' s' : '') + (pv.segments_held ? ', ' + pv.segments_held + ' record' + (pv.segments_held > 1 ? 's' : '') + ' held for the chain rule' : '') + '.';
|
||||
if (pv.segment_note && enabled) segLine += (segLine ? ' ' : '') + pv.segment_note + '.';
|
||||
$('pv-seg-note').hidden = !segLine; setText('pv-seg-note', segLine);
|
||||
var v = View.verifierWords(pv);
|
||||
setText('pv-verifier', v.word); $('pv-verifier').className = 'big-word ' + v.tone;
|
||||
$('pv-verifier-note').hidden = !pv.verifier_note; setText('pv-verifier-note', pv.verifier_note || '');
|
||||
|
|
|
|||
|
|
@ -219,6 +219,7 @@
|
|||
<div class="cell"><div class="k">proven</div><div class="v" id="pv-submitted">0</div><div class="s">proofs sent to the node</div></div>
|
||||
<div class="cell"><div class="k">paid</div><div class="v" id="pv-paid">0</div><div class="s" id="pv-paid-sub">shards paid out</div></div>
|
||||
</div>
|
||||
<p class="note" id="pv-seg-note" hidden></p>
|
||||
<div class="grid2">
|
||||
<div class="card">
|
||||
<div class="card-head"><h3>Verifier</h3><div class="eyebrow">the node's check</div></div>
|
||||
|
|
|
|||
|
|
@ -3,6 +3,6 @@
|
|||
// packaging/windows/Igneum-Miner.iss when the app version moves. Include guards, not #pragma once: rc.exe reads it too.
|
||||
#ifndef IGNEUM_HOST_VERSION_H
|
||||
#define IGNEUM_HOST_VERSION_H
|
||||
#define IGNEUM_HOST_VERSION_STR "0.3.11"
|
||||
#define IGNEUM_HOST_VERSION_RC 0,3,11,0
|
||||
#define IGNEUM_HOST_VERSION_STR "0.3.12"
|
||||
#define IGNEUM_HOST_VERSION_RC 0,3,12,0
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -2095,3 +2095,140 @@ Reported by the aggregation-cost agent: PC 2's `/api/state` answered `{}` (2 byt
|
|||
|
||||
Cause: `ProvingState.paid_wei: u128` and serde_json `to_value` (1.0.151, `value/ser.rs` `serialize_u128`: u64 range or an error); the error became `json!({})`. Fix: the field serialises as a decimal string; `state_json` logs the error once. Test `a_paid_total_over_u64_max_still_serialises_the_whole_state` (`cargo test --offline -q paid_wei`, 1 passed).
|
||||
|
||||
| The fast-time 3-node harness (`tools/proving-v1/net.mjs`, 29950+, suffix 956, every node in trust mode, three vmine voters, v0 at DAA 60, v1 at DAA 120, 4 blocks a segment, unproven after 60 DAA, a tenth to the aggregator; fork b177718e built on this Mac) | run 2, 19:13:01Z to 19:16:19Z, under the run lock: PASSED, 21 checks in 197.3 s (`tools/proving-v1/report-2026-10-05.json`). v1 start = chain block 119 on all three nodes; the native statement identical on all three. Known-finished: segment 119..122's fresh-chain record submitted to n1 at t=131.1 s, relayed, verified (trust) and PAID on n0 1.0 s later at chain block 129, 253,611,648,000,000,000 wei = a tenth of the four credits, the same on every node, the payout address holding it. Chain rule: segment 123..126's fresh-chain record refused ("does not chain to segment 119..122 ... proven (record paid at chain block 129)"), the continuing one (chain_len 8) accepted and paid. Known-failed: segment 127..130 left without a record: a fresh-chain record for 131..134 refused while 127..130 was pending ("pending until DAA 191"); at DAA 192 the status read unproven, a late record for 127..130 refused ("unproven: carried after the deadline"), the fresh-chain record for 131..134 accepted and paid with chain_len 4; `segmentsInWindow` proven 3, unproven 1. The shard side: a v1 shard's `shardWei` = 90% of its block's credit. Run 1 (19:10Z) failed in its own tooling (the signer's argument order), fixed |
|
||||
|
||||
## 5 October 2026 (night), aggregation cost on the RTX 5090: what a per-block aggregation spends and what each lever gives (proving engineer, agg-cost)
|
||||
|
||||
Josh, 5 October 2026: "fix everything else in the numbers tonight". The number under test: the chained segment aggregation cost 9.6 to 9.7 s a block on PC 2's 5090 while the card mined (`chain-pc2-pv1c`, the entry above), 2.2 s on 4 October with the card to itself. Target: under 3 s a block, the miner's slowdown of the prover under 1.5x, the proof statement unchanged. Branch `agg-cost` (worktree `igneum-wt-agg-cost`, from `proving-v1` 219517f). Host changes (statement untouched, `elf/` untouched): the aggregation's stdin build timed apart from the prove call, the deferred-proof count and the SP1 knobs in the RESULT lines, `--mode chain --save-shards` (every shard's compressed proof written next to the results, so `--mode aggregate` re-runs the same proofs under other settings). Jobs: `agg-cost-pc2-1` (21:01:20Z to 21:25:11Z, `tools/proving-v1/pc2-agg-cost.ps1`, the package `igneum-prove-wsl2-aggcost.zip` fetched by `fetch-prove-aggcost` 20:55:39Z, built in WSL2 against the live target dir in 5 s, installed to `/opt/igneum-aggcost`, the live `/opt/igneum` untouched, `--mode id` the pinned pair) and `agg-cost-pc2-2` (21:34:00Z, the same script). The live prover was switched OFF for the runs (its sp1-gpu-server would otherwise be shared through `/tmp/sp1-cuda-0.sock` and carry its own environment; `gpu_server_before running=0`) and ON again at the end. Fixtures: four consecutive live blocks cut from PC 2's own node (86165..86168 at tip 86195, one empty shard each, every one MATCHES natively), the same four for every phase of job 1. App 0.3.9 on PC 2 throughout.
|
||||
|
||||
Known-finished case of the host changes before the GPU (this Mac, CPU, run lock, 20:41Z to 20:44Z): `--mode chain` over `fixtures/chain/block-81046.json` with `--save-shards` (shard 38.5 s, aggregate 43.4 s, the proof file written), then `--mode aggregate` over that saved shard proof with `SP1_WORKER_VERIFY_INTERMEDIATES=false` (46.6 s, the same statement `0x3dedb8ea...`), `--mode verify-segment` VERIFIED in 0.027 s; known-failed: a wrong statement NOT VERIFIED in 0.027 s. Unit tests: `cargo test --release -p igneum-prove-core -p igneum-prove-host`: core 8 passed, host 9 passed and 1 ignored (build lock, 20:53Z).
|
||||
|
||||
### Lever 1, the profile: where a per-block aggregation goes
|
||||
|
||||
| What | Measured (job `agg-cost-pc2-1`) |
|
||||
|---|---|
|
||||
| The host's own share of an aggregation (the stdin build: the AggInput, the proof clones into the request) | 0.000 s on every block, mining or idle (the `stdin` field of every `RESULT chain block` line): everything is inside the one `prove().compressed()` call to the GPU server |
|
||||
| The GPU server's log at `RUST_LOG=info` (phase A0, the same chain of 1, stderr captured) | 1 line: sp1-gpu-server 6.8.1 prints no spans and no timings, so the step costs below are read from the deferred-proof count, not from a profiler |
|
||||
| Aggregation with 1 deferred proof (the first block, no previous proof) against 2 (every chained block), the card mining | 7.9 s against 9.6, 9.6, 9.8 s: the second deferred proof costs 1.7 to 1.9 s under the miner |
|
||||
| The same, the miners paused (phase C, the same fixtures, 21:05:51Z) | 1.7 s against 2.1, 2.1, 2.2 s: the second deferred proof costs 0.4 to 0.5 s alone |
|
||||
| The shard proof of an empty shard | 7.4 to 7.8 s mining, 1.9 to 2.2 s alone |
|
||||
| A whole block (one empty shard plus its aggregation) | 17.1 to 17.3 s mining (end to end 67.4 s for 4 blocks), 4.1 s alone (16.4 s for 4) |
|
||||
| GPU utilisation over the phase (1-s `nvidia-smi` samples) | 93.9% mining (80 samples, the miner's), 15.8% alone (32 samples): the prover alone keeps the card busy a sixth of the time. Its work is short GPU bursts between CPU phases (the executor, the witness and recursion-program generation run on the CPU inside the server), and the miner's kernels fill the gaps |
|
||||
| GPU memory peak | 16,195 MiB mining (the miner's 3.4 GB resident), 14,483 MiB alone |
|
||||
| The slowdown by the miner, same fixtures, same host, 2 min apart | shards 3.6x, the first aggregation 4.6x, a chained aggregation 4.5x, a block 4.2x |
|
||||
| Setup per host process (client plus two key setups) | 13.0 to 15.7 s, mining or not |
|
||||
|
||||
Reading. An aggregation is three or four recursion steps on the card (the aggregator guest's one core shard, its lift, one deferred program per verified proof, the compose), each a burst of under half a second when the card is free. The chained aggregation's extra deferred proof is the only part that grows with the chain rule, 0.4 to 0.5 s alone. Everything else the 9.7 s holds is the miner: with the card at 94% from the lottery kernels, every prover burst waits for a time slice, and a 2.1-s aggregation becomes 9.7 s. The 4 October 2.2 s (two shards, no previous proof, the card to itself) and tonight's 1.7 s (one shard) and 2.1 s (one shard plus the previous proof) agree within the deferred count.
|
||||
|
||||
### Lever 2, batch and tree folds (estimate from the measured step costs; the statement is pinned, no guest was changed tonight)
|
||||
|
||||
A fold of K blocks' shard proofs plus the previous segment proof in ONE aggregator call would cost one core shard, one lift, K + 1 deferred programs and the compose tree in place of K chained aggregations. From the measured rows (alone: a 1-deferred aggregation 1.7 s, each further deferred proof 0.45 s; mining: 7.9 s and 1.8 s):
|
||||
|
||||
| Fold | Deferred proofs per call | Per block, card alone (estimate) | Per block, card mining (estimate) | Rule |
|
||||
|---|---|---|---|---|
|
||||
| chained, as pinned (measured) | 2 | 2.1 s | 9.7 s | one call per block |
|
||||
| batch of 4 | 5 | (1.7 + 4 x 0.45) / 4 = 0.9 s | (7.9 + 4 x 1.8) / 4 = 3.8 s | one call per 4 blocks |
|
||||
| batch of 8 | 9 | (1.7 + 8 x 0.45) / 8 = 0.7 s | (7.9 + 8 x 1.8) / 8 = 2.8 s | one call per 8 blocks |
|
||||
| tree of 4 (2 + 2, then the pair) | 3 per call, 3 calls | 3 x (1.7 + 2 x 0.45) / 4 = 1.9 s | 3 x (7.9 + 2 x 1.8) / 4 = 8.6 s | no gain over the chain: every call pays the fixed part |
|
||||
|
||||
Reading. A batch fold halves to quarters the per-block aggregation but changes the aggregator's statement (`AggInput` carries one block's shards and the guest asserts one block hash), so it is a new pinned guest and a new program id: a provers-off drain and a rollout (proving/README.md, pinned guests). It does not reach 3 s on a mining card by itself (2.8 s at K = 8 is on the line), and the shard proof beside it stays 7.4 s a block on a mining card. The lever that moves both is the card's other job, lever 4. A tree fold gains nothing here because the fixed part of a call (the core shard and the lift) dominates the per-proof part 4 to 1.
|
||||
|
||||
### Levers 3 and 4, two streams and the miner's kernels (job `agg-cost-pc2-2` and the re-run)
|
||||
|
||||
Job `agg-cost-pc2-2` (21:34:00Z to 21:49:22Z) ran with the 5090 idle throughout: job 1's `/api/resume` had left the worker off (below), so the rows that needed the miner (the batch-log2 curve, the two streams beside the miner, the time-slice policy, the chosen combination) are void and wait for a re-run; the idle rows are measured.
|
||||
|
||||
| What | Measured (job `agg-cost-pc2-2`, card idle) |
|
||||
|---|---|
|
||||
| Aggregate-only over job 1's four saved shard proofs (`--mode aggregate --proofs b1;b2;b3;b4 --parent ...`, one process, the same statement `0x3a995f24...` as the chain run), default knobs (phase B0, then C1) | 1.7, 2.0, 2.0, 2.0 s (1, 2, 2, 2 deferred proofs), 8.1 s for four; C1: 1.8, 2.1, 2.1, 2.1 s, 8.3 s |
|
||||
| The same with `SP1_WORKER_VERIFY_INTERMEDIATES=false` (phase B; the server inherits the host's environment, the knob printed in the `sp1 knobs` line) | 1.7, 2.0, 2.0, 2.0 s, 7.8 s for four: no gain (0.3 s over four, inside the run-to-run spread of 0.2 s). The knobs that change the recursion shape (`SP1_WORKER_MAX_COMPOSE_ARITY`, `MAX_REDUCE_ARITY`) were not tried: a different shape is a different recursion key set and the pinned verifier would refuse the proof |
|
||||
| A 4-deferred aggregation (block-344-shards4, four prototype shards of 6.75 M pgas, phase C2) | shards 42.8 s (10.7 s each, the 4 October 10.2 to 10.7 s), aggregation 2.4 s with 4 deferred proofs; GPU peak 28,402 MiB (the prototype shard's 28.3 GB), utilisation 27.7% over the phase. With 1.7 s at one deferred proof and 2.0 to 2.1 s at two: 0.25 s per further deferred proof alone, so a batch of 8 would cost about 3.5 s a call, 0.45 s a block (estimate, the pinned statement forbids it) |
|
||||
| Two host processes at once on the one card (phase G0: chains of 2 on disjoint blocks, started 2 s apart) | both connected to ONE sp1-gpu-server (the first process's child; the socket is per device, `/tmp/sp1-cuda-0.sock`): process 1 shard 2.2 and 3.5 s, aggregation 3.0 and 4.0 s (12.9 s for 2 blocks against 8.2 s alone); process 2 shard 3.3 s, aggregation 3.6 s, then its second block died with `CudaClientError: Failed to read the response: early eof` when process 1 finished and its server exited. GPU 24,911 MiB, utilisation 12.4% and 13.1%. Two streams through SP1 6.8.1's server are serialised on one socket and the second dies with the first: no throughput gain (3 blocks in 33 s against 4 in 16.4 s) and a failure mode; lever 3 is closed on this SP1 version |
|
||||
| Job 3 (`agg-cost-pc2-3`, 22:41:15Z, app 0.3.10, the same script with the socket rule and a card switch): phase A, the app's 5090 miner at 117.0 MH/s mean (n 3, STATUS lines 22:44:45Z to 22:46:11Z), four fresh live blocks 90896..90899 | shards 8.0, 7.8, 7.6, 7.8 s; aggregations 8.0 s (1 deferred), 10.0, 10.0, 10.0 s (2 deferred); 69.5 s for four, 17.8 s a block; GPU 93.8%, peak 16,245 MiB: the job-1 baseline reproduced 100 min later on other blocks |
|
||||
| Job 3's own-miner phases | void: the state reads came back empty (the class below), the card switch did nothing, phase D launched my miner beside the app's (the app's dropped to 62.2 MH/s, mine read 60.6 MH/s), then PC 2's app restarted at 23:03:30Z and the job died with it; no curve point |
|
||||
| The GPU time-slice policy (`nvidia-smi compute-policy --set-timeslice`, the restore job `agg-cost-restore-1`, 23:16:53Z) | "Not Supported" on PC 2 (RTX 5090, driver 13.3, the Windows nvidia-smi, not elevated): the lever is closed on this driver; an elevated try is not worth a slot, the error is the driver's, not a permission's |
|
||||
| The own-miner phases of job 2 | void: no 5090 miner was running to copy the command line from (the worker off since 21:25Z) |
|
||||
|
||||
The curve, job `agg-cost-pc2-6` (01:12:09Z to 01:24:14Z, app 0.3.11, PC 2 to itself; every phase closed before the next job landed on PC 2 at 01:24:21Z). The app's 5090 miner switched off through `/api/cards` (the keys from `settings.json`; the worker was still alive after 120 s, `/api/pause` as the fallback stopped it in 5 s), then the job's OWN miner on the 5090 with the app's command line (`igneum-miner mine ... --worker igneum-worker-cuda.exe --identities 8 --worker-args "--device 0 --pack packs\devnet --race off [--batch-log2 B]"`, the base variant, its STATUS line every 10 s), the same four live blocks 96556..96559 (one empty shard each) proven by `--mode chain` under it, the miner's rate from its own `now=` field (the first two lines skipped). `--batch-log2 B` sets the worker's nonces per kernel launch (2^B; 22 is the worker's default, 4,194,304 nonces, about 35 ms a launch at 120 MH/s; `proto-cuda/nvrtc/worker.cpp`).
|
||||
|
||||
| batch-log2 | Shard proof (4, s) | Aggregation (1 deferred, then 2) (s) | A block (s) | GPU util. (%) | GPU peak (MiB) | Own miner (MH/s wall, n) | Against the card alone (4.1 s a block) |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 22 (the default), phase D | 8.1, 7.8, 7.9, 7.8 | 8.4; 10.3, 10.0, 10.4 | 18.1 | 95.5 | 16,580 | 103.9 (9) | 4.4x |
|
||||
| 20, E20 | 8.1, 7.8, 7.8, 7.8 | 8.4; 10.3, 10.1, 10.1 | 18.0 | 94.9 | 16,461 | 103.7 (8) | 4.4x |
|
||||
| 18, E18 | 7.0, 6.7, 6.7, 6.7 | 7.2; 8.9, 8.8, 8.8 | 15.6 | 91.5 | 16,487 | 99.3 (7), minus 4.4% | 3.8x |
|
||||
| 16, E16 | 5.1, 4.9, 4.9, 4.9 | 5.0; 6.1, 6.2, 6.2 | 11.1 | 85.3 | 16,519 | 83.8 (6), minus 19% | 2.7x |
|
||||
| 16 again, phase H (the job's own choice: the shortest chain) | 5.0, 4.9, 4.8, 4.9 | 4.9; 6.1, 6.2, 6.2 | 11.1 | 85.7 | 16,487 | 84.0 (6) | 2.7x |
|
||||
|
||||
Reading. Between 2^22 and 2^20 nothing moves: the card's time-slice scheduler alternates the two contexts whatever the kernel length above a few milliseconds. From 2^18 down the miner's launches get short enough (about 2 ms at 2^18, 0.5 ms at 2^16) that the prover's bursts find the card sooner, and the miner pays in launch overhead and idle gaps: at 2^16 the prover runs 1.6x faster (18.1 to 11.1 s a block, the chained aggregation 10.2 to 6.2 s) for a fifth of the hash rate, and it is still 2.7x slower than on a card to itself. The trade is about 1 MH/s per 0.37 s of block time at the 2^16 point, and the 3-s aggregation and the 1.5x slowdown are not reachable on a mining card by the kernel length; a 2^14 point (approximate, extrapolated) would be about 8 s a block at about 65 MH/s. The phase E0 (a 4-deferred aggregation under the miner) failed in 0.1 s: its proof paths pointed at `/` where job 1 had left its shard proofs, but job 2's block-344 proofs sit in job 2's own folder (`$JOB` was exported from job 2 on); the 4-deferred cost under the miner stays an estimate (lever 2 above). The app's own 5090 miner ran at 117 MH/s (job 3, 22:44Z) and 110 to 129 MH/s (its STATUS lines at 01:10Z) with the prover beside it, against my miner's 104 MH/s at the default batch: my miner runs the base variant with `--race off` (no tuning file on PC 2), so the curve's rates are relative to each other, not to the app's.
|
||||
|
||||
### Lever 5, the host side under WSL2 (what the chain-mode numbers leave out)
|
||||
|
||||
| What | Measured |
|
||||
|---|---|
|
||||
| The export (`igneum_exportSegments` 0..tip, 75 to 77 MB over curl.exe to a file on `C:`) | 1.1 to 1.5 s |
|
||||
| The cut (`igneum-prove-export` replaying from genesis, then `--mode native`), four blocks | 18 s for four including the native checks (21:01:28Z to 21:01:46Z), about 4 s a block; the export's file sits on `/mnt/c` |
|
||||
| The key setup per host process | 13.0 to 15.7 s on PC 2 (8.0 to 8.5 s on the Mac CPU): `--mode chain` and `--mode aggregate` pay it once per process, the app's loop pays it per shard |
|
||||
| The proof file write through the WSL2 bridge | the 4 October entry ("shard proving on the RTX 5090"): 24 min of unbuffered `save` across `/mnt/c`, fixed by the 4 MB buffer; tonight `--save-shards` wrote the four 1.27 MB proofs inside the chain phase with no visible gap (the A phase's 80.4 s wall against 67.4 s of proving plus 13.0 s of setup) |
|
||||
| Native Linux | not measured: no native Linux machine with an NVIDIA card exists in the project tonight, and the 4 October numbers were also WSL2 (Ubuntu 24.04 under PC 2's Windows). The WSL2 cost inside a `prove()` call is not separable from here; the host-side pieces above are what a native box would also skip or keep |
|
||||
|
||||
### What went wrong, measured
|
||||
|
||||
| What | Fixed |
|
||||
|---|---|
|
||||
| Job 1's per-phase command ran with `$JOB` empty (the bash variables of `vars.sh` were set, not exported, and the command runs in a child bash): `--out /results-A.json`, the saved shard proofs in `/` on the WSL root, so the aggregate-only phases B0, B, C1 and the prototype-shard phase C2 failed in 0.0 s ("No such file") | `export` in `vars.sh`; job 2 reads the proofs from `/` |
|
||||
| Job 1's own-miner phases launched the iGPU miner (the first `igneum-miner mine` process matched; the 5090's is the second) and `if (StartMiner ...)` was always true (PowerShell: a function's emitted RESULT strings are part of its output), so D and E ran with the 5090 idle and the AMD iGPU at 3.4 MH/s: three more idle replicates of the chain (2.0 to 2.2 s shards, 1.8 and 2.2 s aggregations), no curve | the miner matched on `igneum-worker-cuda`, the outcome in a script-scope flag, `--race off` for the own miner (no tuning file on PC 2; a race costs up to 120 s a start) |
|
||||
| Job 1's `/api/resume` at 21:25:11Z answered ok and the 5090 miner stayed off (card state `off`, hash 0.0, 1,760 MiB on the card) until the 0.3.10 restart; job 2 waited its full 600 s for a hash rate and ran its mining phases void | the restore job `tools/proving-v1/pc2-agg-cost-restore.ps1` also posts `/api/start`; the Counter ASIC coordinator opened a task chip for the resume defect |
|
||||
| Jobs 3 and 4 (`agg-cost-pc2-3` 22:41Z on app 0.3.10, `agg-cost-pc2-4` 00:18Z on 0.3.11): every `/api/state` read came back as the two bytes `{}` (job 4's raw-body print: `raw_len=2`; the same reads gave the full state on 0.3.9 at 21:01Z and the AMD agent saw the empty reply at 22:22Z), so the card switch found no card, the app's 5090 miner kept mining, and job 3 ran a second miner beside it (two miners at about 60 MH/s each) while job 4's double-mining guard voided its own-miner phases. The class is the app's, not the reader's: `state_json()` (engine.rs:180) does `serde_json::to_value(st).unwrap_or(json!({}))`, and the value that fails is `ProvingState.paid_wei: u128` (serde_json 1.0.151 refuses a u128 over u64::MAX, 18.45 IGN; the proving-v1 agent's diagnosis): a paid shard averages 1.23 IGN, so the reply empties about 15 paid shards after every app start and comes back at the next restart, which matches the times (full at 21:01Z with paid_wei 0, empty from 22:22Z after the prover had paid from 22:02Z). Fixed on the app branch proving-v1 at 6714a45 (paid_wei as a decimal string, the error logged, an `{"error":...}` reply on any future failure) | job 5 reads the card keys from the app's `settings.json` (`cards`: key to enabled and identities), restores the 5090's 8 identities first (the restore job of 23:16:53Z had set 2: its parser read the next card's value), refuses before any pause when it cannot name the card, waits on the CUDA worker process count for the card to stop, and checks the worker is back at the end |
|
||||
| Job 5 (`agg-cost-pc2-5`, 01:10:44Z) failed at PowerShell's parse in 1 s: `$RestoreIdentities:` inside a double-quoted string (a drive-qualified variable); no card or miner touched | `${RestoreIdentities}:`; the other `$name:` shapes are inside single-quoted bash here-strings |
|
||||
| Job 6's identities step found `settings.json` already at 8 identities under the active key `nvidia:0:NVIDIA GeForce RTX 5090` (a stale key `nvidia:NVIDIA GeForce RTX 5090` carries 2), so no change was sent; job 6's `/api/cards` with the 5090 disabled answered ok but the worker ran on for 120 s, `/api/pause` stopped it in 5 s, and at the end `/api/resume` brought it back in 5 s on 0.3.11 | the card switch keeps the pause as its fallback; the resume path works on 0.3.11 |
|
||||
| PC 2 ran three jobs at once from 01:24Z (`run-prover-on-pc2-20261006` at 01:24:21Z, the ledger suites build at 01:26:15Z, while agg-cost-pc2-6's closing report was still being uploaded): the app does not serialise jobs, "one job per machine at a time" holds only by the coordinator's word; job 6 had closed at 01:24:14Z, so its rows are clean | nothing of mine to fix; a rule for the job runner |
|
||||
| The make-package gate ran the exporter's side files (`block-N.json.node-plan.json`) as fixtures and failed; its execute step took the exclusive `measure` lock for a cycle count and queued 25 min behind a packbench run | the glob skips `.node-plan.json`; the execute step runs under the `run` lock (a count, not a time) |
|
||||
|
||||
### 6 October 2026, 07:12Z to 07:17Z, the host's chain mode with --save-shards records and --prev, on the Mac's CPU
|
||||
|
||||
`tools/lock/with-lock.sh run`, `SP1_PROVER=cpu igneum-prove-host --mode chain --chain proving/fixtures/chain/block-81046.json,block-81047.json --save-shards --out chain-a.json`, then `--chain block-81048.json --save-shards --prev segment-81047-aggregated.bin --out chain-b.json` (the app branch at ce8f34a, Apple M5 Max, CPU prover). The flags the app's segment path needs, before PC 2 (approximate figures: a CPU run, one sample each):
|
||||
|
||||
| Step | Value |
|
||||
|---|---|
|
||||
| Shard proof, CPU, empty block | 34.7 s and 36.3 s |
|
||||
| Aggregation, CPU, 1 then 2 deferred proofs | 39.1 s, 50.8 s |
|
||||
| Chain of 2, end to end | 160.9 s |
|
||||
| Per-shard records written | 2 (number, block_hash, shard, statement, proof_sha256, proof_bytes 1,272,897, proof_file, prove_seconds) |
|
||||
| `--prev` run: base_chain_len, final chain_len | 2, 3 (the chain continued; a wrong previous proof is refused by number and parent hash) |
|
||||
|
||||
### 6 October 2026, 07:52Z to 08:24Z, the segment-aligned prover beside the miner on PC 2's RTX 5090 (job `segments-pc2-pv1c`)
|
||||
|
||||
`tools/proving-v1/pc2-segments.ps1` (app branch 330207d; the host from the package `igneum-prove-wsl2-segal`, built on PC 2 in 7 s warm to `/opt/igneum-segal`, pinned guests unchanged); the app's own prover OFF for the run through `/api/prove`, ON again at the end; the app's miner running (8 identities, batch-log2 22); `SP1_PROVER=cuda`, the stock 6.8.1 GPU server; a 1-s nvidia-smi sampler under every chain. Payouts read on node 1 (read-only, `igneum_getProofRecords` per block at 08:30Z). The miner's rate from the app's uploaded log (`status: ... MH/s` every 30 s, run win-1ccfe586-20261005-235130).
|
||||
|
||||
| Figure | Value | Note |
|
||||
|---|---|---|
|
||||
| Segments claimed in 30 min | 9 (114470, 114654, 114862, 115022, 115198, 115366, 115542, 115710, 115870) | one every 210 s; 32.1 min of loop |
|
||||
| Candidates per pass | 32 to 38 whole segments inside the margin | margin 580 to 589 DAA at claim |
|
||||
| Export (the chain to the segment's last block) | 99.6 to 100.6 MB in 1.4 to 1.6 s | once per segment |
|
||||
| Cut (8 fixtures, the exporter) | 45.1 to 46.0 s | the exporter replays from genesis per block; the next lever |
|
||||
| Chain run wall (8 shards, 8 aggregations, one key setup) | 159.7 to 160.6 s | host `--mode chain --save-shards` |
|
||||
| Shard proofs, 8 per segment | 63.0 to 63.5 s (7.9 s a shard) | empty blocks |
|
||||
| Aggregation, 8 chained | 80.4 to 81.1 s (10.1 s a block) | the fixed cost per block beside the miner |
|
||||
| End to end per segment (export, cut, chain, sign, submit) | 210.0 to 211.2 s | |
|
||||
| GPU memory peak during a chain | 16,484 to 17,573 MiB (miner resident) | the 24 GB tier's gate holds |
|
||||
| GPU utilisation during a chain | 94.9 to 95.3% | |
|
||||
| Shard records accepted | 72 of 72 | 8 per segment |
|
||||
| Shard records paid on chain | 72 of 72 | 0.905 to 2.719 IGN a shard (90% of the credit); carried 180 to 226 blocks after the block |
|
||||
| Segment records accepted | 0 of 9 | every one refused: "does not chain to segment N-8..N-1 (chain_len 8), which is pending until DAA ..." |
|
||||
| Miner alone (the app's prover off), 07:25 to 07:51Z | 117.86 MH/s mean (n=52) | min 46.37 is the switch-off dip at 07:22Z |
|
||||
| Miner beside the segment prover, 07:55 to 08:24Z | 104.90 MH/s mean (n=58, min 98.39, max 119.24) | 12.96 MH/s = 11.0% of the miner, at 95% GPU utilisation from the prover |
|
||||
| The 0.3.11 prover as shipped beside the miner (5 October row) | 5.0 MH/s = 4.0% | one shard per 46 s; this run proves 8 shards per 210 s, 2.8x the shards |
|
||||
| Node 1's v1 window at 08:24Z | pending 59, proven 0, unproven 16, paid segments 0 | unchanged by the run: the chain rule |
|
||||
|
||||
What the refusal is (the fork, `igneum/exec/src/proving.rs` `check_segment_record`): a fresh record (chain_len = N) is valid only when the previous segment is UNPROVEN at the carrier, and the record's own deadline is the previous segment's deadline plus one segment length in DAA, so a fresh record is valid for 8 DAA (about 8 s) per segment and must be carried inside them. With one prover every previous segment is pending at proof time. Fixed on the fork branch behind `proving_v1_fresh_rule_daa` (0f0dda95): from the switch a fresh record is valid whenever the previous segment is not proven; the app holds a refused record and offers it again every pass until the deadline (272b025).
|
||||
|
||||
Run b (`segments-pc2-pv1b`, 07:20Z to 07:51Z) claimed nothing in 88 passes: the driver's segment keys were doubles against int64 hashtable keys (fixed in 330207d); its 30 minutes are the miner-alone baseline above.
|
||||
|
||||
### 6 October 2026, 08:26Z to 08:35Z, the fast-time harness on the fresh-record rule (Mac, `tools/lock/with-lock.sh run`)
|
||||
|
||||
`IGNEUM_PV1_BIN=vendor/igneum-node/target-pv1/release node tools/proving-v1/net.mjs --segment 8 --unproven 10 [--fresh-rule 0]` (fork 0f0dda95, 3 nodes at 60x, ports 29950+):
|
||||
|
||||
| Case | Checks | Time |
|
||||
|---|---|---|
|
||||
| The rule as shipped (no switch): fresh refused while the previous segment is pending (known-failed), accepted after it is unproven | 22 passed | 166.2 s |
|
||||
| `--fresh-rule 0`: fresh accepted while the previous segment is pending, `freshAdmissible` true, still refused after a proven one, the second offer a duplicate ("segment already paid") | 23 passed | 139.9 s |
|
||||
|
||||
|
|
|
|||
84
docs/plans/morning-2026-10-06.md
Normal file
84
docs/plans/morning-2026-10-06.md
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
# Morning summary, 6 October 2026
|
||||
|
||||
Written for Josh at 08:40 UTC. Every number is in `docs/bench-log.md` or the named plan with the command that produced it. Failures are listed with the passes.
|
||||
|
||||
## The headline
|
||||
|
||||
Everything on the overnight list landed. The one thing that could have gone wrong, switching the live chain to program class v3, held.
|
||||
|
||||
| Piece | State | Number |
|
||||
|---|---|---|
|
||||
| 0.3.11 (class v3 + proving v1) | On master 630da6b, three master CI runs green, rolled out to the Mac, PC 2, the seed, both hand nodes; PC 1 took it at 07:00 | Live manifest 0.3.11, nine-field override, digest 0139ab9d |
|
||||
| Counter ASIC 2.0 crossing | Crossed at 03:51:42 UTC, watcher verdict PASS | 58.7 blocks a minute before, 59.2 after; three hourly swaps since, no pause, 0 refusals |
|
||||
| 12 GB proving | Floor broken on a patched SP1 server (`prover-floor` bcc6d68); real card lands today, test on PC 2 | Proves alone: 10.3 GB, 5.7 s a shard. Mines AND proves as a core-only prover handing its proof to a big-card aggregator: 8.1 GB beside the miner, 19.8 s a shard (measured 09:15 on the 5090's allocation). 16 GB mines and proves compressed (12.9 GB, 17.4 s). 8 GB stays out |
|
||||
| On-die recompute chip | Emulated on the 5090's own L2 (`ledger-pc2` 564acab) | 0.256x honest, 5.1x worse per joule |
|
||||
| Ledger | Rounds 2 and 3 closed (`fud-close` d16bc3b, `ledger-rebase` abb08a5) | 47 consequence rows, 42 closed or taken, 14 decisions |
|
||||
| Branches ready for later cuts | pool-v0, rig-install, repro-bench b776199, ember-tune 9a6469f, ota-k2, asic-history, proving-methods | measured where PC 2 allowed |
|
||||
|
||||
## Since you got up (07:00 to 08:40 UTC)
|
||||
|
||||
| What | State |
|
||||
|---|---|
|
||||
| PC 1 | Back on 0.3.11 at 07:00, 5090 and 9070 XT mining; integrated card off. The update needed your click because the app's hourly rollout slot had not come; fixed as a catch-up rule (`update-catchup` 2207cd7, 0.3.12) |
|
||||
| Mac | Mining paused through the app (persists); its node runs |
|
||||
| Chain | 19 miner ids, 225 MH/s on the two PCs |
|
||||
| Zero proven segments | Solved: the prover claims a whole segment and proves its 8 blocks in order (9 segments in 30 minutes on PC 2 beside the miner, 72 of 72 shard records paid, 11 percent hash cost). The segment record itself needs a consensus switch on the node fork (`proving_v1_fresh_rule_daa`), folded into 0.3.12 |
|
||||
| Ember Tune re-run on PC 1 | Failed at 07:56 with no rows: the playbook wrote the test engine's settings with a byte-order mark, the engine parsed defaults, sat idle 35 minutes. Nothing was set on either card; the app restored its miners by itself. Fixed (8273494, watchdog 1e9550e, CI check). Re-run `ember-tune-pc1-3` needs one more click when you are back |
|
||||
| GPU list order | Cards ordered by performance, integrated last (`card-order` ffb2bfa, 0.3.12) |
|
||||
| Counter ASIC 3.0 | Running, all seven items plus a new item 8. See below |
|
||||
| 0.3.12 | Being prepared: the app items plus the node fork with the segment switch; stops at the publish gate for your go |
|
||||
|
||||
## Counter ASIC 3.0 so far
|
||||
|
||||
The finding that matters: the chip that wins is not the clever recompute chip 2.0 priced. It is a stored-dataset chip, the whole dataset in DRAM, a 28 nm memory-controller die doing dependent reads.
|
||||
|
||||
| Attacker | Per chip vs 5090 | Per joule vs 5090 | Source |
|
||||
|---|---|---|---|
|
||||
| On-die recompute chip (2.0's model) | 0.92x with the 3x allowance | 1.86x | chip-model-v3 |
|
||||
| Same, emulated on the 5090's L2 | 0.256x | 0.2x | M16 inline bench |
|
||||
| Stored-dataset chip, GDDR7 | 1.22x | 5.1x | item 1 |
|
||||
| Stored-dataset chip, HBM3 | about 1.2x | 7.5x to 9.2x | item 1 |
|
||||
| Ethash precedent (E3, A10) | | 2.1x to 4.8x | history rows 3, 4 |
|
||||
| Stored-dataset chip with the latency shadow filled (item 8, N = 100,000, parity cores) | | 2.7x vs 5090, 1.4x vs M5 Max | item 8 Mac rows |
|
||||
|
||||
Why: at the hash the 5090 spends about 55 W on memory and the rest keeping a GPU alive at 0.15 percent of its integer budget. The lever is RandomX's lever: make the hash use the rest of the chip. The 5090 can hide about 330,000 operations per hash behind its 128 reads; today it hides 512. Item 8 measures that fill: on the Mac it costs 1.5 percent of rate at 100,000 ops, the verifier barely notices, and the chip's edge drops from 5x to 2.7x. The 5090's rows are queued on PC 2. The deciding number is the chip core's energy per op against a GPU's ALU, which item 8 is pricing.
|
||||
|
||||
**Closed at 08:50.** One class v4 candidate, measured on the hash's own numbers and ready for its six-gate run on your word: mixer x8 plus 100,000 operations of program work per hash. The 5090 loses 0.2 percent of rate and the M5 Max 1.5 percent; the verifier adds 0.17 ms per warp; bit-exact on Metal, CUDA, Apple OpenCL and the CPU emulation. The chip must then carry a 14,000-lane ALU array, and its per-joule edge over the 5090 falls from 5.6x to 2.1x at a core as efficient as the GPU's, 1.5x at a realistic one. Your test as a number: the chip crosses 2x only if its datapath spends under half the energy per op that a GPU does. The cost per tier: a 5090 draws 431 W instead of 350 for the same blocks (a rig pays about 23 percent more electricity), the M5 Max 37 W instead of 21, a pool user sees nothing. The 9070 XT and 4060-class rows are owed, the AMD ones because PC 1 was left alone.
|
||||
|
||||
Other items: item 2 (per-day random derivation) works bit-exact at no hash cost and drops the recompute chip to 0.29x to 0.43x, but at full size its CPU verifier is over the 10 ms gate on an old core; it goes in as a reserve, the half-size draw passes, and the class v4 candidate is the pairing of derivation class and program length under one verifier gate. Item 3: cryptanalysis brief and budget line, USD 80,000 to 160,000, one firm plus one academic group, verdict GO to commission, nobody contacted. Items 4 and 5: share-pattern detector in the observer (fires on a fabricated fixed design, quiet on the devnet), issuance trigger at USD 20,000 a day, FPGA soft overlay 0.3x to 0.4x a 5090 per watt, layer 9 ranked above layer 7. Items 6 and 7 in flight.
|
||||
|
||||
## Decisions for you
|
||||
|
||||
The full list with recommendations is in `consequences-decisions.md` (14) and the 3.0 status file. The ones that bite first:
|
||||
|
||||
0. **Run the six gates on the class v4 candidate**, or wait for the 9070 XT and 4060-class rows first (3.0 status, decision 2).
|
||||
1. **The public claim "under 2x".** True of the recompute chip per chip, false of the stored-dataset chip per joule. Two re-wordings are drafted in the 3.0 status file; nothing on the site changed. Pick one before any public push.
|
||||
2. **The segment rule.** Answered at 08:50: it is a consensus rule (as shipped a fresh segment record is valid for an 8-second window). The fix is on the node fork behind a new switch `proving_v1_fresh_rule_daa`, so 0.3.12 carries the node and goes out as a two-manifest publish with the switch at tip + 14,400. Measured on PC 2 beside the miner: 9 whole segments in 30 minutes, 72 of 72 shard records paid, 11 percent of hash rate.
|
||||
3. **0.3.12 go.** The state-reply fix, the update catch-up, the card order, Ember Tune and its guards, plus the node fork with the segment switch (two-manifest publish, switch at tip + 14,400). No prompt on any machine.
|
||||
4. **The 12 GB card.** Into PC 2 when it lands (PC 1 has no prover toolchain); the playbooks are written. Core-only provers need a pool-protocol change (a core hand-off format only aggregators accept, the compressor's credit, the prover's signature over the proof hash); nothing on chain changes. Decide whether that goes into the pool spec now.
|
||||
5. **Cryptanalysis budget** (D11 and item 3), **growth mapping (b)** (D4), **a release-tag convention**, **the bounty only once escrowed**.
|
||||
|
||||
## What went wrong, plainly
|
||||
|
||||
- Epoch-34 pack outage, 18:23 UTC, 56 minutes of both PCs down: a pack-attempt bug in the worker loader. Hot fix shipped by job; class fix in 0.3.10.
|
||||
- GitHub Actions outage forced a PC-built 0.3.10. 0.3.11's first CI run then failed in the census crate nobody updated for class v3; fixed (2a62735), rerun green.
|
||||
- Credits ran out at 19:58 UTC and killed three agents; resumed.
|
||||
- The 9070 XT dropped off the bus four times; the reading was wrong, the relay's "PC1" is PC 2.
|
||||
- A 2.2x verifier regression on the mixer branch, caught before publish.
|
||||
- Two Mac-only v3 outages caught by the Metal gate before publish.
|
||||
- "12 GB proves" was false on the shipped prover; the floor was SP1's server code; patched.
|
||||
- A job quit PC 1's app at 22:31 UTC (a second engine ran the urgent updater, whose installer sent quit). Rule written, CI gate added; PC 1 stayed down all night. Fixed in e600e63.
|
||||
- Three dark proving windows on PC 2, all the same state-reply bug (paid_wei above u64::MAX empties the reply); 0.3.12's first item.
|
||||
- The Mac miner lost its node subscription for 11 minutes after a node restart (C43, 0.3.12).
|
||||
- The 0.3.11 update on PC 1 waited on the hourly slot and needed your click; catch-up rule in 0.3.12.
|
||||
- Ember's first real run burned 35 minutes and your prompt on a byte-order mark; fixed with a watchdog and a CI check.
|
||||
- The 07:45 summary task never fired on its own and its manual run stalled on a tool prompt; this document was written by hand.
|
||||
|
||||
## Today's hands list
|
||||
|
||||
1. 0.3.12 go when the shipper reports it green.
|
||||
2. One click for the Ember re-run on PC 1.
|
||||
3. The 3060 into PC 2; the playbook runs the same fixture as the sweeps.
|
||||
4. 16:00 UTC: the fee-switch check (every prover on 0.3.11 before H = 210,000 at about 19:15 UTC).
|
||||
5. USB copies of the key backup (10:00 reminder).
|
||||
6. The decisions above.
|
||||
|
|
@ -132,8 +132,8 @@ The GPU server of SP1 6.8.1 sets the memory, not the shard: a floor of 13.9 GB f
|
|||
|---|---|---|---|
|
||||
| 32 GB (RTX 5090) | the prototype shard, 28.3 GB, 10.8 s; the v1 shard 20.4 GB, 4.3 s | the prototype shard 30.1 GB, 33 s; the v1 shard 22.2 GB, 13.2 s | on, mine and prove, today |
|
||||
| 24 GB (RTX 4090, 3090) | the v1 shard 20.4 GB; the prototype shard does NOT fit (28.3 GB) | the v1 shard 22.2 GB measured on the 5090's allocation (2.3 GB spare on a 24 GB card; approximate for the card itself) | on, mine and prove, with the line "until the devnet's fee switch its shards are the prototype size, which needs 32 GB, so this card proves from the switch on" |
|
||||
| 16 GB (RTX 5080, 4080) | the shipped server: an empty shard only (13.9 GB); the patched server v3 b37defef at threshold 2^27: the v1 shard 12,915 MiB and 4.3 s, the prototype shard 13,459 MiB and 16.8 s (measured by the prover-floor agent on the 5090's allocation, job `floor-sweep-3`, 00:13 to 00:17Z 6 October; 2^27 + 2^26 gives 16,115 MiB, over the card) | the shipped server: nothing (15.7 GB for an empty shard); the patched server: about 14.7 GB at 2^27 beside the miner (approximate: the measured 1.8 GB the miner adds; the pair is the agent's sweep 4) | off on the shipped server, with the line; on once the patched server ships (the packaging row below) and the pair is measured |
|
||||
| 12 GB (RTX 3060, 4070) | the shipped server: nothing (the floor is 13.9 GB, and the server refuses the card outright); the patched server v3 b37defef at threshold 2^26 (`SP1_GPU_ELEMENT_THRESHOLD=67108864`, the 12 GB profile): **the v1 shard 10,291 MiB and 5.7 s, an empty shard 9,971 MiB and 3.3 s**, the card's 2,089 MiB idle inside the peak and the server's own working set about 8.2 GB (6,535 MiB after Setup), so a 12 GB card proves alone with about 3 GB over it (measured by the prover-floor agent on the 5090's allocation, `floor-sweep-3`; the on-order RTX 3060 run is pending) | the pair (the miner's 1.7 to 1.8 GB and 3x beside it) is the agent's sweep 4; under 9.0 GB mine-and-prove is not yet shown | off on the shipped server; "proves alone" on the patched one once it ships (the packaging row below), mine-and-prove after the pair. Josh's "make sure we can prove on 12 GB cards" is answered on the 5090's allocation and OPEN on the card itself: the prover-floor agent (branch prover-floor, 5 October night) read SP1 v6.8.1's GPU server source (`sp1-gpu/crates/prover_components/src/builder.rs` lines 35 to 39): it reads the card's memory, adds 4 and panics under 24 ("Unsupported GPU memory ... must be at least 24GB"), and builds its core (ELEMENT_THRESHOLD 2^28 + 2^27 elements + 2^21), recursion (2^27), shrink (2^25) and wrap (85 M element) provers at Setup whatever the mode, which is the 13.9 GB floor; no knob reaches them, so the fix is a server rebuilt from source on PC 2 (WSL2, nvcc 12.8, CUDA_ARCHS=120) with those sizes cut, measured on the same fixtures and recipe as the curve above (D2 carries the curve) |
|
||||
| 16 GB (RTX 5080, 4080) | the shipped server: an empty shard only (13.9 GB); the patched server v3 b37defef at threshold 2^27: the v1 shard 12,915 MiB and 4.3 s, the prototype shard 13,459 MiB and 16.8 s (measured by the prover-floor agent on the 5090's allocation, job `floor-sweep-3`, 00:13 to 00:17Z 6 October; 2^27 + 2^26 gives 16,115 MiB, over the card) | the shipped server: nothing (15.7 GB for an empty shard); the patched server at 2^27 beside the miner (the 5090 mining at 95%, 338 W, same card; `floor-sweep-4`, 00:35 to 00:38Z): the v1 shard 14,786 MiB total with the miner's 3,833 MiB resident inside it, the server's own 10,953 MiB, 17.4 s a shard; on a 16 GB card that is 10.95 GB server + 1.7 GB miner = 12.7 GB plus the display, under the 15.0 GB line | off on the shipped server, with the line; on (mines and proves, 17 s a v1 shard, 4.3x the alone time) once the patched server ships (the packaging row below) |
|
||||
| 12 GB (RTX 3060, 4070) | the shipped server: nothing (the floor is 13.9 GB, and the server refuses the card outright); the patched server v3 b37defef at threshold 2^26 (`SP1_GPU_ELEMENT_THRESHOLD=67108864`, the 12 GB profile): **the v1 shard 10,291 MiB and 5.7 s, an empty shard 9,971 MiB and 3.3 s**, the card's 2,089 MiB idle inside the peak and the server's own working set about 8.2 GB (6,535 MiB after Setup), so a 12 GB card proves alone with about 3 GB over it (measured by the prover-floor agent on the 5090's allocation, `floor-sweep-3`; the on-order RTX 3060 run is pending) | measured beside the miner (`floor-sweep-4`): at 2^26 the v1 shard 12,066 MiB total with the miner's 3,833 MiB inside, the server's own 8,233 MiB, 24.4 s; the empty shard 11,586 MiB, 13.0 s; 2^25 gains nothing (12,066 MiB, 43.5 s). On a 12 GB card that is 8.2 GB server + 1.7 GB miner = 9.9 GB before the display, over the 9.0 GB line Josh set, so mine-and-prove on 12 GB is NOT claimed | off on the shipped server; "proves alone" on the patched one once it ships (the packaging row below); mine-and-prove stays off on 12 GB (9.9 GB plus the display, over the 9.0 GB line). The public gate stays "12 GB proves; 16 GB mines and proves", both on the patched server, both pending a run on the card itself. Josh's "make sure we can prove on 12 GB cards" is answered on the 5090's allocation and OPEN on the card itself: the prover-floor agent (branch prover-floor, 5 October night) read SP1 v6.8.1's GPU server source (`sp1-gpu/crates/prover_components/src/builder.rs` lines 35 to 39): it reads the card's memory, adds 4 and panics under 24 ("Unsupported GPU memory ... must be at least 24GB"), and builds its core (ELEMENT_THRESHOLD 2^28 + 2^27 elements + 2^21), recursion (2^27), shrink (2^25) and wrap (85 M element) provers at Setup whatever the mode, which is the 13.9 GB floor; no knob reaches them, so the fix is a server rebuilt from source on PC 2 (WSL2, nvcc 12.8, CUDA_ARCHS=120) with those sizes cut, measured on the same fixtures and recipe as the curve above (D2 carries the curve) |
|
||||
| under 12 GB | nothing | nothing | off, mine only |
|
||||
| AMD-only and Apple machines | nothing on the GPU: no zkVM proves on an AMD GPU today (`docs/analysis/amd-proving.md`, branch amd-prove); the CPU prover is about 5 minutes a shard at a 30 GB RSS whatever the shard size (PC 1, bench-log "the SP1 CPU prover on PC 1") | | off, "mines and does not prove"; the only non-NVIDIA path with a shipped backend is RISC Zero's Metal prover behind the `ProofSystem` seam (a second guest and pinned id, a verifier for both formats, no shared aggregation): an open item, not 0.3.11 |
|
||||
|
||||
|
|
@ -143,10 +143,81 @@ The aggregation-cost agent's first rows (branch agg-cost, 5 October 2026 night,
|
|||
|
||||
The re-plans of block 344 at 2.25 M and 4.5 M pgas peak at 28.3 to 28.4 GB alone (the server's buffers step up between 4.7 M and 20 M cycles and are flat to 60 M), so no shard size between the v1 budget and the prototype one changes a tier; with the miner the adopted shard proves 3.1x slower (13.2 s against 4.2 s) and the chained aggregation 9.7 s against 2.5 s: a mining 24 GB card delivers one adopted-size shard plus one aggregation in about 23 s, inside T by 25x.
|
||||
|
||||
## Segment-aligned proving (6 October 2026, Josh: "find a way to solve this")
|
||||
|
||||
**The fault was the work order, not the rules.** The shipped loop took the newest open shard each pass (`prover::choose`), so one prover scattered one block in about 45 across the segment grid and no segment ever had all its blocks proven: pending 56, proven 0, unproven 20 at 06:28Z. No consensus parameter moves.
|
||||
|
||||
**The change (app branch, commit ce8f34a, app and host):**
|
||||
|
||||
| Part | What it does | Where |
|
||||
|---|---|---|
|
||||
| Whole-segment claiming | the work list (lookback 600, the record window) grouped into whole untouched segments: every block present, every shard open (past its 10-DAA exclusive window), unpaid, not in our pool | `app/igneum-app/src/segments.rs` `whole_segments` |
|
||||
| The choice | candidates inside their deadline by a margin (240 DAA, or 1.5x the last segment's wall time), ranked by FNV-1a of (first block, this prover's key hash): deterministic per prover, different between provers, so several provers spread over the candidates with no coordinator; an attempted segment is not retried | `segments::candidates`, `rank`, `need_daa` |
|
||||
| The statement check | for the best three: executed and pending; fresh only when the previous segment is not paid and no verified record of it waits in the pool (the chain rule would refuse a fresh record once that one pays); chained (`--prev`) when the previous is paid and its proof is in this node's pool | `prover.rs` `pick_segment` |
|
||||
| The work | one export to the segment's last block, one fixture per block, one host run `--mode chain --save-shards [--prev]` that proves every shard and aggregates the segment in one process (one key setup), then every shard record signed and submitted (the shard payouts, 90% of the credit) and the segment record signed and submitted (the aggregator share, 10%) | `prover.rs` `prove_segment` |
|
||||
| The fallback | when no whole segment qualifies, the per-block path as shipped | `prover::choose` |
|
||||
| The host | `--mode chain` takes `--prev <aggregated.bin>` (chain_len continues; a previous proof that is not the parent block's is refused by number and parent hash) and with `--save-shards` writes per-shard records (statement, proof sha256, file, time) into the results | `proving/igneum-prove/host/src/main.rs` `run_chain` |
|
||||
| The tile | "Segments: N proven whole, M paid (x IGN to the aggregator), the last in T s" and the segment path's last line; the state carries `segments_submitted`, `segments_paid`, `segment_paid_wei`, `segment_last_s` | `ui/app.js`, `state.rs` |
|
||||
|
||||
Tests: the grid, the grouping (missing block, paid shard, our shard in the pool, exclusive shard), the margin and the attempted set, the per-key order (deterministic, different between two keys), the margin from the last time: 6 unit tests in `segments.rs`; 120 app tests, 8 core and 9 host tests pass. The host flags were run on the Mac's CPU first (bench-log, 07:12Z to 07:17Z): per-shard records written for a chain of 2, then a chain of 1 continued from it (`base_chain_len` 2, final `chain_len` 3).
|
||||
|
||||
**Item 2, "own pool only", answered from the source:** a relayed proof record carries its proof bytes (`protocol/flows/src/v10/proving.rs`: `IgneumProofRecordMessage { record, proof }`, 8 MB bound, handed to the pool with `local = false`), and so does a segment record (message 75). So "this node's pool" is every record relayed to it, and any aggregator can already fold any 8 proven blocks it has received; no node or consensus change is needed for that. What does limit carriage: a block template carries only entries the node's own verifier marked verified (`template_segment_section`, `template_section`), so a node with the verifier `Off` (node 1) never carries a record and a node with `Trust` carries unverified ones; PC 2's node runs the host and verifies. With one prover the carrier is PC 2's own next block.
|
||||
|
||||
**What one 5090 completes (arithmetic from the 5 October rows, the measurement below replaces it):** a chain of 8 empty blocks took 135.6 s cold beside the miner; one export and one key setup per segment instead of eight; so about one segment every 150 to 200 s, 9 to 12 segments an hour out of 450 (2 to 3%), against none. The aggregator share of a segment is 8 x 0.088 IGN = 0.70 IGN plus the 8 shards' 90% share; the forfeited share of the other 97% stays in the escrow until the fleet grows (47 mining cards or 6 dedicated provers for 100% at 1 block/s).
|
||||
|
||||
**PC 2 measurement (job `segments-pc2-pv1c`, 07:52Z to 08:24Z, `tools/proving-v1/pc2-segments.ps1`; bench-log "the segment-aligned prover beside the miner"):**
|
||||
|
||||
| Figure | Value |
|
||||
|---|---|
|
||||
| Whole segments proven in 30 min, one 5090 beside its miner | 9, one every 210 s (export 1.5 s, cut 45 s, chain 160 s: 8 shards 63 s, 8 aggregations 80 s) |
|
||||
| Shard records accepted and paid | 72 of 72, 0.91 to 2.72 IGN a shard (90% of the credit), carried 180 to 226 blocks after the block |
|
||||
| Segment records accepted | 0 of 9: refused by the chain rule as shipped (below) |
|
||||
| Miner's cost | 117.86 to 104.90 MH/s, 13.0 MH/s = 11.0% (the shipped prover: 5.0 MH/s, 4.0%, for 2.8x fewer shards) |
|
||||
| GPU memory peak | 16.5 to 17.6 GB with the miner resident; 24 GB tier unchanged |
|
||||
|
||||
**The second fault, found by the measurement: the chain rule as shipped.** `check_segment_record` accepts a fresh record (chain_len = N) only when the previous segment is UNPROVEN at the carrier. A segment's own deadline is its last block's DAA plus 600, the previous segment's deadline is 8 DAA earlier, so a fresh record is valid for 8 DAA (8 s on devnet) and must be proven before and carried inside them. The refusal on the live node, 9 times: "segment 114470..114477 does not chain to segment 114462..114469 (chain_len 8), which is pending until DAA 169681". The fast-time harness passed on 5 October because its chain continued from proven segments (case 2) and its fresh case ran exactly in that window (case 3, 4 DAA at N = 4). No prover-side move escapes it: the record must be proven, submitted and carried inside the window, which the 210-s proof cannot meet.
|
||||
|
||||
**The fix (fork branch 0f0dda95, behind a switch, never by default):** `proving_v1_fresh_rule_daa`; from it a fresh record is valid whenever the previous segment is not proven (pending or unproven) at the carrier; after a proven one a record must still chain. Two records that do not chain each attest their own blocks against the native statement, so nothing is lost but the longer proof chain, which restarts. In the consensus digest only once set (the v1 pattern), so a 0.3.12 node on the live devnet keeps the 0.3.11 digest until the override file sets it; `igneum_getProvingStatus.v1.freshRuleDaa/freshRuleActive` and `igneum_getSegmentStatement.freshAdmissible` report it. Tests: the digest moves once set; fresh after a pending segment passes from the switch, is refused before it, never after a proven one; the harness `--fresh-rule 0` inverts case 3. This is a rule change in the execution layer, not a parameter tuning, and the code proved it unavoidable: Josh's "no consensus parameter change unless the code proves it is unavoidable" is met by the 8-DAA window above and the nine refusals. The 0.3.12 coordinator sets the height (the same tip + 14,400 rule) in the override object with the release.
|
||||
|
||||
**Until the switch:** the app (272b025) holds a refused segment record and offers it again every pass until the segment's deadline, so on the shipped rule it lands only if a carrier falls inside the 8-DAA window, and from the switch it lands on the first retry; the shard records (90% of the credit) land either way, 8 per segment.
|
||||
|
||||
**Per tier, with this change and the switch:**
|
||||
|
||||
| Card | What it does | Per 30 min, empty blocks (measured on the 5090, approximate elsewhere) |
|
||||
|---|---|---|
|
||||
| 32 GB mining and proving (5090) | 9 whole segments, 72 shards paid, 9 aggregator shares once the switch is set | miner 11.0% down; 72 x 0.9 IGN = 65 IGN of shard payouts measured, plus 9 x 0.70 IGN aggregator share from the switch |
|
||||
| 24 GB mining and proving | the same path at the 16.5 to 17.6 GB peak measured; the fee-switch shard not yet measured on a 24 GB card | approximate: the 5090's numbers |
|
||||
| 16 GB | mines and proves on the patched server only (prover-floor rows); segment path untested there | pending the prover-floor agent's build |
|
||||
| 12 GB prove-only | proves alone on the patched server (10.3 GB); a dedicated prover takes 4 s a block alone (agg-cost rows), so about one segment every 40 s | approximate: 45 segments per 30 min, 6 such cards for 100% |
|
||||
| A rig (several cards) | one prover loop per machine today; the segment path claims one segment at a time on the aggregation card | the per-card loop is the next item |
|
||||
|
||||
## v1 live on devnet (6 October 2026, C47)
|
||||
|
||||
v1 active at 154,800 (crossed at DAA 154,814, 03:51:42Z); first segment record: none, because on a one-prover devnet no segment can be proven. The app's aggregator (`aggregate_once`, 0.3.11) needs a shard proof of every shard of every block of the segment in its node's pool, and PC 2 alone proves 13 shards per 10 minutes of about 600 blocks (2.2% coverage), so a run of 8 consecutive proven blocks never occurs: node 1 at 04:16Z reads segmentsInWindow pending 55, proven 0, unproven 20, paidSegments 0, and PC 2's app log (run win-1ccfe586-20261005-235130, 04:11Z to 04:16Z) reads every 42 s "aggregator: segment N..N+7: waiting for shard proofs N/0 ... N+7/0 in this node's pool", all 8 missing, each segment then past its 600-DAA deadline unproven. No fault in the node, the app or the record path; the fast-time harness passed because its shards ran at 90% coverage. Meanwhile the aggregator share (a tenth of every block's pool credit) stays in the escrow; shard payouts continue; miners and block watchers see nothing. What ends it: coverage at 8 consecutive blocks, 47 mining 5090-class cards with the shard loop as shipped in 0.3.11 (13 shards per 10 minutes a card), 18 mining cards through the chain mode, or 6 (approximate) proving-only cards, at 1 block/s on empty blocks (the fleet table above), or the segment length lowered on a small devnet (`proving_v1_segment_blocks`, a consensus param, so a digest change). No PC 2 job and no 0.3.12 item follow from this; the open item is the fleet, not the code.
|
||||
|
||||
## The empty `/api/state` reply (6 October 2026)
|
||||
|
||||
The aggregation-cost agent's jobs read the two bytes `{}` from `/api/state` on PC 2 at 22:22Z, 22:41Z and 00:18Z (0.3.10 and 0.3.11); the 21:01Z reply was full. Cause, from the app source and node 1's RPC: `ProvingState.paid_wei` is a `u128`, and serde_json's `to_value` refuses a u128 over u64::MAX (18,446,744,073,709,551,615 wei, 18.45 IGN); `state_json()` turned that refusal into `json!({})` with no log line. A paid shard is 1.23 IGN on average (node 1, `igneum_getProvingStatus`: 814.64 IGN over 663 shards at 00:3xZ), so the fifteenth paid shard after an app start empties the reply. PC 2's prover was blind to the root-owned socket from 20:00:56Z to 22:01Z (paid_wei stayed 0, hence the full reply at 21:01Z), proved from 22:02:13Z, and crossed 18.45 IGN inside its first 15 paid shards, before 22:22Z. Every app restart resets the counter, so the reply comes back for about 15 shards and goes again.
|
||||
|
||||
What it means: the dashboard on a proving machine shows nothing within about 12 minutes of its prover's first payout; every PC playbook that reads a card from `/api/state` fails the same way (the agent's job 5 reads settings.json instead). Mining, proving and payouts are untouched; it is the status page only.
|
||||
|
||||
Fix on the app branch: `paid_wei` serialises as a decimal string (the dashboard already reads it with `Number()`), `state_json` logs `[error] state_json: ...` once instead of answering `{}`, and the reply on any future serialisation error carries `error` and `version` rather than nothing; unit test `a_paid_total_over_u64_max_still_serialises_the_whole_state`. Ships with 0.3.11 if the shipper takes the new code tip, otherwise 0.3.12; until then the workaround is settings.json for the card keys.
|
||||
Fix on the app branch: `paid_wei` serialises as a decimal string (the dashboard already reads it with `Number()`), `state_json` logs `[error] state_json: ...` once instead of answering `{}`, and the reply on any future serialisation error carries `error` and `version` rather than nothing; unit test `a_paid_total_over_u64_max_still_serialises_the_whole_state`. Not in 0.3.11 (that tree closed at 22c2363, master 630da6b, published); 6714a45 heads 0.3.12, the morning's first cut, app only, before PC 1's relaunch (coordinator, counter-asic-2-rollout.md 8a); until then the workaround is settings.json for the card keys.
|
||||
|
||||
## Aggregation cost (5 October, night)
|
||||
|
||||
Josh, 5 October 2026: "fix everything else in the numbers tonight". Branch `agg-cost`; every number in `docs/bench-log.md`, "aggregation cost on the RTX 5090", with its job id. The proof statement and the pinned guests are unchanged: every existing fixture proof still verifies (`verify-segment` 0.027 s on the Mac, 0.036 to 0.041 s on PC 2).
|
||||
|
||||
| What | Before (5 October evening, `chain-pc2-pv1c`) | After (5 October night) |
|
||||
|---|---|---|
|
||||
| Chained aggregation, the card mining | 9.6 to 9.7 s a block | 9.6 to 9.8 s a block, the same (job `agg-cost-pc2-1`, phase A); the miner's presence is the whole cost: 2.1 to 2.2 s a block with the card to itself, 1.7 s unchained |
|
||||
| Shard proof (empty shard), the card mining | 7.3 to 7.7 s | 7.4 to 7.8 s; 1.9 to 2.2 s with the card to itself |
|
||||
| The miner's slowdown of the prover | 3 to 4x (against 4 October) | measured on the same fixtures 2 min apart: shards 3.6x, chained aggregation 4.5x, a block 4.2x |
|
||||
| Where the time goes | not profiled | the host's share 0.000 s (the prove call is everything); the GPU server prints no timings; the second deferred proof (the chain rule) costs 0.4 to 0.5 s alone and 1.7 to 1.9 s under the miner; the prover alone keeps the card busy 15.8% of the time, the miner 93.9% |
|
||||
| SP1 knobs (`SP1_WORKER_VERIFY_INTERMEDIATES=false`) | not tried | no gain: 7.8 s against 8.1 s over four aggregations, inside the spread; the shape knobs would change the recursion keys the pinned verifier accepts |
|
||||
| Batch fold (K blocks in one aggregator call) | not estimated | estimate from the measured step costs: 0.9 s a block alone and 3.8 s mining at K = 4, 0.7 and 2.8 s at K = 8 (0.25 s per further deferred proof alone, 1.8 s mining); a tree fold gains nothing. A new pinned guest and program id either way, so not tonight |
|
||||
| Two prover processes on one card | not tried | closed on SP1 6.8.1: both share one GPU server socket, run slower together (6.9 s a block against 4.1) and the second dies with the first (`early eof`) |
|
||||
| The miner's kernel length (`--batch-log2` of the CUDA worker, 2^B nonces a launch; job `agg-cost-pc2-6`, the 5090 alone with the job's own miner) | not tried | 2^22 (the default) and 2^20: 18.1 and 18.0 s a block, 10.0 to 10.4 s a chained aggregation, 104 MH/s; 2^18: 15.6 s, 8.8 s, 99 MH/s (minus 4%); 2^16: 11.1 s, 6.1 to 6.2 s, 84 MH/s (minus 19%), reproduced |
|
||||
| The GPU time-slice policy (`nvidia-smi compute-policy --set-timeslice`) | not tried | "Not Supported" on PC 2 (driver 13.3, Windows): closed |
|
||||
| The chosen combination | the defaults | the defaults stay: batch-log2 22 and SP1's default knobs. The one knob that moves the prover (2^16) costs a fifth of the hash rate all the time for a prover that is busy a few seconds a minute on the devnet; it is Josh's trade, not a default (below) |
|
||||
|
||||
Reading. The per-block aggregation is 2.1 s and a block 4.1 s on a 5090 that only proves, 9.7 and 17.5 s on one that also mines; no knob, fold or stream on tonight's SP1 changes the first pair, and only the miner's kernel length changes the second, at 1 MH/s per 0.37 s of block time. So "under 3 s a block" and "under 1.5x" are met on a card that is not mining and are not reachable on one that is. What that means per tier: a 5090 that mines and proves delivers a proven empty block every 17.5 s (6 cards for 1 block/s), the same card proving only every 4.1 s (2 cards, plus the shard work of full blocks: the fleet table above), and a batch fold of the aggregator (a new pinned guest) would bring the proving-only card to about 2.7 s a block and the mining one to about 12 s. What is being done: the app and host defaults are left as measured; the plan's open decision for Josh is whether a card that holds a shard assignment should drop to 2^16 for the proof's minute (1.6x faster proof, 19% of its hash rate for that minute) or whether proving-only cards carry the aggregation (the clean 2.1 s), and the batch fold goes on the next pin's list. The state class found on the way (`/api/state` answering `{}` once `paid_wei` passes u64::MAX, fixed on the app branch at 6714a45) is in the bench log with the rest.
|
||||
|
|
|
|||
193
docs/plans/release-0.3.12.md
Normal file
193
docs/plans/release-0.3.12.md
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
# Igneum Miner 0.3.12: the fresh-record rule switch (proving v1) and the app cut, prepared to the publish gate, 6 October 2026
|
||||
|
||||
Release engineer, from 08:05 UTC, on the coordinator's instruction ("prepare 0.3.12, APP ONLY, up to the publish gate and STOP there; Josh
|
||||
gives the go"), widened at 08:55Z on its clock: "no longer app only", the proving agent proved the segment record rule needs a consensus
|
||||
switch, so the node fork `proving-v1` 0f0dda95 (`proving_v1_fresh_rule_daa`: never by default, in the digest only once set; from it a fresh
|
||||
segment record is valid whenever the previous segment is not proven) and the app's segment-aligned prover (272b025, docs aea2f6a) ride in it.
|
||||
Worktree `/Users/joshm/Projects/igneum-wt-ship0312`, branch `release-0.3.12` from master ddfcdac; `vendor/` symlinked to the main checkout's (46
|
||||
entries); the fork worktree `vendor/igneum-node-0312`, branch `release-0.3.12-node` = 0f0dda95 cherry-picked onto 89dfcb95 (its parent ece42979
|
||||
is inside 89dfcb95, so the rebase is the one commit: `params.rs`, `exec/proving.rs`, `exec/rpc.rs`, `daemon.rs`) = **83089544**. The 0.3.11 recipe (`release-0.3.11.md`) throughout; every Mac build under the main checkout's lock;
|
||||
igneum-josh commits. Times are UTC.
|
||||
|
||||
## 1. What 0.3.12 carries
|
||||
|
||||
| Change | Where | State |
|
||||
|---|---|---|
|
||||
| `/api/state` never answers `{}` again: `paid_wei` (u128) is a decimal string, the error reply is logged; test | `proving-v1` app 6714a45 (the first item) | merged 9bcf4cd (the `docs/bench-log.md` conflict: both sides kept, the log is append-only) |
|
||||
| An update published over an hour before the engine started skips the hourly rollout slot; `manifest::unix_from_rfc3339` + tests | `update-catchup` 2207cd7 | merged b984c17 |
|
||||
| The GPU list ordered by performance (usable, discrete before integrated, rate in 5 MH/s buckets, memory); 3 UI tests | `card-order` ffb2bfa | merged c6608c1 |
|
||||
| HiveOS local mode carries the override (`OVERRIDE=` in the Flight Sheet's extra config, written to `data/override-params.json` by `h-run.sh`, C41), rigs mine only until a Linux prover ships, `IDENTITIES=auto` by VRAM, the per-card README table | `hive-words` 98271ff (packaging/hive only) | merged b0a6231 |
|
||||
| Ember Tune: two-knob plans + priors + the UI line; every quit names its source (b671c8b); a second engine never runs the updater (e600e63, C35); no pipe into a second engine (8ab9068); `jobrun.rs` elevated `follow_file` (1e9550e); the BOM fix + CI check (8273494); Power control switch (49bbe14 = 3562f26); `igneum-gpu-telemetry.exe` (ADLX) built by `build-windows.sh` and carried in the Windows inputs | `ember-tune` 9a6469f | NOT MERGED: conflicts in seven files against the 0.3.10/0.3.11 app (its base ca8d9f3 predates both): `ci.yml`, `config.rs`, `engine.rs` (the detect path, the power-cap plan, the test module), `ui/app.js` (four hunks against miner-ui-2's View), `ui/index.html` (the settings panel 0.3.10 removed), `proto-opencl/README.md`, `bench-log.md`. Its agent is rebasing it onto release-0.3.12 (section 2) |
|
||||
| The hidden-console builder for every elevated launch, `windows-spawn-check.mjs`, the PC 1 console-watch scripts | `job-console` 13755b9 (+ 3562f26 Power control) | NOT MERGED: conflicts in six files (`ci.yml`, `config.rs`, `engine.rs`, `jobrun.rs`, `app.js`, `index.html`), base a93199a; carried by the ember-tune rebase (it already holds 3562f26) |
|
||||
| The miner's gRPC resubscribe after a node restart (C43, the 0.3.11 finding) | no commit exists (the ledger entries b19fe5f, 0751dde, c8c831c only) | OWED, listed in section 10 |
|
||||
| The fresh-record rule switch: `proving_v1_fresh_rule_daa` (Option, never by default; a node with the field set prints it and carries it in the digest; a fresh segment record is valid from it whenever the previous segment is not proven) | fork `proving-v1` 0f0dda95 on ece42979 | cherry-picked onto 89dfcb95 as 83089544 (`release-0.3.12-node`) |
|
||||
| The segment-aligned prover: a segment record the chain rule refuses is held and offered again every pass until the segment closes; the fast-time harness on the fresh-record rule (both cases); the prover host and export; the WSL2 prover package script; `infra/fast-time/override-60x.json` (measured by its agent: 9 segments per 30 min on PC 2, 72 of 72 shards paid, 11% hash cost, 17.6 GB peak) | `proving-v1` app 272b025 + docs aea2f6a (on 6714a45) | merged 49e0e2c (clean) |
|
||||
| The packaged line (C34): the ten-field object of section 4 | 7dd3ff7 | `packaged-config.sh --test` passes |
|
||||
| The six version files | 81e4ecb (`--check`: 0.3.12 in all 6) | |
|
||||
|
||||
Left out on the coordinator's word: prover-floor's server (its packaging row is 0.3.13), explorer d7e797c, pool-v0, rig-install, ota-k2, the
|
||||
ledger forks.
|
||||
|
||||
Changelog line (draft, for the manifest notes at the go): "Igneum Miner 0.3.12: the fresh-record rule for proving v1 from DAA 192,000 (a fresh
|
||||
segment record is valid whenever the previous segment is not proven) and the segment-aligned prover; the GPU list in performance order; an
|
||||
old update no longer waits for the hour; Ember Tune (every card tuned for MH per watt, Power control off by default, the app never asks for
|
||||
administrator rights on its own); a second engine never installs over the app; /api/state always answers; HiveOS rigs carry the override.
|
||||
Node 83089544."
|
||||
|
||||
## 2. The branch
|
||||
|
||||
| Commit | What |
|
||||
|---|---|
|
||||
| 9bcf4cd, b984c17, c6608c1, b0a6231 | the four merges above, in the coordinator's order (proving-v1 first) |
|
||||
| 81e4ecb | `Igneum Miner 0.3.12: the six version files` |
|
||||
| ebea8b6 | the ember-tune rebase tip 7f6c4e6 (with job-console 13755b9 inside), merged as one branch (section 3) |
|
||||
| 11e8ca6, ab01f48, 01abcc2 | the plan |
|
||||
| 062c3f8 | `node-source.pin` 83089544 with the second inputs push (the Windows-build commit of 0.3.12) |
|
||||
| 37b6a7f | `tools/proving-v1/pc2-agg-cost.ps1`: `pkill -f sp1-gpu-server` (the CI root-socket check) |
|
||||
| 88df58e | master d3b64cb merged (docs only): the release tip, CI green |
|
||||
| 6532adf | `make-payload.sh`: on CI the AMD telemetry helper is taken from the unpacked inputs (the worker glob `igneum-worker-*.exe` missed `igneum-gpu-telemetry.exe`, so the first payload, run 37435975425, shipped without it: the inputs had it, the zip did not). The CI commit of 0.3.12 |
|
||||
|
||||
Checks on 81e4ecb before the rebase landed: the app `cargo test --release -p igneum-app` under the lock: ok 115 (lib) + 28 (ota-sign) + 8
|
||||
(prove-verify), 0 failed (08:19:22 to 08:19:28Z, warm target cloned from the 0.3.11 worktree); the UI tests `notices`, `update-card`, `view`:
|
||||
23 of 23.
|
||||
|
||||
## 3. Builds and artefacts
|
||||
|
||||
| What | Command | Result |
|
||||
|---|---|---|
|
||||
| The HiveOS package (first build, app-only cut) | the 0.3.11 node and workers with hive-words' scripts | 08:19:45Z: b0a20917... (24,501,272); superseded below once the node changed |
|
||||
| The merged tree | ember-tune 7f6c4e6 (release-0.3.12 b0a6231 merged INTO ember-tune as c5918c7, job-console 13755b9 cherry-picked on top; 0.3.11's six-section View and card order kept whole, Ember Tune's TuneLine block and the Power control switch added in 0.3.11's markup, `engine.rs` keeps the detect arm with the tune fields in `hotplug::apply_pref`, both test modules, `jobrun.rs` the hidden-console builder plus `follow_file`) merged as one branch | **ebea8b6**, 08:22Z (the CI commit is 062c3f8); the version files still 0.3.12 in all 6; packaged line, `node-source.pin` and `vendor/` untouched against master |
|
||||
| The app | `cargo test --release -p igneum-app` under the lock, then `cargo build --release` | 08:22:56 to 08:23:06Z: ok 133 + 28 + 8, 0 failed; `igneum-app 0.3.12` (2,273,664) |
|
||||
| The UI and relay tests | `node --test` notices, update-card, view, tune-line; `relay/test/*.test.mjs` | 26 of 26; 23 of 23 |
|
||||
| The CI checks on the Mac | identity, no-conflict-markers, copied-sources, signer-pipe, prover-socket, second-engine, bash-body (self-test + tree), kit-path (self-test + tree), windows-spawn (self-test + tree), pinned-guests, no-secrets, check-workflow-shell | all ok; `link-check` passes after `node site/build.mjs` (as ci.yml runs it: the committed `litepaper.html` points at `/bench#counter-asic-2-0-the-numbers`, an id the site build creates from `bench-log.md`) |
|
||||
| The Windows workers and the AMD telemetry helper | `proto-cuda/nvrtc/build-windows.sh` (mingw) under the lock, 08:23Z | the worker SOURCES are unchanged against master (`git diff master HEAD -- proto-cuda proto-opencl proto-metal igneum-pow`: only `build-windows.sh`, the new `gpu-telemetry.c` and its `.rc`), so the inputs carry the 0.3.11-verified workers that mined all night, igneum-worker-cuda.exe 2b3b8c92885442179f6bf2907c6f3eb453dc4a19908d90fd05981a09b7c2674c (1,536,512) and igneum-worker-opencl.exe edc4a75da3b93d814caa69fd635010780d63d5b622ec24c3741d433c584f91e3 (478,208), not this morning's rebuild of the same sources (12bfaa27..., e0fd7042...: mingw PE builds are not byte-reproducible); NEW igneum-gpu-telemetry.exe 8d679b52b19af3cbd6bf4fd6f77d337b2fb78af13d02627e7aa7e92507993459 (387,584; ADLX, SetupAPI, PDH; the Igneum resources, version 0.3.0 as the workers carry) |
|
||||
| The Windows inputs | `IGNEUM_WIN_RELEASE=<fork>/target-integration/x86_64-pc-windows-gnu/release IGNEUM_NODE_SRC=vendor/igneum-node-0312 packaging/windows/push-inputs.sh`, 08:24:29Z (a deploy of the downloads folder only; the manifest untouched) | node 89dfcb95 (igneumd.exe be8e83c0..., igneum-miner.exe 1ba1a249..., PC 2's 0.3.11 build), the two workers above, the telemetry helper, the mingw DLLs and nvrtc; signed, verified, live (HTTP 200); `node-source.pin` unchanged 89dfcb95 |
|
||||
| The DMG (first build, app-only cut) | the 0.3.11 node | 08:24:33Z: ff630e9d... (41,630,620); superseded below once the node changed |
|
||||
| The fork's Mac node, 83089544 | `CARGO_TARGET_DIR=vendor/igneum-node/target-0312 cargo build --release -j 4 -p kaspad -p igneum-miner --features kaspad/igneum-pow` from `vendor/igneum-node-0312`, under the lock; the target dir cloned by APFS from `target-0311`; `target-integration` in the fork worktree links to it | 08:38:13 to 08:41:31Z: igneumd **746a931fde9b840ca444a03cd757854e2e2ce7ebc4782ddd00ed161644d705f9** (41,386,160), igneum-miner 5381683e5717d91416c5a97456e0d050dd9645b1e27bce7d55808ce621cc1a26 (8,763,936); `igneumd/2.1.0-83089544` |
|
||||
| The seed's Linux node (glibc 2.36 target, zig) | `NODE_SRC=<abs fork> TARGET_DIR=vendor/igneum-node/target-0312-linux OUT_DIR=<scratch>/cross infra/cross/build-linux.sh` under the lock | 08:38:21 to 08:41:18Z (175 s): igneumd **4f142d5148f218f1286e24e7c4a167aa2f54262336f96e7cf281f520c714fc6f** (47,919,144), igneum-miner 38397ae66c265b63db8e5458b46e7feb942121a7dc5625919df0a8d35e7a1ba1 (9,861,072); version.txt names 83089544 |
|
||||
| The prover host and export (the pinned guests unchanged) | `cargo build --release -j 4 -p igneum-prove-export -p igneum-prove-host` in `proving/igneum-prove` under the lock (the target cloned from the 0.3.11 worktree) | 08:39:07Z: igneum-prove-host b90d58d0529ce29f0e7ca8ae780a6442f92edcf1c71752fc60fbc72bc5c11fd8 (58,626,560), igneum-prove-export b60056127d32bda363c0e305e73e9f699a5774c0988aee5bfd28a0fc61a56b9a (2,808,160); `--mode id`: shard program id 0x2b1a81cb413236cf063077b46ed3111628f6c41036bcf6e23ee4cbbf5679ef7a, the pin of 0.3.9 to 0.3.11; `pinned-guests-check` ok, `proving/igneum-prove/elf/` untouched |
|
||||
| The HiveOS package | `NODE_OUT=<scratch>/cross WORKERS_OUT=<the 0.3.11 Linux workers> VERSION=0.3.12 OUT=<scratch>/hive packaging/hive/make-hive-package.sh` (the Linux workers 4aaff27f.../82d90890... unchanged: their sources are) | 08:41:54Z: `igneum-hive-0.3.12.tar.gz` **7972af92e7cd9a032303eca4d95b533f53e0e68d1b9cae5bfe406a5b7c30a454** (24,506,282); `h-run.sh` writes `data/override-params.json` from `OVERRIDE` and starts the node with `--override-params-file` (the 0.3.11 open item closed); the node inside is 83089544 |
|
||||
| The DMG | `NODE=<fork igneumd> MINER=<fork igneum-miner> PROVE_HOST/PROVE_EXPORT=<this tree's build> packaging/mac/build-dmg.sh` under the lock | 08:42:49Z: `Igneum-Miner-0.3.12.dmg` **7a4a5f5f772956e983127280a5ec62a4fcfaf903b3afa38fe3a89a37cee23520** (41,702,535), engine 0.3.12, node 83089544 (igneumd 41,163,744 inside, stripped by the DMG build), the new prover host and export, `igneum-bench` from `proto-metal/main.swift` (unchanged, 66ec0e78...), `packaged-config` carries the ten-field object, hdiutil checksum valid |
|
||||
| The node suites with the igneum-pow feature (the coordinator's ask; the PC runner's test units carry no features field, so this is the Mac's run; the PC 2 run is owed to the Counter ASIC 3.0 coordinator's window, section 3a) | `CARGO_TARGET_DIR=vendor/igneum-node/target-0312 cargo test --release -j 4 -p kaspa-consensus -p kaspa-consensus-core -p igneum-exec -p kaspa-pow -p igneum-miner -p kaspa-p2p-flows --features kaspa-consensus/igneum-pow,kaspa-pow/igneum-pow` from the fork, under the lock, 08:39:31 to 08:45:01Z | igneum-exec 17 of 17, igneum-miner 18 of 18, kaspa-consensus 97 passed, 2 failed, 4 ignored. The two: (1) `pruning_proof::igneum_m20_tests::witnesses_are_checked_in_epoch_order_under_their_own_seeds` (`igneum_m20_tests.rs:122`: the expected `EpochSeeds.era` is all zeros, the code draws `515e...`: the test predates the era draw of class v3) FAILS THE SAME on 89dfcb95 (run 08:45:48Z on the 0.3.11 fork): the known M20 era fail, NOT fixed by 0f0dda95, still owed; (2) `finality::tests::ban_is_decided_by_the_carrying_block_so_nodes_agree_on_every_voter_list` (`finality.rs:1883`) failed inside the full crate run and PASSES alone on both 83089544 and 89dfcb95: order-dependent, not a regression of this cut, owed as flaky. kaspa-consensus-core 107 passed, 1 failed (`config::params::tests::fast_time_60x_file_is_the_devnet_at_60x`: `infra/fast-time/override-60x.json` does not parse into `OverrideParams`, "duplicate field `proving_v1_activation_daa`" at line 64: the file has carried a second proving-v1 block since c2544be on 5 October, so the test fails on master's file and on 89dfcb95 alike; not a 0.3.12 regression, the file is owed a dedupe), `db_compat` 7 of 7, kaspa-pow 14 of 14, kaspa-p2p-flows 33 of 33 (08:47:13Z, no fail-fast). Net: 3 failures, each present on 89dfcb95, none from 0f0dda95 |
|
||||
| PC 2 build-and-suite job | `IGNEUM_WIN_RELEASE=<scratch>/pc2-out node tools/build-job.mjs run --node vendor/igneum-node-0312 --target 1ccfe586 --targets linux,windows --node-tests "kaspa-consensus kaspa-consensus-core igneum-exec kaspa-pow igneum-miner kaspa-p2p-flows" --app-tests igneum-app` from this worktree, published 08:54:56Z on the prover-floor agent's "PC 2 is yours" (its floor-core-alone and floor-core-miner closed 08:48:37Z and 08:53:13Z, after the Counter ASIC 3.0 coordinator's release at 08:43:24Z); CPU only, the prover on, nothing else touched. The coordinator's later order (after the prover-floor agent's SECOND pair) arrived once the job had run; the app's queue serialised them anyway: this job ended 09:02:04Z and floor-build-6 started 09:02:05Z, then floor-core2-miner closed 09:13:22Z with the prover on and the miners never stopped, so nothing ran beside a GPU row | `build-20261006-085456`: started 08:55:24Z, done 09:02:04Z (400 s), every stage ok: Linux node 135 s (igneumd 34877b86..., igneum-miner c779777f...), Windows node 165 s (igneumd.exe 5bbcbd59f592fa31bf31c18516cef81cc0e7e537d398382fc8063e3402d80917, igneum-miner.exe af973318...; PC-built, NOT shipped: the inputs carry the Mac cross-build f580b4aa..., placed under the scratchpad), the app both targets; `RESULT test node [the six crates] exit 0 44 s` (without the igneum-pow feature, the runner's shape: the M20 era test and the fast-time file test are outside its reach there) and `RESULT test app/igneum-app exit 0 6 s` |
|
||||
| The Windows node exes | `CARGO_TARGET_DIR=vendor/igneum-node/target-0312-win proto-cuda/windows-node/cross-build.sh <fork> 4` on the Mac (mingw, the 0.3.5/0.3.6/0.3.9 path; PC 2 is the Counter ASIC 3.0 coordinator's this morning), the target cloned from `target-release-win`, under the lock | 08:40:43 to 08:48:05Z (6 min 42 s): igneumd.exe **f580b4aad1e19a47742d0d836a56dad36b9380d3890ca115b1babced4d83a8db** (52,177,920), igneum-miner.exe 06c17d4c23c8b1327793bebcd9b2cba115045ea91b68baea8cb92b232a94678b (11,040,768); static (KERNEL32, advapi32, api-ms-win-core only) |
|
||||
| The Windows inputs, second push | `IGNEUM_WIN_RELEASE=<target-0312-win>/x86_64-pc-windows-gnu/release IGNEUM_NODE_SRC=vendor/igneum-node-0312 packaging/windows/push-inputs.sh`, 08:48:28Z | node 83089544 (the two exes above), the 0.3.11-verified workers 2b3b8c92.../edc4a75d..., the telemetry helper 8d679b52..., the mingw DLLs and nvrtc; `payload-inputs.zip` f8e567bd164b382d32a33fb488df658fa68555092ac6bdac85387d0a8bd5d547 (65,259,161), signed and verified, live (HTTP 200); `node-source.pin` 83089544 committed as **062c3f8**, the CI commit of 0.3.12 |
|
||||
|
||||
| The Windows installer and zip, first runs | runs 37435975425 (ebea8b6, no telemetry helper) and 37436904041 (6532adf, the 0.3.11 node) | superseded |
|
||||
| The Windows installer and zip | `windows.yml` run 37438673235 on 062c3f8 (dispatched 08:49:12Z after the second inputs push) | `Igneum-Miner-Setup-0.3.12.exe` **f11a296acf1ea3efa8a6151efa357cc5c222e3b2ffec5701ea4bcefe29307810** (45,270,093); `igneum-windows-app.zip` **a6f33ef21bb1d1682f48ca22332d50c0302f4b4f552a99157581f3249c42ea3b** (65,507,597): igneumd.exe f580b4aa... (the cross-build, 52,177,920), igneum-miner.exe, igneum-app.exe 0.3.12 (3,700,736), the two workers, `igneum-gpu-telemetry.exe` (387,584) this time, the mingw DLLs, nvrtc64_120_0.dll and nvrtc-builtins64_128.dll |
|
||||
|
||||
## 4. The override objects and the digests (the 0.3.12 Mac node 746a931f..., ports 60975/60976, 22 s each, under `run`)
|
||||
|
||||
| Override file | Lines | Digest |
|
||||
|---|---|---|
|
||||
| none | `igneumd/2.1.0-83089544`, no activation line | c562d70e1428c9789823cc40067623b4767f7c555ce7ff4ea11c1498f013ef6c, EQUAL to 0.3.11's no-file digest: the new field is never by default and leaves the digest alone until set |
|
||||
| the fleet's live nine-field object | the six activation lines of 0.3.11, no fresh-rule line | **0139ab9dc2992d449ec787d8f021974933631eb55740ab4b6ce9d5c226e72888**, EQUAL to the fleet's digest today: publish 1 (the binary) changes no handshake, a 0.3.12 node and a 0.3.11 node on the nine-field file accept each other |
|
||||
| the ten-field object at the FIRST pin (`proving_v1_fresh_rule_daa` 192000, void: the floor failed at the go) | the six lines plus the fresh-rule line at 192000 | bd786a4b521e87c05bce3da4c46b4f4696deb16dfbdc913f181c980a8eb51688 (never published) |
|
||||
| the ten-field object as SHIPPED (`proving_v1_fresh_rule_daa` 198000) | the six lines plus `Proving v1 fresh-record rule from the override file: from DAA score 198000 a fresh segment record is valid whenever the previous segment is not proven` | **7bd98cc4118616455709d5e32a30b799e6e67caa42d2b5d09875cd49848a7ed7** (read 11:22:01Z on 746a931f...) |
|
||||
|
||||
The ten-field object (the packaged line 7dd3ff7, the manifest of publish 2, the hand nodes' and the seed's files at step 2):
|
||||
|
||||
```
|
||||
{"difficulty_v2_activation_daa":33000,"proving_v0_activation_daa":84100,"fees_v1_activation_daa":210000,"finality_v3_activation_daa":135200,"program_class_v3_activation_daa":154800,"proving_v1_activation_daa":154800,"proving_v1_segment_blocks":8,"proving_v1_unproven_daa":600,"proving_v1_aggregator_share_bps":1000,"proving_v1_fresh_rule_daa":198000}
|
||||
```
|
||||
|
||||
N was first pinned at 192,000 (tip + 14,400 for a publish near 10:15Z; the floor would hold while the tip was at or under 181,200, about
|
||||
11:15Z). Josh's go came at 11:20Z with the tip at 181,582: the floor read 10,418, under 10,800, so N was RE-PINNED to 198,000 (tip + 14,400
|
||||
for publish 2 near 11:55Z; the floor holds until tip 187,200, about 12:55Z): the packaged line 8a6b133, the DMG rebuilt 11:22:08Z
|
||||
(7bcbb8a94038ea7a87ebfab514b6771f93b8fce2d991340bd5610713dc9548e5, 41,702,608), the installer rebuilt on CI (windows-ci 37455874734 on 8a6b133,
|
||||
green 11:27:33Z: `Igneum-Miner-Setup-0.3.12.exe` a6b3ea275373411e9f988ecd4ecc79f2cda5f68d54d681662dcbf7590689aef0, 45,275,988; `igneum-windows-app.zip`
|
||||
7ab28e772670dc58428cda4c9ff584b35f70507057d525a85d04391ee145dc53, 65,507,595), the digest re-read, publish 1 delayed by 8 minutes. The lesson for
|
||||
the next cut: pin N at the go, not at the forecast, or pin with a 3,600 margin over tip + 14,400 when the go is more than an hour out. A 0.3.11 node given the ten-field file dies on the unknown field
|
||||
(`deny_unknown_fields`), which is why publish 2 comes only after every node runs the 0.3.12 binary (the reviewer's C39, the 0.3.11 order).
|
||||
|
||||
## 5. The publish gate: what runs at Josh's go, in which order (the 0.3.11 two-publish shape)
|
||||
|
||||
This was the plan at the gate; sections 6 and 7 record what ran. Runbook: the session scratchpad's `r0312/rollout-0312.sh` (every step a function; `step_floor` before each publish).
|
||||
|
||||
| Step | What | Gate |
|
||||
|---|---|---|
|
||||
| 0 the floor | `step_floor`: 192,000 minus the tip's DAA at least 10,800 | read before publish 1 and again before publish 2 |
|
||||
| 1a the hand nodes, the seed | the observer and node 1 on the 0.3.12 binary with the NINE-field file (`IGNEUMD=<fork>/target-integration/release/igneumd IGNEUMD_COMMIT=83089544 infra/devnet/restart-hand-nodes.sh '<nine>'`), then the seed (`IGNEUMD_LINUX=<scratch>/cross/igneumd IGNEUMD_LINUX_SHA256=4f142d51... infra/devnet/restart-seed.sh '<nine>'`); every one prints 0139ab9d..., nobody is refused; then `step_mac_miners` (the Mac's miner does not reconnect to a restarted node 1 by itself, C43) | Josh's go |
|
||||
| 1b publish 1 | `node tools/ship-app.mjs 0.3.12 --node vendor/igneum-node-0312 --branch release-0.3.12 --public --activation-height 154800 --deadline-note "program class v3 + proving v1" --notes '<section 1>' --from ci`: ci "already" (the green Windows run), fetch, dmg "already", copy, manifest with `consensus` CARRIED OVER (the nine-field object; the digest stays 0139ab9d...), deploy, verify (`--from console` after the public index settles at the edge), the console item; `--public` carries the HiveOS package 7972af92... | after 1a |
|
||||
| 1c update-now | the Mac (d937c69d) first; the laptop (37ba0461) with it if it is on the air; PC 2 (1ccfe586) on the Counter ASIC 3.0 coordinator's word (PC 2 is its this morning; the proving agent's constraints: the app's prover stays on, no quit or restart of anything but the update's own); PC 1 (ae432dc7) last, once Josh has relaunched its app (down since 22:31:06Z yesterday, on 0.3.10: it takes the nine-field object and 0.3.12 at its relaunch through the manifest; Power control is off by default so nothing asks for administrator rights) | each machine's STATUS line back on 0.3.12 with 0139ab9d... |
|
||||
| 2a the floor again | `step_floor` | >= 10,800 or re-pin |
|
||||
| 2b the hand nodes, the seed | the same two scripts with the TEN-field file; each prints bd786a4b... and refuses the nine-field side until it switches; `step_mac_miners` again | every app node on the 0.3.12 binary (1c) |
|
||||
| 2b publish 2 | `publish-manifest.sh --version 0.3.12 --override '<ten>' --activation-height 192000 --deadline-note "proving v1 fresh-record rule" --notes '<section 1>' --public --deploy` | after the hand nodes |
|
||||
| 2b update-now (switch) | the Mac and the laptop, then PC 2 on the 3.0 coordinator's word, then PC 1: each app writes the ten-field file at the manifest take and restarts its node at a safe moment (the Mac's node is node 1, already switched: nothing to restart) | |
|
||||
| the sweep | every node prints bd786a4b521e87c05bce3da4c46b4f4696deb16dfbdc913f181c980a8eb51688; the fresh-record rule arms at DAA 192,000 | |
|
||||
|
||||
One line for Josh, per machine, when he says go: the Mac and PC 2 each restart their engine once for 0.3.12 (under a minute, the miner back on
|
||||
the next template) and their node once more for the fresh-record switch (a few seconds, mining resumes on the same chain); PC 1 does the
|
||||
same at its relaunch and, with Power control off by default, never asks for administrator rights again (its 5090 runs uncapped until he
|
||||
switches Power control on in Settings); the observer, node 1 and the seed are restarted by hand twice; from DAA 198,000 (about 15:55Z) a
|
||||
prover may file a fresh segment record whenever the previous segment is not proven, so paid segments stop stalling behind an unproven one;
|
||||
until the switch nothing changes in consensus (digest 0139ab9d... through publish 1).
|
||||
|
||||
## 6. The rollout (Josh's go 11:20Z through the coordinator; two publishes)
|
||||
|
||||
Baseline 11:20:22Z: tip 181,582; the observer, node 1 and the seed on 89dfcb95 at 0139ab9d; the Mac app 0.3.11 (its miner PAUSED since
|
||||
07:10Z on Josh's order "stop mining on the Mac", not the C42 class: I resumed it once at 11:26:11Z before the order reached me and the
|
||||
coordinator re-paused it; it stays paused, no restart-miners after the hand restarts); PC 2 0.3.11 at 113 MH/s; PC 1 0.3.11, relaunched by
|
||||
Josh at 11:17Z with the 5090 and a 4070 in the enclosure; the laptop and Sam's Mac off the air.
|
||||
|
||||
| Step | Time | Result |
|
||||
|---|---|---|
|
||||
| 0 the floor | 11:20:22Z | 10,418 < 10,800: FAILED at 192,000; re-pinned to 198,000 (section 4), publish 1 delayed to the installer rebuild |
|
||||
| 1a the observer, node 1 | 11:22:12Z (pid 92464), 11:22:24Z (pid 92624) | `igneumd/2.1.0-83089544` on the nine-field file, digest 0139ab9d... (unchanged, nobody refused) |
|
||||
| 1a the seed | 11:22:45Z (MainPID 136418) | the same binary 4f142d51..., the same digest |
|
||||
| 1a restart-miners, the Mac | 11:22:56Z | ran; nothing to restart, the miner is paused on Josh's order (above) |
|
||||
| 1b publish 1 | the ship 11:28:30 to 11:31:55Z from cf1ad2b (master f11b02e merged first: the preflight refuses a tree behind origin/master) | ci "already" (37455874734), fetch "already" (the re-pinned installer), dmg "already", copy ok, manifest 0.3.12 with `consensus` CARRIED OVER (the nine-field object, activation 154800), deploy ok, verify refused the public index at the edge (every cut); `--from console` 11:44:41Z: item #368 |
|
||||
| 1c update-now, the Mac | 11:32:18Z | engine restart 11:32:56Z (run `mac-d937c69d-20261006-113256`), "updated to Igneum Miner 0.3.12 from 0.3.11", STATUS "0.00 MH/s, paused, node 5 peers, synced" (node 1) |
|
||||
| 1c update-now, PC 2 | 11:32:46Z (the 3.0 coordinator's mkdir lock `/tmp/igneum-devnet/pc2-ca3.lock` absent; `pc2-ca3.clear` is a note, not a lock) | engine restart 11:33:41Z (run `win-1ccfe586-20261006-113341`), igneumd 83089544 started 11:33:45Z on the nine-field file (0139ab9d), worker ready 11:34:39Z, mining 11:34:42Z, 0 faults |
|
||||
| 1c update-now, PC 1 | 11:33:28Z (on the prover-floor agent's "PC 1 build closed" 11:31:34Z and the coordinator's "PC 1 back") | the installer downloaded and verified 11:34:06Z, "per-user install, no administrator prompt", engine restart 11:34:12Z (run `win-ae432dc7-20261006-113412`), cards "RTX 5090, RTX 4070 [discrete], AMD integrated [off]", STATUS mining 11:35:13Z, the 5090's race base 140.2 MH/s, digest 0139ab9d |
|
||||
| 2a the floor | 11:36:53Z | tip 182,570; 15,430 >= 10,800 at 198,000 |
|
||||
| 2b the observer, node 1 | 11:36:55Z (pid 13642), 11:37:07Z (pid 13777) | the ten-field file, digest **7bd98cc4118616455709d5e32a30b799e6e67caa42d2b5d09875cd49848a7ed7** |
|
||||
| 2b the seed | 11:37:25Z (MainPID 136590) | 7bd98cc4... |
|
||||
| 2b publish 2 | 11:37:36Z | `publish-manifest.sh --version 0.3.12 --override '<ten>' --activation-height 198000 --deadline-note "proving v1 fresh-record rule" --public --deploy`; the HiveOS package 7972af92... served at `/public/igneum-miner-hive.tar.gz` and `dl/public/igneum-hive-0.3.12.tar.gz` (HTTP 200, 24,506,282), the 0.3.11 package removed |
|
||||
| 2b switch, the Mac | 11:40:30Z | ran 11:40:58Z: "consensus override changed; the node restarts with it at a safe moment"; its node is node 1 (external), already on 7bd98cc4, nothing to restart |
|
||||
| 2b switch, PC 2 | 11:40:56Z | ran 11:41:23Z, "restarting the node with the new consensus parameters", igneumd started 11:41:25Z (pid 18732) on 7bd98cc4..., mining again 11:43:42Z, 113.0 MH/s at 11:44:42Z |
|
||||
| 2b switch, PC 1 (last) | 11:42:54Z | ran 11:43:28Z, node restarted 11:43:29Z (pid 5556) on 7bd98cc4..., "waiting" 11:43:44 to 11:44:14Z, mining 11:44:44Z, 100.95 MH/s ramping at 11:45:14Z: its miners read 0 MH/s for about a minute after the node restart before coming back (the C43 class: the miner waits out the restarted node instead of resubscribing at once; the coordinator's note); the Ember Tune run 3 on PC 1 (ember-tune-pc1-3, 11:44:58Z) then took the box, after this restart, not under it |
|
||||
| the laptop, Sam's Mac | off the air | they take 0.3.12 and the ten-field object through the manifest when they return; no 0.3.11 app was on the air to take the ten-field file before its binary (C39) |
|
||||
|
||||
## 7. The digest sweep (closed 11:45:20Z)
|
||||
|
||||
| Node | Binary | Digest | Since |
|
||||
|---|---|---|---|
|
||||
| the observer | `igneumd/2.1.0-83089544` (746a931f...) | 7bd98cc4... | 11:36:55Z |
|
||||
| node 1 | the same | 7bd98cc4... | 11:37:07Z |
|
||||
| the seed | 83089544 (4f142d51..., glibc 2.36 target) | 7bd98cc4... | 11:37:25Z |
|
||||
| PC 2 | the installer's igneumd.exe f580b4aa... (the Mac cross-build) | 7bd98cc4... | 11:41:25Z |
|
||||
| PC 1 | the same | 7bd98cc4... | 11:43:29Z |
|
||||
| the Mac | attached to node 1 | node 1's | 11:37:07Z |
|
||||
| the laptop, Sam's Mac | 0.3.10 / 0.3.9 | pending | off the air |
|
||||
|
||||
Tip 183,154 at 11:45:20Z, no refusals on the hand nodes after the switch; the fresh-record rule arms at DAA 198,000 (about 15:55Z at 0.98 DAA/s).
|
||||
The fleet during the window: PC 2 and PC 1 mined on 0139ab9d while the hand nodes and the seed were on 7bd98cc4 (11:37 to 11:41Z); each
|
||||
rejoined at its switch; the Mac's miner paused throughout on Josh's order.
|
||||
|
||||
## 8. CI
|
||||
|
||||
| Run | On | Result |
|
||||
|---|---|---|
|
||||
| `ci` 37435705568 | ebea8b6 (the branch push, 08:22:40Z) | green (pow tests and census build, simulators, site build + link check + identity, the PowerShell 5.1 parse job) |
|
||||
| `windows-ci` 37435975425 | ebea8b6 | green 08:30:58Z (the parse job 08:25:16 to 08:25:56Z; engine, window host, payload, installer, smoke run 08:26:00 to 08:30:58Z); superseded by the run below (no telemetry helper in its payload) |
|
||||
| `ci` 37436904569 | 6532adf (the branch push, 08:33Z) | (pending) |
|
||||
| `windows-ci` 37436904041 | 6532adf | green 08:39:20Z; superseded by the run below (the node changed) |
|
||||
| `ci` 37436904569 | 6532adf | green |
|
||||
| `windows-ci` 37438673235 | 062c3f8 (`gh workflow run windows.yml --ref release-0.3.12`, 08:49:12Z, after the second inputs push) | green 08:54:27Z (the parse job 08:49:18 to 08:49:59Z; engine, window host, payload, installer, smoke run 08:50:02 to 08:54:27Z against the 83089544 inputs); fetched 09:03:17Z with `OTA_SKIP=1 CONSOLE_SKIP=1 packaging/windows/fetch-ci-artifacts.sh 37438673235` into the downloads folder, NOT deployed |
|
||||
| `ci` 37438674529 | 062c3f8 | FAILED in one step, `prover-socket-check.sh`: `tools/proving-v1/pc2-agg-cost.ps1` (in through the proving-v1 merge) ends its root prover with `pkill -x sp1-gpu-server`, and the check wants `pkill -f`; the line now reads `pkill -f ... ; rm -f /tmp/sp1-cuda-*.sock` (one playbook line, nothing the app or the packaging reads; `git diff 062c3f8 <fix> -- app packaging proto-cuda proto-opencl proto-metal vendor` is empty, so the Windows artefacts of 37438673235 stand, as 0.3.11's did across 3b0262f and 2a62735); the rerun is the row below |
|
||||
| `ci` 37440456687 | 37b6a7f (the one-line playbook fix) | green 09:07Z |
|
||||
| `ci` 37440559793 | 88df58e (master d3b64cb merged in: the morning summary, docs only; the release tip) | green 09:08:08Z. The 0.3.12 CI verdict is therefore run 37440559793 on 88df58e; the Windows build is run 37438673235 on 062c3f8, the same app, packaging and node sources (`git diff 062c3f8 88df58e -- app packaging proto-cuda proto-opencl proto-metal vendor igneum-pow` is empty) |
|
||||
|
||||
The ship state file `~/.cache/igneum/ship/0.3.12.json` carries `sha` = 062c3f8 (the Windows-build commit, which the ci step looks up by commit; the tree is 88df58e, docs and one playbook line later, as 0.3.11's was two docs commits past its build commit),
|
||||
`forkCommit` 89dfcb95, `bumpedAt` before the DMG's mtime (so the dmg step reads "already"), and `runId` once the Windows run is green.
|
||||
|
||||
## 9. Owed to the next cut (0.3.13)
|
||||
|
||||
| Item | What |
|
||||
|---|---|
|
||||
| C43, the miner's dead gRPC channel | no commit exists; `igneum-miner mine grpc://` must re-subscribe after its node restarts (the 0.3.11 finding: 11 min of 26 MH/s burned on the Mac); until then every hand restart of node 1 is followed by `restart --what miners` |
|
||||
| prover-floor's server | its packaging row is 0.3.13 (the coordinator's word) |
|
||||
| explorer d7e797c, pool-v0, rig-install, ota-k2, the ledger forks | left out on the coordinator's word |
|
||||
| the Windows workers' reproducibility | mingw PE builds differ byte-for-byte between builds of the same sources (12bfaa27 vs 2b3b8c92 today); a `-Wl,--no-insert-timestamp` (or `SOURCE_DATE_EPOCH`) in `build-windows.sh` would make G5's one-commit rule checkable by hash |
|
||||
| the site build's non-idempotence and the pre-push flip | unchanged from 0.3.10 and 0.3.11 |
|
||||
|
|
@ -64,6 +64,7 @@
|
|||
"proving_v1_activation_daa": 18446744073709551615,
|
||||
"proving_v1_segment_blocks": 8,
|
||||
"proving_v1_unproven_daa": 10,
|
||||
"proving_v1_fresh_rule_daa": 0,
|
||||
"proving_v1_aggregator_share_bps": 1000,
|
||||
"fees": {"pgas": {"version": 1, "cycles_per_pgas": 1000, "intrinsic_pgas_per_tx": 300, "modexp_base": 10, "modexp_per_byte_numer": 1, "modexp_per_byte_denom": 10}, "block_proving_gas_limit": 120000, "shard_proving_gas_budget": 30000, "min_execution_base_fee_wei": 100000000000, "min_proving_base_fee_wei": 10000000000000, "initial_execution_base_fee_wei": 100000000000, "initial_proving_base_fee_wei": 10000000000000, "base_fee_change_denominator": 8}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@
|
|||
<key>CFBundleVersion</key>
|
||||
<string>VERSION_STAMP</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>0.3.11</string>
|
||||
<string>0.3.12</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>APPL</string>
|
||||
<key>CFBundleExecutable</key>
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ DL_HOST="https://dl.igneum.network"
|
|||
# carry the devnet's activation height here, the same N as every other devnet node, before it is cut (Mac and CI alike:
|
||||
# make-payload.sh sources this file). Rule and order: docs/plans/difficulty-v2-rollout-devnet.md.
|
||||
# Example: NODE_OVERRIDE_PARAMS='{"difficulty_v2_activation_daa": 123456}'
|
||||
NODE_OVERRIDE_PARAMS='{"difficulty_v2_activation_daa":33000,"proving_v0_activation_daa":84100,"fees_v1_activation_daa":210000,"finality_v3_activation_daa":135200,"program_class_v3_activation_daa":154800,"proving_v1_activation_daa":154800,"proving_v1_segment_blocks":8,"proving_v1_unproven_daa":600,"proving_v1_aggregator_share_bps":1000}'
|
||||
NODE_OVERRIDE_PARAMS='{"difficulty_v2_activation_daa":33000,"proving_v0_activation_daa":84100,"fees_v1_activation_daa":210000,"finality_v3_activation_daa":135200,"program_class_v3_activation_daa":154800,"proving_v1_activation_daa":154800,"proving_v1_segment_blocks":8,"proving_v1_unproven_daa":600,"proving_v1_aggregator_share_bps":1000,"proving_v1_fresh_rule_daa":198000}'
|
||||
|
||||
# igneum_secret_file <env var name> <base name> -> the file to read: the variable when set, else <base>.next when it
|
||||
# exists, else <base>; IGNEUM_CONFIG_DIR (tests) replaces ~/.config/igneum
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
#define ArtDir "..\..\brand\icons"
|
||||
#endif
|
||||
#ifndef AppVersion
|
||||
#define AppVersion "0.3.11"
|
||||
#define AppVersion "0.3.12"
|
||||
#endif
|
||||
#define AppName "Igneum Miner"
|
||||
#define Publisher "Igneum"
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
89dfcb95be5ace1a2b7a4fb18d88aeb22023cab4
|
||||
830895447497bdea5e4779c9aa9d4118eac3cbbc
|
||||
|
|
|
|||
|
|
@ -93,9 +93,16 @@ fn run() -> Result<()> {
|
|||
// proof (the chain rule of design 5.3), the measurement of docs/plans/proving-v1.md step 2
|
||||
let list = arg("--chain").or_else(|| args.get(1).filter(|a| !a.starts_with("--")).cloned()).context("--chain <f1.json,f2.json,...> (consecutive fixtures)")?;
|
||||
let fixtures: Vec<String> = list.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect();
|
||||
return run_chain(&pinned, &fixtures, prover, out_path.as_deref());
|
||||
// --save-shards writes every shard's compressed proof next to the results (block-N-shard-i-compressed.bin),
|
||||
// so `--mode aggregate` can re-run the aggregation of the same proofs under other settings
|
||||
let save_shards = args.iter().any(|a| a == "--save-shards");
|
||||
// --prev <file>: the previous segment's aggregated proof; the chain continues from it (chain_len grows past
|
||||
// the segment length, the chain rule of spec 7.8) instead of starting fresh. The app's segment path (6 October
|
||||
// 2026) passes it when the node reports the previous segment paid and its proof in the pool.
|
||||
let prev = arg("--prev");
|
||||
return run_chain(&pinned, &fixtures, prover, out_path.as_deref(), save_shards, prev.as_deref());
|
||||
}
|
||||
let path = args.get(1).filter(|a| !a.starts_with("--")).context("usage: igneum-prove-host <fixture.json> [--mode native|execute|shard|compressed|block|all] [--shard N] [--budget <test pgas>] [--prover 0x..] [--out results.json]; --mode chain --chain <f1,f2,...> [--prover 0x..] [--out results.json]; --mode aggregate --proofs <a.bin,...> --parent 0x.. [--prev prev.bin] [--out results.json]; --mode verify --proof <file> --statement 0x..; --mode verify-segment --proof <file> --statement 0x..; --mode id")?;
|
||||
let path = args.get(1).filter(|a| !a.starts_with("--")).context("usage: igneum-prove-host <fixture.json> [--mode native|execute|shard|compressed|block|all] [--shard N] [--budget <test pgas>] [--prover 0x..] [--out results.json]; --mode chain --chain <f1,f2,...> [--prover 0x..] [--out results.json] [--save-shards] [--prev prev.bin]; --mode aggregate --proofs <a.bin,...> --parent 0x.. [--prev prev.bin] [--out results.json]; --mode verify --proof <file> --statement 0x..; --mode verify-segment --proof <file> --statement 0x..; --mode id")?;
|
||||
let shard_index: usize = arg("--shard").map(|s| s.parse()).transpose()?.unwrap_or(0);
|
||||
// `--budget <pgas>`: re-plan the fixture's block at a TEST budget (the S_p curve of 5 October 2026); the fixture's
|
||||
// own per-shard plan is then not compared (the chain and the sums still are), and `--out` records the cut
|
||||
|
|
@ -577,6 +584,8 @@ fn setup_sp1(pinned: &pinned::Pinned, results: &mut serde_json::Map<String, serd
|
|||
let sp1 = Sp1ProofSystem::from_env(pinned.shard_elf(), pinned.agg_elf())?;
|
||||
let setup_s = t.elapsed().as_secs_f64();
|
||||
println!("RESULT setup: {:.2} s, ProofSystem v{} shard program id {} aggregator id {} at {}", setup_s, Sp1ProofSystem::VERSION, sp1.program_id(), sp1.aggregator_id(), now());
|
||||
println!("RESULT sp1 knobs: {}", Sp1ProofSystem::env_knobs());
|
||||
results.insert("sp1_env_knobs".into(), Sp1ProofSystem::env_knobs().into());
|
||||
if sp1.program_id() != pinned.shard_id || sp1.aggregator_id() != pinned.agg_id {
|
||||
bail!("SP1's key setup derived shard program id {} and aggregator id {} from the embedded guests, the pinned manifest says {} and {}: this host would make proofs no other node accepts (re-pin with proving/igneum-prove/pin-guests.sh)", sp1.program_id(), sp1.aggregator_id(), pinned.shard_id, pinned.agg_id);
|
||||
}
|
||||
|
|
@ -616,7 +625,7 @@ fn out_dir_of(out_path: Option<&str>) -> std::path::PathBuf {
|
|||
/// `--mode chain`: every fixture in order, consecutive on the chain (number and parent hash), each block's shards
|
||||
/// proven compressed and aggregated with the previous block's aggregated proof (`AggInput.prev`, the chain rule),
|
||||
/// every proof verified. One RESULT line per shard, per block (with the running totals) and for the chain.
|
||||
fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_path: Option<&str>) -> Result<()> {
|
||||
fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_path: Option<&str>, save_shards: bool, prev_path: Option<&str>) -> Result<()> {
|
||||
if fixtures.is_empty() {
|
||||
bail!("--chain needs at least one fixture");
|
||||
}
|
||||
|
|
@ -654,16 +663,34 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_
|
|||
println!("RESULT chain native block {}: {} shard(s), pgas {}, gas {}, pre {} post {}", f.block.env.number, shards.len(), outcome.pgas_used, outcome.gas_used, pre_root, outcome.state_root);
|
||||
built.push((f.block.env.number, f.block.env.hash, shards, pre_root));
|
||||
}
|
||||
// the previous segment's proof: the chain continues from it (its block must be the parent of the first fixture)
|
||||
let mut prev: Option<proof_system::Sp1SegmentProof> = match prev_path {
|
||||
None => None,
|
||||
Some(p) => {
|
||||
let bytes = std::fs::read(p).with_context(|| format!("read {p}"))?;
|
||||
let proof: sp1_sdk::SP1ProofWithPublicValues = bincode::deserialize(&bytes).with_context(|| format!("{p} is not a bincode SP1 proof"))?;
|
||||
let output = BlockOutput::from_bytes(proof.public_values.as_slice()).with_context(|| format!("{p}: public values are not a block statement"))?;
|
||||
if output.number + 1 != first || output.block_hash != loaded[0].1.block.env.parent_hash {
|
||||
bail!("--prev attests block {} ({}), the chain starts at block {first} with parent {}: the previous proof must be the parent block's", output.number, output.block_hash, loaded[0].1.block.env.parent_hash);
|
||||
}
|
||||
println!("RESULT chain prev: block {} chain_len {} (the chain continues from it)", output.number, output.chain_len);
|
||||
Some(proof_system::Sp1SegmentProof { proof, output })
|
||||
}
|
||||
};
|
||||
let base_len = prev.as_ref().map(|p| p.output.chain_len).unwrap_or(0);
|
||||
let sp1 = setup_sp1(pinned, &mut results)?;
|
||||
let chain_t = Instant::now();
|
||||
let mut prev: Option<proof_system::Sp1SegmentProof> = None;
|
||||
let mut blocks_json = Vec::with_capacity(built.len());
|
||||
let (mut shard_total, mut agg_total, mut shards_total) = (0.0f64, 0.0f64, 0usize);
|
||||
let out_dir = out_dir_of(out_path);
|
||||
let mut shard_files: Vec<String> = Vec::new();
|
||||
for (number, _hash, shards, _) in &built {
|
||||
let block_t = Instant::now();
|
||||
let mut proofs: Vec<Sp1ShardProof> = Vec::with_capacity(shards.len());
|
||||
let mut shard_secs = Vec::new();
|
||||
// with --save-shards: what a shard's proof record carries (the statement, the proof's sha256, the file), so
|
||||
// the app's segment path signs and submits every shard of the chain from one run
|
||||
let mut shard_records: Vec<serde_json::Value> = Vec::new();
|
||||
for s in shards {
|
||||
let i = s.output.shard_index;
|
||||
stage(&format!("chain block {number} compressed shard {i}"));
|
||||
|
|
@ -676,12 +703,24 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_
|
|||
}
|
||||
shard_secs.push(dt);
|
||||
shard_total += dt;
|
||||
if save_shards {
|
||||
let file = out_dir.join(format!("block-{number}-shard-{i}-compressed.bin"));
|
||||
let bytes = bincode::serialize(&p.proof)?;
|
||||
std::fs::write(&file, &bytes).with_context(|| format!("write {}", file.display()))?;
|
||||
let proof_hash: [u8; 32] = sha2::Sha256::digest(&bytes).into();
|
||||
shard_records.push(serde_json::json!({
|
||||
"number": number, "block_hash": s.input.env.hash.to_string(), "shard": i, "statement": alloy_primitives::keccak256(s.output.to_bytes()).to_string(),
|
||||
"proof_sha256": format!("0x{}", hex::encode(proof_hash)), "proof_bytes": bytes.len(), "proof_file": file.display().to_string(), "prove_seconds": dt,
|
||||
}));
|
||||
shard_files.push(file.display().to_string());
|
||||
}
|
||||
proofs.push(p);
|
||||
}
|
||||
shards_total += proofs.len();
|
||||
stage(&format!("chain block {number} aggregate {} shards{}", proofs.len(), if prev.is_some() { " with the previous block proof" } else { "" }));
|
||||
let seg = sp1.aggregate(prev.as_ref(), &proofs)?;
|
||||
let adt = sp1.last_timing("aggregate").unwrap_or_default().as_secs_f64();
|
||||
let sdt = sp1.last_timing("aggregate-stdin").unwrap_or_default().as_secs_f64();
|
||||
agg_total += adt;
|
||||
let claim = SegmentClaim::from_block(&seg.output);
|
||||
let ok = sp1.verify_segment(&seg, &claim);
|
||||
|
|
@ -690,9 +729,10 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_
|
|||
let block_s = block_t.elapsed().as_secs_f64();
|
||||
let cumulative = chain_t.elapsed().as_secs_f64();
|
||||
println!(
|
||||
"RESULT chain block {number}: {} shards ({:.1} s of shard proofs), aggregate prove {adt:.1} s, proof {bytes} bytes, verify {vdt:.3} s, {}; chain_len {}, agg_vk {}; this block {block_s:.1} s, cumulative {cumulative:.1} s over {} block(s) at {}",
|
||||
"RESULT chain block {number}: {} shards ({:.1} s of shard proofs), aggregate prove {adt:.1} s (stdin {sdt:.3} s, {} deferred proofs), proof {bytes} bytes, verify {vdt:.3} s, {}; chain_len {}, agg_vk {}; this block {block_s:.1} s, cumulative {cumulative:.1} s over {} block(s) at {}",
|
||||
seg.output.shard_count,
|
||||
shard_secs.iter().sum::<f64>(),
|
||||
proofs.len() + usize::from(prev.is_some()),
|
||||
if ok { "VERIFIED (shard program id, aggregator id and claim checked)" } else { "VERIFY FAILED" },
|
||||
seg.output.chain_len,
|
||||
seg.output.agg_vk,
|
||||
|
|
@ -702,19 +742,21 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_
|
|||
if !ok {
|
||||
bail!("the aggregated proof of block {number} did not verify");
|
||||
}
|
||||
let expected_len = blocks_json.len() as u64 + 1;
|
||||
let expected_len = base_len + blocks_json.len() as u64 + 1;
|
||||
if seg.output.chain_len != expected_len {
|
||||
bail!("block {number}: chain_len {} is not {expected_len}", seg.output.chain_len);
|
||||
}
|
||||
blocks_json.push(serde_json::json!({
|
||||
"number": number, "shards": seg.output.shard_count, "shard_prove_seconds": shard_secs, "aggregate_prove_seconds": adt,
|
||||
"number": number, "shards": seg.output.shard_count, "shard_prove_seconds": shard_secs, "aggregate_prove_seconds": adt, "aggregate_stdin_seconds": sdt,
|
||||
"aggregate_verify_seconds": vdt, "proof_bytes": bytes, "chain_len": seg.output.chain_len, "block_seconds": block_s, "cumulative_seconds": cumulative,
|
||||
"post_root": seg.output.post_root.to_string(), "statement": alloy_primitives::keccak256(seg.output.to_bytes()).to_string(),
|
||||
"shard_records": shard_records,
|
||||
}));
|
||||
prev = Some(seg);
|
||||
}
|
||||
let seg = prev.unwrap();
|
||||
let total = chain_t.elapsed().as_secs_f64();
|
||||
results.insert("base_chain_len".into(), base_len.into());
|
||||
let (statement, bytes) = segment_results(&seg, &out_dir, &mut results)?;
|
||||
println!(
|
||||
"RESULT chain: {} blocks {first}..={last}, {shards_total} shards, shard proofs {shard_total:.1} s, aggregation {agg_total:.1} s, end to end {total:.1} s; final proof {bytes} bytes attests chain_len {} (statement {statement}), pre {} post {} provers {} at {}",
|
||||
|
|
@ -732,6 +774,7 @@ fn run_chain(pinned: &pinned::Pinned, fixtures: &[String], prover: Address, out_
|
|||
results.insert("shard_prove_seconds_total".into(), shard_total.into());
|
||||
results.insert("aggregate_prove_seconds_total".into(), agg_total.into());
|
||||
results.insert("chain_seconds".into(), total.into());
|
||||
results.insert("shard_proof_files".into(), shard_files.into_iter().map(serde_json::Value::from).collect::<Vec<_>>().into());
|
||||
drop(sp1);
|
||||
finish(results, out_path.map(|s| s.to_string()))
|
||||
}
|
||||
|
|
@ -794,16 +837,18 @@ fn run_aggregate(pinned: &pinned::Pinned, proofs: &str, parent: &str, prev_path:
|
|||
for shards in &blocks {
|
||||
let number = shards[0].output.number;
|
||||
stage(&format!("aggregate block {number}, {} shards", shards.len()));
|
||||
let deferred = shards.len() + usize::from(prev.is_some());
|
||||
let seg = sp1.aggregate(prev.as_ref(), shards)?;
|
||||
let adt = sp1.last_timing("aggregate").unwrap_or_default().as_secs_f64();
|
||||
let sdt = sp1.last_timing("aggregate-stdin").unwrap_or_default().as_secs_f64();
|
||||
let claim = SegmentClaim::from_block(&seg.output);
|
||||
let ok = sp1.verify_segment(&seg, &claim);
|
||||
let vdt = sp1.last_timing("verify-block").unwrap_or_default().as_secs_f64();
|
||||
println!("RESULT aggregate block {number}: {} shards, prove {adt:.1} s, proof {} bytes, verify {vdt:.3} s, {}; chain_len {}, post {} at {}", seg.output.shard_count, bincode::serialize(&seg.proof)?.len(), if ok { "VERIFIED" } else { "VERIFY FAILED" }, seg.output.chain_len, seg.output.post_root, now());
|
||||
println!("RESULT aggregate block {number}: {} shards, prove {adt:.1} s (stdin {sdt:.3} s, {deferred} deferred proofs), proof {} bytes, verify {vdt:.3} s, {}; chain_len {}, post {} at {}", seg.output.shard_count, bincode::serialize(&seg.proof)?.len(), if ok { "VERIFIED" } else { "VERIFY FAILED" }, seg.output.chain_len, seg.output.post_root, now());
|
||||
if !ok {
|
||||
bail!("the aggregated proof of block {number} did not verify");
|
||||
}
|
||||
per_block.push(serde_json::json!({ "number": number, "shards": seg.output.shard_count, "aggregate_prove_seconds": adt, "aggregate_verify_seconds": vdt, "chain_len": seg.output.chain_len }));
|
||||
per_block.push(serde_json::json!({ "number": number, "shards": seg.output.shard_count, "aggregate_prove_seconds": adt, "aggregate_stdin_seconds": sdt, "deferred_proofs": deferred, "aggregate_verify_seconds": vdt, "chain_len": seg.output.chain_len }));
|
||||
prev = Some(seg);
|
||||
}
|
||||
let seg = prev.unwrap();
|
||||
|
|
|
|||
|
|
@ -259,6 +259,16 @@ impl Sp1ProofSystem {
|
|||
self.timings.lock().unwrap().push((what.to_string(), dt));
|
||||
}
|
||||
|
||||
/// The SP1 prover knobs set in this process's environment (the GPU server inherits them; the names from
|
||||
/// sp1-core-executor 6.8.1 `opts.rs` and sp1-prover 6.8.1 `worker/config.rs`), for the RESULT lines, so a
|
||||
/// measurement names the settings it ran under. "none" when the defaults apply.
|
||||
pub fn env_knobs() -> String {
|
||||
let fixed = ["SHARD_SIZE", "ELEMENT_THRESHOLD", "HEIGHT_THRESHOLD", "FULL_SIZE_SHARDS", "MINIMAL_TRACE_CHUNK_THRESHOLD", "TRACE_CHUNK_SLOTS", "MEMORY_LIMIT", "WITHOUT_VK_VERIFICATION", "RUST_LOG"];
|
||||
let mut out: Vec<String> = std::env::vars().filter(|(k, _)| k.starts_with("SP1_WORKER_") || fixed.contains(&k.as_str())).map(|(k, v)| format!("{k}={v}")).collect();
|
||||
out.sort();
|
||||
if out.is_empty() { "none".into() } else { out.join(" ") }
|
||||
}
|
||||
|
||||
pub fn last_timing(&self, what: &str) -> Option<Duration> {
|
||||
self.timings.lock().unwrap().iter().rev().find(|(k, _)| k == what).map(|(_, d)| *d)
|
||||
}
|
||||
|
|
@ -286,6 +296,9 @@ impl ProofSystem for Sp1ProofSystem {
|
|||
/// The aggregator guest over the shard proofs (and the previous segment's proof when given), by recursion.
|
||||
fn aggregate(&self, prev: Option<&Sp1SegmentProof>, shards: &[Sp1ShardProof]) -> Result<Sp1SegmentProof> {
|
||||
let first = shards.first().ok_or_else(|| anyhow!("no shards"))?;
|
||||
// 5 October 2026 (aggregation cost): the stdin build (the proof clones into the request) is timed apart
|
||||
// from the prove call, so the host's own share of an aggregation is visible next to the GPU's.
|
||||
let t_stdin = Instant::now();
|
||||
let mut stdin = SP1Stdin::new();
|
||||
let input = AggInput {
|
||||
shard_vk: self.shard_vk_hash(),
|
||||
|
|
@ -302,6 +315,7 @@ impl ProofSystem for Sp1ProofSystem {
|
|||
let SP1Proof::Compressed(proof) = p.proof.proof.clone() else { return Err(anyhow!("the previous block proof is not a compressed proof")) };
|
||||
stdin.write_proof(*proof, self.agg_vk.vk.clone());
|
||||
}
|
||||
self.record("aggregate-stdin", t_stdin.elapsed());
|
||||
let t = Instant::now();
|
||||
let proof = self.client.prove(&self.agg_pk, stdin).compressed().run()?;
|
||||
self.record("aggregate", t.elapsed());
|
||||
|
|
|
|||
|
|
@ -34,9 +34,13 @@ if [ "${SKIP_GATE:-0}" != "1" ]; then
|
|||
fi
|
||||
H="$ROOT/proving/igneum-prove/target/release/igneum-prove-host"
|
||||
for f in "$ROOT"/proving/fixtures/block-*.json; do
|
||||
# the exporter's side files (block-N.json.node-plan.json, 5 October 2026) are not fixtures
|
||||
case "$f" in *.node-plan.json) continue ;; esac
|
||||
if ! "$H" "$f" --mode native >>"$GATE_LOG" 2>&1; then echo "GATE FAILED: native run of $(basename "$f"); see $GATE_LOG"; exit 1; fi
|
||||
done
|
||||
if ! "$ROOT/tools/lock/with-lock.sh" measure "$H" "$ROOT/proving/fixtures/block-338-shard1.json" --mode execute --shard 0 >>"$GATE_LOG" 2>&1; then
|
||||
# the execute step reports a cycle count, not a time: the `run` lock (tools/lock/with-lock.sh: counts, not ms), so the
|
||||
# gate does not queue behind every build and measurement on the Mac (5 October 2026: 25 min behind a packbench run)
|
||||
if ! "$ROOT/tools/lock/with-lock.sh" run "$H" "$ROOT/proving/fixtures/block-338-shard1.json" --mode execute --shard 0 >>"$GATE_LOG" 2>&1; then
|
||||
echo "GATE FAILED: the guest did not execute the shard fixture (the exact failure the PC hit on 4 October); see $GATE_LOG"; exit 1
|
||||
fi
|
||||
"$H" --mode id | tee -a "$GATE_LOG" # the pinned program ids this package carries (the PC's build embeds the same elf/ files)
|
||||
|
|
|
|||
50
tools/proving-v1/agg-cost-table.mjs
Normal file
50
tools/proving-v1/agg-cost-table.mjs
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
#!/usr/bin/env node
|
||||
// The aggregation-cost curve from one or more pc2-agg-cost.ps1 jobs (5 October 2026 night): per phase, the shard and
|
||||
// aggregation seconds per block (the chain and aggregate-only RESULT lines), the GPU utilisation and memory peak of
|
||||
// the phase, the own miner's rate (MH/s wall) and its batch-log2. Reads the job uploads from the log intake like
|
||||
// tools/jobs.mjs (DATABASE_URL in ~/.config/igneum/env). Prints a markdown table.
|
||||
// node tools/proving-v1/agg-cost-table.mjs agg-cost-pc2-1 agg-cost-pc2-2 ...
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
const ids = process.argv.slice(2);
|
||||
if (!ids.length) { console.error('usage: agg-cost-table.mjs <job id> [...]'); process.exit(2); }
|
||||
const m = /^DATABASE_URL=(.*)$/m.exec(readFileSync(`${homedir()}/.config/igneum/env`, 'utf8'));
|
||||
const url = m[1].trim().replace(/^['"]|['"]$/g, '');
|
||||
const host = new URL(url).hostname.replace('-pooler', '');
|
||||
const sql = async (query, params = []) => {
|
||||
const r = await fetch(`https://${host}/sql`, { method: 'POST', headers: { 'Neon-Connection-String': url, 'Content-Type': 'application/json' }, body: JSON.stringify({ query, params }) });
|
||||
const j = await r.json();
|
||||
if (!r.ok) throw new Error(j.message || JSON.stringify(j));
|
||||
return j.rows;
|
||||
};
|
||||
const f1 = x => (Math.round(x * 10) / 10).toFixed(1);
|
||||
const rows = [];
|
||||
for (const id of ids) {
|
||||
const ups = await sql('SELECT lines FROM miner_logs WHERE run_id LIKE $1 ORDER BY received_at DESC LIMIT 1', [`job-${id}-%`]);
|
||||
if (!ups.length) { console.error(`${id}: no upload`); continue; }
|
||||
const lines = ups[0].lines.split('\n');
|
||||
const phases = new Map();
|
||||
const ph = l => { if (!phases.has(l)) phases.set(l, { job: id, label: l, shard: [], agg: [], deferred: [], util: '', mem: '', rate: '', batch: '', note: '' }); return phases.get(l); };
|
||||
for (const raw of lines) {
|
||||
const l = raw.replace(/^\d+(\.\d+)? job \S+: /, '');
|
||||
let x;
|
||||
if ((x = /^([A-Z0-9]+): RESULT chain block \d+ shard \d+: compressed prove ([\d.]+) s/.exec(l))) ph(x[1]).shard.push(Number(x[2]));
|
||||
else if ((x = /^([A-Z0-9]+): RESULT (?:chain|aggregate) block \d+: .*?(?:aggregate )?prove ([\d.]+) s \(stdin [\d.]+ s, (\d+) deferred/.exec(l))) { ph(x[1]).agg.push(Number(x[2])); ph(x[1]).deferred.push(Number(x[3])); }
|
||||
else if ((x = /^RESULT phase_gpu ([A-Z0-9]+) samples=\d+ memory_used_max_mib=(\d+) util_mean_pct=([\d.]+)/.exec(l))) { ph(x[1]).mem = x[2]; ph(x[1]).util = x[3]; }
|
||||
else if ((x = /^RESULT ([A-Z0-9]+) miner rate (n=\d+ mean=[\d.]+)/.exec(l))) ph(x[1]).rate = x[2].replace('n=', 'n ').replace(' mean=', ', mean ');
|
||||
else if ((x = /^RESULT ([A-Z0-9]+) own miner started .* batch_log2=(\d+)/.exec(l))) ph(x[1]).batch = x[2];
|
||||
else if ((x = /^RESULT ([A-Z0-9]+) own miner (not hashing|EXITED|: no app miner)/.exec(l))) ph(x[1]).note = 'own miner ' + x[2];
|
||||
else if ((x = /^RESULT phase ([A-Z0-9]+) end .* exit (\d+) wall ([\d.]+) s/.exec(l))) { const p = ph(x[1]); p.exit = x[2]; p.wall = x[3]; }
|
||||
else if ((x = /^RESULT phase ([A-Z0-9]+) skipped/.exec(l))) ph(x[1]).note = 'skipped';
|
||||
else if ((x = /^RESULT (H (?:choice|knobs)): (.*)$/.exec(l))) console.error(`${id}: ${x[1]}: ${x[2]}`);
|
||||
}
|
||||
for (const p of phases.values()) rows.push(p);
|
||||
}
|
||||
const mean = a => a.length ? a.reduce((s, v) => s + v, 0) / a.length : NaN;
|
||||
console.log('| Job | Phase | batch-log2 | Shard s (each) | Aggregation s (each, deferred proofs) | Block s (shard + aggregation, mean) | GPU util % | GPU peak MiB | Miner MH/s | Note |');
|
||||
console.log('|---|---|---|---|---|---|---|---|---|---|');
|
||||
for (const p of rows) {
|
||||
const chained = p.agg.filter((_, i) => p.deferred[i] >= 2);
|
||||
const block = p.shard.length && p.agg.length ? f1(mean(p.shard) + mean(chained.length ? chained : p.agg)) : '';
|
||||
console.log(`| ${p.job} | ${p.label} | ${p.batch || (p.label.startsWith('E') && /^E\d+$/.test(p.label) ? p.label.slice(1) : '')} | ${p.shard.map(f1).join(', ')} | ${p.agg.map((a, i) => `${f1(a)} (${p.deferred[i]})`).join(', ')} | ${block} | ${p.util} | ${p.mem} | ${p.rate} | ${[p.note, p.exit && p.exit !== '0' ? `exit ${p.exit}` : ''].filter(Boolean).join('; ')} |`);
|
||||
}
|
||||
35
tools/proving-v1/miner-rate.mjs
Normal file
35
tools/proving-v1/miner-rate.mjs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
#!/usr/bin/env node
|
||||
// The hash rate of one miner over a UTC window, from its STATUS lines in the log intake (the app uploads the miner's
|
||||
// log every minute; each STATUS line starts with a unix timestamp and carries `now=<MH/s> MH/s wall`, the last
|
||||
// interval's rate). For the aggregation-cost phases that keep the app's own miner running (docs/bench-log.md,
|
||||
// 5 October 2026 night), where the job cannot read the app's state from PowerShell 5.1.
|
||||
// node tools/proving-v1/miner-rate.mjs <label> <start ISO> <end ISO> e.g. miner-nvidia-1ccfe586-1 2026-10-05T21:00:00Z 2026-10-05T21:05:00Z
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { homedir } from 'node:os';
|
||||
const [label, start, end] = process.argv.slice(2);
|
||||
if (!label || !start || !end) { console.error('usage: miner-rate.mjs <label> <start ISO> <end ISO>'); process.exit(2); }
|
||||
const m = /^DATABASE_URL=(.*)$/m.exec(readFileSync(`${homedir()}/.config/igneum/env`, 'utf8'));
|
||||
const url = m[1].trim().replace(/^['"]|['"]$/g, '');
|
||||
const host = new URL(url).hostname.replace('-pooler', '');
|
||||
const sql = async (query, params = []) => {
|
||||
const r = await fetch(`https://${host}/sql`, { method: 'POST', headers: { 'Neon-Connection-String': url, 'Content-Type': 'application/json' }, body: JSON.stringify({ query, params }) });
|
||||
const j = await r.json();
|
||||
if (!r.ok) throw new Error(j.message || JSON.stringify(j));
|
||||
return j.rows;
|
||||
};
|
||||
const t0 = Date.parse(start) / 1000, t1 = Date.parse(end) / 1000;
|
||||
// every upload received up to 15 min after the window (the log is re-uploaded whole, so the newest covering upload holds every line)
|
||||
const rows = await sql(`SELECT received_at, lines FROM miner_logs WHERE label = $1 AND received_at > to_timestamp($2) AND received_at < to_timestamp($3) ORDER BY received_at DESC LIMIT 20`, [label, t0, t1 + 900]);
|
||||
const seen = new Map();
|
||||
for (const r of rows) {
|
||||
for (const l of r.lines.split('\n')) {
|
||||
const mm = /^(\d+(?:\.\d+)?) STATUS .* now=([\d.]+) MH\/s wall/.exec(l);
|
||||
if (!mm) continue;
|
||||
const t = Number(mm[1]);
|
||||
if (t >= t0 && t <= t1) seen.set(mm[1], Number(mm[2]));
|
||||
}
|
||||
}
|
||||
const h = [...seen.values()].sort((a, b) => a - b);
|
||||
if (!h.length) { console.log(`${label} ${start}..${end}: no STATUS lines in the window (${rows.length} uploads read)`); process.exit(1); }
|
||||
const mean = h.reduce((a, b) => a + b, 0) / h.length;
|
||||
console.log(`${label} ${start}..${end}: n=${h.length} mean=${mean.toFixed(2)} p50=${h[Math.floor((h.length - 1) / 2)].toFixed(2)} min=${h[0].toFixed(2)} max=${h[h.length - 1].toFixed(2)} MH/s wall (now= of the STATUS lines, ${rows.length} uploads read)`);
|
||||
|
|
@ -37,6 +37,10 @@ const args = process.argv.slice(2);
|
|||
const flag = (name, dflt) => { const i = args.indexOf(name); return i >= 0 && args[i + 1] !== undefined ? +args[i + 1] : dflt; };
|
||||
const SECS = flag('--secs', 1200);
|
||||
const V0 = 60, V1 = flag('--v1', 120), SEG = flag('--segment', 4), UNPROVEN = flag('--unproven', 60), SHARE_BPS = 1000;
|
||||
// --fresh-rule <daa>: the fresh-record rule switch (6 October 2026); default never (the 0.3.11 rule), so case 3 keeps
|
||||
// its known-failed line; with --fresh-rule 0 a fresh record after a PENDING segment is accepted (the new rule) and
|
||||
// case 2's refusal after a PROVEN one still stands
|
||||
const FRESH_RULE = process.argv.includes('--fresh-rule') ? Number(process.argv[process.argv.indexOf('--fresh-rule') + 1]) : null;
|
||||
const started = [];
|
||||
const t0 = Date.now();
|
||||
const since = () => ((Date.now() - t0) / 1000).toFixed(1);
|
||||
|
|
@ -56,6 +60,7 @@ let overrideText = readFileSync(FILE, 'utf8')
|
|||
.replace(/"proving_v1_segment_blocks":\s*\d+/, `"proving_v1_segment_blocks": ${SEG}`)
|
||||
.replace(/"proving_v1_unproven_daa":\s*\d+/, `"proving_v1_unproven_daa": ${UNPROVEN}`)
|
||||
.replace(/"proving_v1_aggregator_share_bps":\s*\d+/, `"proving_v1_aggregator_share_bps": ${SHARE_BPS}`)
|
||||
.replace(/"proving_v1_fresh_rule_daa":\s*\d+/, FRESH_RULE === null ? '"proving_v1_fresh_rule_daa": 18446744073709551615' : `"proving_v1_fresh_rule_daa": ${FRESH_RULE}`)
|
||||
.replace(/"skip_proof_of_work":\s*(true|false)/, '"skip_proof_of_work": true');
|
||||
for (const re of [/"skip_proof_of_work": true/, new RegExp(`"proving_v1_activation_daa": ${V1}`), new RegExp(`"proving_v1_segment_blocks": ${SEG}`), new RegExp(`"proving_v1_unproven_daa": ${UNPROVEN}`)]) if (!re.test(overrideText)) throw new Error(`override edit failed: ${re}`);
|
||||
writeFileSync(override, overrideText);
|
||||
|
|
@ -211,7 +216,13 @@ try {
|
|||
const lastHash3 = stmt3early.blocks[stmt3early.blocks.length - 1].hash;
|
||||
const fresh3 = signSegment(seg3[0], seg3[1], lastHash3, stmt3early.publicValuesFresh, 'seg3-fresh');
|
||||
const subEarly = await n0.eth('igneum_submitSegmentRecord', [{ record: fresh3.record, proof: fresh3.proof }]);
|
||||
check('known-failed: segment 3 cannot start a fresh chain while segment 2 is pending', subEarly.accepted === false && /pending until DAA/.test(subEarly.reason), subEarly);
|
||||
if (FRESH_RULE === null) {
|
||||
check('known-failed: segment 3 cannot start a fresh chain while segment 2 is pending', subEarly.accepted === false && /pending until DAA/.test(subEarly.reason), subEarly);
|
||||
} else {
|
||||
check('fresh-record rule: segment 3 starts a fresh chain while segment 2 is pending', subEarly.accepted === true, subEarly);
|
||||
const stmt3now = await n0.eth('igneum_getSegmentStatement', ['0x' + seg3[0].toString(16)]);
|
||||
check('fresh-record rule: the statement reports the fresh record admissible', stmt3now.freshAdmissible === true, { freshAdmissible: stmt3now.freshAdmissible });
|
||||
}
|
||||
await waitDaa(n0, deadline2 + 2, 'past segment 2 deadline');
|
||||
const stmt2late = await n0.eth('igneum_getSegmentStatement', ['0x' + seg2[0].toString(16)]);
|
||||
check('known-failed: segment 2 is unproven after its deadline', stmt2late.status.status === 'unproven', stmt2late.status);
|
||||
|
|
@ -220,7 +231,13 @@ try {
|
|||
const subLate = await n0.eth('igneum_submitSegmentRecord', [{ record: late2.record, proof: late2.proof }]);
|
||||
check('known-failed: a late record for segment 2 pays nothing (refused as unproven)', subLate.accepted === false && /unproven/.test(subLate.reason), subLate);
|
||||
const subRestart = await n1.eth('igneum_submitSegmentRecord', [{ record: fresh3.record, proof: fresh3.proof }]);
|
||||
check('segment 3 restarts the chain with a fresh-chain record after the unproven segment', subRestart.accepted === true, subRestart);
|
||||
if (FRESH_RULE === null) {
|
||||
check('segment 3 restarts the chain with a fresh-chain record after the unproven segment', subRestart.accepted === true, subRestart);
|
||||
} else {
|
||||
// under the fresh-record rule the record was accepted while segment 2 was still pending (above) and is carried
|
||||
// by now: the second offer is a duplicate or a paid segment, never a chain-rule refusal
|
||||
check('fresh-record rule: the second offer of segment 3\'s record is a duplicate, not a chain-rule refusal', subRestart.accepted === true || !/does not chain/.test(subRestart.reason || ''), subRestart);
|
||||
}
|
||||
const paid3 = await waitSegmentPaid(n0, seg3[0]);
|
||||
check('segment 3 paid with chain_len 4', hexn(paid3.paid.chainLen) === SEG, paid3.paid);
|
||||
st = await n0.eth('igneum_getProvingStatus');
|
||||
|
|
|
|||
57
tools/proving-v1/pc2-agg-cost-restore.ps1
Normal file
57
tools/proving-v1/pc2-agg-cost-restore.ps1
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
# Aggregation cost, the restore job (5 October 2026, night): what pc2-agg-cost.ps1's finally block does, for the case
|
||||
# where the app restarted under the job (the 0.3.10 rollout; a running job dies with the app and its finally block
|
||||
# never runs): stops any miner or CUDA worker the job started itself (a process whose parent is not igneum-app.exe),
|
||||
# resumes the app's miners, switches the prover back on, resets the GPU compute-policy timeslice. Idempotent.
|
||||
$ErrorActionPreference = 'Continue'
|
||||
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
||||
$urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
||||
$base = (Get-Content $urlFile -Raw).Trim().TrimEnd('/')
|
||||
# the state as text parsed by regex (app 0.3.10's response comes back empty to PowerShell 5.1's JSON reader)
|
||||
function State {
|
||||
try {
|
||||
$raw = (Invoke-WebRequest -UseBasicParsing -Uri "$base/api/state" -TimeoutSec 20).Content
|
||||
$cards = @()
|
||||
# one card = the text from its "index" key to the next card's (the fields come in any order inside it; the first
|
||||
# version took the first "identities" AFTER "vendor", which belongs to the NEXT card: 23:16:53Z, the 5090 set to the
|
||||
# iGPU's 2 identities)
|
||||
$idx = @([regex]::Matches($raw, '\{"index":\d+,"key":"') | ForEach-Object { $_.Index })
|
||||
for ($i = 0; $i -lt $idx.Count; $i++) {
|
||||
$end = if ($i + 1 -lt $idx.Count) { $idx[$i + 1] } else { $raw.Length }
|
||||
$seg = $raw.Substring($idx[$i], $end - $idx[$i])
|
||||
$f = { param($re, $def) $x = [regex]::Match($seg, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
||||
$v = (& $f '"vendor":"([^"]*)"' '')
|
||||
if (-not $v) { continue }
|
||||
$cards += [pscustomobject]@{ key = (& $f '"key":"([^"]*)"' ''); name = (& $f '"name":"([^"]*)"' ''); vendor = $v; enabled = ((& $f '"enabled":(true|false)' 'true') -eq 'true'); identities = [int](& $f '"identities":(\d+)' '1'); state = (& $f '"state":"([^"]*)"' ''); hash_now = [double](& $f '"hash_now":([\d.eE+-]+)' '0') }
|
||||
}
|
||||
$g = { param($re, $def) $x = [regex]::Match($raw, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
||||
return [pscustomobject]@{ version = (& $g '"version":"([^"]*)"' ''); settings = [pscustomobject]@{ prove = ((& $g '"settings":\{[^}]*?"prove":(true|false)' 'false') -eq 'true') }; mining = [pscustomobject]@{ paused = ((& $g '"mining":\{[^}]*?"paused":(true|false)' 'false') -eq 'true'); cards = $cards }; proving = [pscustomobject]@{ status = (& $g '"proving":\{[^}]*?"status":"([^"]*)"' '') } }
|
||||
} catch { $null }
|
||||
}
|
||||
function Post($path, $body) { try { (Invoke-RestMethod -Method Post -Uri "$base$path" -ContentType 'application/json' -Body ($body | ConvertTo-Json -Compress -Depth 5) -TimeoutSec 15) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
||||
$st = State
|
||||
"RESULT before $(Stamp) app $($st.version) paused=$($st.mining.paused) prove=$($st.settings.prove) proving=$($st.proving.status)"
|
||||
$apps = @(Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-app*' } | ForEach-Object { $_.Id })
|
||||
"RESULT app_pids $(Stamp) $($apps -join ',')"
|
||||
$killed = 0
|
||||
foreach ($p in (Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -or $_.Name -like 'igneum-worker*' })) {
|
||||
$parent = Get-CimInstance Win32_Process -Filter "ProcessId = $($p.ParentProcessId)" -ErrorAction SilentlyContinue
|
||||
$stray = (-not $parent) -or (($parent.Name -notlike 'igneum-app*') -and ($parent.Name -notlike 'igneum-miner*'))
|
||||
if ($stray) { "RESULT stray $(Stamp) pid $($p.ProcessId) $($p.Name) parent=$($p.ParentProcessId) ($(if ($parent) { $parent.Name } else { 'gone' })): stopped"; Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue; $killed++ }
|
||||
}
|
||||
"RESULT strays_stopped $(Stamp) $killed"
|
||||
& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
||||
function Card($st) { if ($null -eq $st) { return $null }; $st.mining.cards | Where-Object { $_.vendor -eq 'nvidia' } | Select-Object -First 1 }
|
||||
$c = Card (State)
|
||||
if ($c) {
|
||||
$stc = State
|
||||
"RESULT cards_before $(Stamp) $(($stc.mining.cards | ForEach-Object { "$($_.name):enabled=$($_.enabled):identities=$($_.identities):$($_.state)" }) -join ',')"
|
||||
$list = @(); foreach ($k in $stc.mining.cards) { $list += @{ key = $k.key; enabled = $(if ($k.key -eq $c.key) { $true } else { [bool]$k.enabled }); identities = [int]$k.identities } }
|
||||
"RESULT card_on $(Stamp) $(Post '/api/cards' @{ cards = $list })"
|
||||
}
|
||||
"RESULT timeslice $(Stamp) $((& nvidia-smi compute-policy --set-timeslice=0 2>&1) -join ' ')"
|
||||
"RESULT resume $(Stamp) $(Post '/api/resume' @{})"
|
||||
"RESULT start $(Stamp) $(Post '/api/start' @{})"
|
||||
"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})"
|
||||
Start-Sleep -Seconds 20
|
||||
$st = State
|
||||
"RESULT after $(Stamp) app $($st.version) paused=$($st.mining.paused) prove=$($st.settings.prove) proving=$($st.proving.status) cards=$(($st.mining.cards | ForEach-Object { "$($_.name):$($_.state):$([math]::Round($_.hash_now,1))" }) -join ',')"
|
||||
433
tools/proving-v1/pc2-agg-cost.ps1
Normal file
433
tools/proving-v1/pc2-agg-cost.ps1
Normal file
|
|
@ -0,0 +1,433 @@
|
|||
# Aggregation cost (5 October 2026, night; Josh: "fix everything else in the numbers tonight"): what a per-block
|
||||
# aggregation on PC 2's RTX 5090 spends and what each lever gives. A signed `run` job (shell powershell, not elevated).
|
||||
# The live prover is switched OFF for the run (its sp1-gpu-server would otherwise be shared, socket /tmp/sp1-cuda-0.sock)
|
||||
# and ON again at the end; the app's miners are paused for the idle and own-miner phases and resumed at the end
|
||||
# (try/finally). The live /opt/igneum host is untouched: this build lands in /opt/igneum-aggcost.
|
||||
# Phases (the set at the top; every number is a RESULT line prefixed with the phase):
|
||||
# A0 app miner mining, chain of 1 with RUST_LOG=info: the SP1 GPU server's own log lines (the profile)
|
||||
# A app miner mining, chain of 4 (--save-shards), default knobs: tonight's baseline, shard proofs kept
|
||||
# B0 app miner mining, aggregate-only over A's shard proofs, default knobs
|
||||
# B app miner mining, aggregate-only, SP1_WORKER_VERIFY_INTERMEDIATES=false
|
||||
# C miners paused: chain of 4, default knobs (the idle baseline tonight)
|
||||
# C1 miners paused: aggregate-only over A's shard proofs
|
||||
# C2 miners paused: block-344-shards4 (four full shards, one block, --save-shards): a 4-deferred aggregation idle
|
||||
# D own miner, the app's command line, batch-log2 22 (the worker's default): chain of 4; MH/s from its STATUS lines
|
||||
# E own miner at batch-log2 20, 18, 16: chain of 4 each; MH/s each
|
||||
# E0 own miner 22: aggregate-only over C2's four shard proofs (a 4-deferred aggregation under mining)
|
||||
# F own miner 22 with nvidia-smi compute-policy --set-timeslice=1 (SHORT), chain of 4; the policy restored after
|
||||
# G0 miners paused: two host processes at once (chain of 2 each, disjoint blocks) on the one card
|
||||
# G own miner 22: two host processes at once (chain of 2 each, disjoint blocks) on the one GPU server
|
||||
# H own miner at $BestBatch with the best knobs: chain of 4 (the combination the plan adopts)
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$PhaseSet = @('A0','A','B0','B','C','C1','C2','D','E')
|
||||
$ShardSourceDir = '' # job 2: the WSL directory holding job 1's block-N-shard-i-compressed.bin and block-344 proofs ('' = this job's own)
|
||||
$ShardSourceFirst = 0 # job 2: job 1's first block number (its four shard proofs are consecutive from it)
|
||||
$ShardSourceParent = '' # job 2: the parent chain block hash of that first block
|
||||
$BestBatch = 20
|
||||
$BestKnobs = ''
|
||||
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
||||
$urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
||||
if (-not (Test-Path $urlFile)) { $urlFile = Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
||||
$base = (Get-Content $urlFile -Raw).Trim().TrimEnd('/')
|
||||
$app = if ($env:IGNEUM_APP_DIR) { $env:IGNEUM_APP_DIR } else { Join-Path $env:LOCALAPPDATA 'igneum\app' }
|
||||
# The app's state as plain text parsed by regex (job 3, 22:41Z, app 0.3.10: Invoke-RestMethod's JSON conversion came back
|
||||
# empty to PowerShell 5.1, so the card switch found no card and the own miner ran beside the app's). The card objects are
|
||||
# serialised in struct order (src/state.rs CardState: key ... vendor ... enabled, state, hash_now), so the fields after a
|
||||
# "vendor" belong to that card and the last "key" before it is its key.
|
||||
$script:stateErr = ''
|
||||
$script:nvidiaSeg = ''
|
||||
function State {
|
||||
try {
|
||||
$raw = (Invoke-WebRequest -UseBasicParsing -Uri "$base/api/state" -TimeoutSec 20).Content
|
||||
if (-not $raw) { $script:stateErr = 'empty body'; return $null }
|
||||
$cards = @()
|
||||
# one card = the text from its "index" key to the next card's (the fields come in any order inside it; the first
|
||||
# version took the first "identities" AFTER "vendor", which belongs to the NEXT card: 23:16:53Z, the 5090 set to the
|
||||
# iGPU's 2 identities)
|
||||
$idx = @([regex]::Matches($raw, '\{"index":\d+,"key":"') | ForEach-Object { $_.Index })
|
||||
for ($i = 0; $i -lt $idx.Count; $i++) {
|
||||
$end = if ($i + 1 -lt $idx.Count) { $idx[$i + 1] } else { $raw.Length }
|
||||
$seg = $raw.Substring($idx[$i], $end - $idx[$i])
|
||||
$f = { param($re, $def) $x = [regex]::Match($seg, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
||||
$v = (& $f '"vendor":"([^"]*)"' '')
|
||||
if (-not $v) { continue }
|
||||
if ($v -eq 'nvidia' -and -not $script:nvidiaSeg) { $script:nvidiaSeg = $seg.Substring(0, [Math]::Min(700, $seg.Length)) }
|
||||
$cards += [pscustomobject]@{ key = (& $f '"key":"([^"]*)"' ''); name = (& $f '"name":"([^"]*)"' ''); vendor = $v; enabled = ((& $f '"enabled":(true|false)' 'true') -eq 'true'); identities = [int](& $f '"identities":(\d+)' '1'); state = (& $f '"state":"([^"]*)"' ''); hash_now = [double](& $f '"hash_now":([\d.eE+-]+)' '0') }
|
||||
}
|
||||
$g = { param($re, $def) $x = [regex]::Match($raw, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
||||
$script:stateErr = ''
|
||||
return [pscustomobject]@{
|
||||
version = (& $g '"version":"([^"]*)"' ''); machine_id = (& $g '"machine_id":"([^"]*)"' '')
|
||||
settings = [pscustomobject]@{ prove = ((& $g '"settings":\{[^}]*?"prove":(true|false)' 'false') -eq 'true') }
|
||||
mining = [pscustomobject]@{ paused = ((& $g '"mining":\{[^}]*?"paused":(true|false)' 'false') -eq 'true'); cards = $cards }
|
||||
proving = [pscustomobject]@{ status = (& $g '"proving":\{[^}]*?"status":"([^"]*)"' '') }
|
||||
raw_len = $raw.Length
|
||||
}
|
||||
} catch { $script:stateErr = "$_"; $null }
|
||||
}
|
||||
function Post($path, $body) { try { (Invoke-RestMethod -Method Post -Uri "$base$path" -ContentType 'application/json' -Body ($body | ConvertTo-Json -Compress -Depth 5) -TimeoutSec 15) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
||||
function Rpc($port, $method, $params) {
|
||||
try { (Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$port" -ContentType 'application/json' -Body (@{jsonrpc='2.0'; id=1; method=$method; params=$params} | ConvertTo-Json -Compress -Depth 6) -TimeoutSec 180).result } catch { $null }
|
||||
}
|
||||
function Card($st) { if ($null -eq $st) { return $null }; $st.mining.cards | Where-Object { $_.vendor -eq 'nvidia' } | Select-Object -First 1 }
|
||||
function Smi { try { (& nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu,power.draw --format=csv,noheader,nounits 2>$null) -join ' | ' } catch { 'nvidia-smi failed' } }
|
||||
# The card keys come from the app's own settings.json (`cards`: key -> {enabled, identities, ...}), never from /api/state:
|
||||
# that reply is "{}" whenever the state fails to serialise (job 4, 00:18Z: raw_len=2; the app's state_json falls back
|
||||
# to an empty object), which no parser can read. The state is still read for the hash rate when it answers.
|
||||
function CardPrefs {
|
||||
$f = Join-Path $app 'settings.json'
|
||||
if (-not (Test-Path $f)) { return @{} }
|
||||
$raw = Get-Content $f -Raw
|
||||
$prefs = @{}
|
||||
$cm = [regex]::Match($raw, '"cards":\s*\{')
|
||||
if (-not $cm.Success) { return $prefs }
|
||||
$i = $cm.Index + $cm.Length; $depth = 1; $start = $i
|
||||
while ($i -lt $raw.Length -and $depth -gt 0) { $ch = $raw[$i]; if ($ch -eq '{') { $depth++ } elseif ($ch -eq '}') { $depth-- }; $i++ }
|
||||
$body = $raw.Substring($start, $i - $start - 1)
|
||||
foreach ($m in [regex]::Matches($body, '"([^"]+)":\s*\{([^}]*)\}')) {
|
||||
$k = $m.Groups[1].Value; $v = $m.Groups[2].Value
|
||||
$en = [regex]::Match($v, '"enabled":\s*(true|false)'); $id = [regex]::Match($v, '"identities":\s*(\d+)')
|
||||
$prefs[$k] = @{ enabled = $(if ($en.Success) { $en.Groups[1].Value -eq 'true' } else { $true }); identities = $(if ($id.Success) { [int]$id.Groups[1].Value } else { 1 }) }
|
||||
}
|
||||
return $prefs
|
||||
}
|
||||
function NvidiaKey { $p = CardPrefs; ($p.Keys | Where-Object { $_ -like 'nvidia:*' } | Select-Object -First 1) }
|
||||
function CardSwitch($on, $identities) {
|
||||
$p = CardPrefs; $nk = NvidiaKey
|
||||
if (-not $nk) { return "no nvidia card key in settings.json (keys: $($p.Keys -join ','))" }
|
||||
$list = @(); foreach ($k in $p.Keys) { $list += @{ key = $k; enabled = $(if ($k -eq $nk) { [bool]$on } else { [bool]$p[$k].enabled }); identities = $(if ($k -eq $nk -and $identities) { [int]$identities } else { [int]$p[$k].identities }) } }
|
||||
"prefs_before=$(($p.Keys | ForEach-Object { "$($_):enabled=$($p[$_].enabled):identities=$($p[$_].identities)" }) -join ',') " + (Post '/api/cards' @{ cards = $list })
|
||||
}
|
||||
function CudaWorkers { (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' } | Measure-Object).Count }
|
||||
$job = $env:IGNEUM_JOB_DIR
|
||||
if (-not $job) { $job = Join-Path $env:TEMP 'igneum-agg-cost' }
|
||||
New-Item -ItemType Directory -Force -Path $job | Out-Null
|
||||
$jobStart = Get-Date
|
||||
$st0 = State
|
||||
# the 5090 back to its 8 identities (the restore job of 23:16:53Z set it to 2 through the parser fault above)
|
||||
$RestoreIdentities = 8
|
||||
$p0 = CardPrefs; $nk0 = NvidiaKey
|
||||
"RESULT card_prefs $(Stamp) settings.json cards: $(($p0.Keys | ForEach-Object { "$($_):enabled=$($p0[$_].enabled):identities=$($p0[$_].identities)" }) -join ',') nvidia_key='$nk0'"
|
||||
if (-not $nk0) { "RESULT refused $(Stamp) no nvidia card key in settings.json: nothing is paused and nothing runs"; exit 1 }
|
||||
if ($p0[$nk0].identities -ne $RestoreIdentities) {
|
||||
"RESULT identities_set $(Stamp) $nk0 $($p0[$nk0].identities) -> ${RestoreIdentities}: $(CardSwitch ([bool]$p0[$nk0].enabled) $RestoreIdentities)"
|
||||
Start-Sleep -Seconds 5; $p1 = CardPrefs
|
||||
"RESULT identities_after $(Stamp) settings.json says identities=$($p1[$nk0].identities) enabled=$($p1[$nk0].enabled)"
|
||||
}
|
||||
"RESULT state_read $(Stamp) ok=$($null -ne $st0) raw_len=$($st0.raw_len) cards=$(($st0.mining.cards | Measure-Object).Count) err=$script:stateErr"
|
||||
"RESULT nvidia_card_json $(Stamp) $script:nvidiaSeg"
|
||||
if ($st0 -and ($st0.mining.cards | Measure-Object).Count -eq 0) { "RESULT state_head $(Stamp) no card parsed; the body starts: $((Invoke-WebRequest -UseBasicParsing -Uri "$base/api/state" -TimeoutSec 20).Content.Substring(0, 1500))" }
|
||||
"RESULT start $(Stamp) app $($st0.version) machine $($st0.machine_id) prove_setting=$($st0.settings.prove) proving=$($st0.proving.status) paused=$($st0.mining.paused) phases=$($PhaseSet -join ',') shard_source='$ShardSourceDir' $ShardSourceFirst $ShardSourceParent"
|
||||
"RESULT gpus $(Stamp) $(Smi)"
|
||||
$evmPort = $null
|
||||
foreach ($p in 26790, 26800, 26810) { if (Rpc $p 'igneum_getProvingStatus' @()) { $evmPort = $p; break } }
|
||||
# the NVIDIA worker must be hashing (the app's own miner) before the mining phases mean anything
|
||||
$waited = 0
|
||||
while ($waited -lt 180) {
|
||||
$c = Card (State)
|
||||
if ($c -and $c.hash_now -gt 10) { break }
|
||||
if ($waited % 60 -eq 0) { "WAIT $(Stamp) nvidia worker state=$(if ($c) { $c.state } else { 'na' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' })" }
|
||||
Start-Sleep -Seconds 15; $waited += 15
|
||||
}
|
||||
$c = Card (State)
|
||||
"RESULT miner $(Stamp) nvidia worker $(if ($c -and $c.hash_now -gt 10) { 'hashing' } else { 'NOT hashing: the app-miner phases are void' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' }) after $waited s"
|
||||
# the app's miner command line, for the own-miner phases (the same binary, args, cwd and tuning file)
|
||||
$mp = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' -and $_.CommandLine -like '*igneum-worker-cuda*' } | Select-Object -First 1
|
||||
$minerExe = $null; $minerArgs = $null
|
||||
if ($mp) {
|
||||
$minerExe = $mp.ExecutablePath
|
||||
$cl = $mp.CommandLine
|
||||
if ($cl -match '^\s*"[^"]*"\s*(.*)$') { $minerArgs = $Matches[1] } elseif ($cl -match '^\s*\S+\s+(.*)$') { $minerArgs = $Matches[1] }
|
||||
$minerArgs = $minerArgs -replace '--status-secs \d+', '--status-secs 10'
|
||||
}
|
||||
if (-not $mp) {
|
||||
# the 5090 miner is not running (job 2, 21:34Z: "off" after job 1's resume): the same command line as the app builds for
|
||||
# it (engine.rs miner_args), synthesised from the iGPU miner's line when that one runs, else from the app's install
|
||||
$amd = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' } | Select-Object -First 1
|
||||
$prog = Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner'
|
||||
$cuda = Join-Path $prog 'igneum-worker-cuda.exe'
|
||||
if ($amd -and $amd.CommandLine -match '--evm-address (0x[0-9a-fA-F]{40})') {
|
||||
$evm = $Matches[1]; $minerExe = $amd.ExecutablePath
|
||||
$rpc = if ($amd.CommandLine -match ' mine (\S+) ') { $Matches[1] } else { 'grpc://127.0.0.1:26610' }
|
||||
$minerArgs = "mine $rpc 1 100000000 nvidia-$($st0.machine_id.Substring(0,8))-1 --worker `"$cuda`" --status-secs 10 --exit-on-seed-change --evm-address $evm --identities 8 --payout-label nvidia-$($st0.machine_id.Substring(0,8))-1 --prepare-packs packs\prepare --worker-args `"--device 0 --pack packs\devnet`""
|
||||
"RESULT miner_cmdline_synthesised $(Stamp) from the iGPU miner's line; exe=$minerExe cuda_worker_exists=$(Test-Path $cuda)"
|
||||
} else { "RESULT miner_cmdline_synthesised $(Stamp) no running miner to copy from: the own-miner phases are void" }
|
||||
}
|
||||
$tuning = Join-Path $app 'tuning.json'
|
||||
if (Test-Path $tuning) { $env:IGNEUM_TUNING_FILE = $tuning }
|
||||
"RESULT miner_cmdline $(Stamp) exe=$minerExe tuning_file=$(Test-Path $tuning) args=$minerArgs"
|
||||
# strays from an earlier run that died with an app restart (job 3, 23:03Z): a miner or worker whose parent is not the
|
||||
# app (nor a miner) is ours and is stopped before anything is measured
|
||||
$killed = 0
|
||||
foreach ($p in (Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -or $_.Name -like 'igneum-worker*' })) {
|
||||
$parent = Get-CimInstance Win32_Process -Filter "ProcessId = $($p.ParentProcessId)" -ErrorAction SilentlyContinue
|
||||
if ((-not $parent) -or (($parent.Name -notlike 'igneum-app*') -and ($parent.Name -notlike 'igneum-miner*'))) { "RESULT stray $(Stamp) pid $($p.ProcessId) $($p.Name) parent=$($p.ParentProcessId): stopped"; Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue; $killed++ }
|
||||
}
|
||||
"RESULT strays_stopped $(Stamp) $killed"
|
||||
# the live prover off for the whole run: its sp1-gpu-server would be shared with ours (same socket, its environment)
|
||||
"RESULT prove_off $(Stamp) $(Post '/api/prove' @{on=$false})"
|
||||
# WSL side: the package, the export, the fixtures, the phase runner
|
||||
$pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-aggcost\igneum-prove-wsl2'
|
||||
if (-not (Test-Path $pkg)) { $pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-aggcost' }
|
||||
"RESULT package $(Stamp) $pkg exists=$(Test-Path (Join-Path $pkg 'package'))"
|
||||
function WslPath($p) { $w = (& wsl.exe -d Ubuntu-24.04 -u root -- wslpath -a ($p -replace '\\', '/') 2>$null); if ($w) { ($w -replace "`0", '').Trim() } else { '/mnt/c' + ($p.Substring(2) -replace '\\', '/') } }
|
||||
$pkgW = WslPath $pkg; $jobW = WslPath $job
|
||||
$srcW = if ($ShardSourceDir) { $ShardSourceDir.TrimEnd('/') } else { $jobW }
|
||||
$tipHex = Rpc $evmPort 'eth_blockNumber' @()
|
||||
$tip = [Convert]::ToInt64($tipHex, 16)
|
||||
$first = $tip - 30; $last = $first + 3
|
||||
"RESULT chain $(Stamp) node_evm_port=$evmPort tip=$tip blocks $first..$last"
|
||||
$t = Get-Date
|
||||
$body = (@{jsonrpc='2.0'; id=1; method='igneum_exportSegments'; params=@('0x0', ('0x{0:x}' -f $last))} | ConvertTo-Json -Compress)
|
||||
$seqFile = Join-Path $job 'seq.json'; $bodyFile = Join-Path $job 'export-request.json'
|
||||
[IO.File]::WriteAllText($bodyFile, $body, (New-Object System.Text.UTF8Encoding $false))
|
||||
& curl.exe -s -S -m 600 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bodyFile" -o $seqFile 2>&1 | ForEach-Object { "curl: $_" }
|
||||
if (-not (Test-Path $seqFile) -or (Get-Item $seqFile).Length -lt 1000) { "RESULT export FAILED: no reply file"; exit 1 }
|
||||
"RESULT export $(Stamp) $((Get-Item $seqFile).Length) bytes in $([math]::Round(((Get-Date) - $t).TotalSeconds,1)) s"
|
||||
$vars = @"
|
||||
export PKG='$pkgW'
|
||||
export JOB='$jobW'
|
||||
export SRC='$srcW'
|
||||
export FIRST=$first
|
||||
export LAST=$last
|
||||
"@
|
||||
[IO.File]::WriteAllText((Join-Path $job 'vars.sh'), ($vars -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
||||
$prep = @'
|
||||
set -uo pipefail
|
||||
. "$(dirname "$0")/vars.sh"
|
||||
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
||||
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}"
|
||||
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
DEST="$HOME/igneum-prove-aggcost"; LIVE_TARGET="$HOME/igneum-prove/proving/igneum-prove/target"
|
||||
mkdir -p "$DEST"
|
||||
rsync -a --delete --exclude target "$PKG/package/" "$DEST/"
|
||||
find "$DEST" -name target -prune -o -type f -exec touch {} + 2>/dev/null
|
||||
grep -o '"program_id": "0x[0-9a-f]*"' "$DEST/proving/igneum-prove/elf/manifest.json" | sed 's/^/RESULT manifest /'
|
||||
cd "$DEST/proving/igneum-prove"
|
||||
t0=$(date +%s)
|
||||
if ! CARGO_TARGET_DIR="$LIVE_TARGET" cargo build --release -p igneum-prove-export -p igneum-prove-host --features igneum-prove-host/cuda 2>&1 | tail -3; then echo "RESULT build FAILED"; exit 1; fi
|
||||
echo "RESULT build $(stamp) exit 0 in $(( $(date +%s) - t0 )) s"
|
||||
mkdir -p /opt/igneum-aggcost && cp "$LIVE_TARGET/release/igneum-prove-host" "$LIVE_TARGET/release/igneum-prove-export" /opt/igneum-aggcost/
|
||||
H=/opt/igneum-aggcost/igneum-prove-host; X=/opt/igneum-aggcost/igneum-prove-export
|
||||
echo "RESULT installed $(sha256sum $H | cut -c1-16) host, $(sha256sum $X | cut -c1-16) export; live /opt/igneum untouched: $(sha256sum /opt/igneum/igneum-prove-host | cut -c1-16)"
|
||||
$H --mode id | sed 's/^/RESULT aggcost-host /'
|
||||
python3 -c "import json,sys; d=json.load(open('$JOB/seq.json')); json.dump(d['result'], open('$JOB/export.json','w'))"
|
||||
LIST=""
|
||||
for n in $(seq $FIRST $LAST); do
|
||||
if ! $X "$JOB/export.json" $n "$JOB/block-$n.json" --source "PC 2 live devnet export, aggregation cost run" 2>&1 | tail -1 | sed "s/^/export $n: /"; then echo "RESULT cut $n FAILED"; exit 1; fi
|
||||
$H "$JOB/block-$n.json" --mode native 2>&1 | grep -E "^RESULT (native|plan)" | sed "s/^/block $n /"
|
||||
LIST="$LIST${LIST:+,}$JOB/block-$n.json"
|
||||
done
|
||||
echo "$LIST" > "$JOB/fixtures.txt"
|
||||
cp "$DEST/proving/fixtures/block-344-shards4.json" "$JOB/"
|
||||
# the parent hash of every fixture, for --mode aggregate
|
||||
python3 - <<PY
|
||||
import json
|
||||
for n in range($FIRST, $LAST + 1):
|
||||
f = json.load(open('$JOB/block-%d.json' % n))
|
||||
print('RESULT parent', n, f['block']['env']['parent_hash'] if 'parent_hash' in f['block']['env'] else f['block']['env'].get('parentHash'))
|
||||
open('$JOB/parent-%d.txt' % n, 'w').write(f['block']['env'].get('parent_hash') or f['block']['env'].get('parentHash'))
|
||||
f = json.load(open('$JOB/block-344-shards4.json'))
|
||||
open('$JOB/parent-344.txt', 'w').write(f['block']['env'].get('parent_hash') or f['block']['env'].get('parentHash'))
|
||||
PY
|
||||
echo "RESULT fixtures $(stamp) $LIST"
|
||||
# the live prover's GPU server must be gone (its environment and socket would be ours otherwise); and this job runs as
|
||||
# root, so the socket it leaves would be root-owned and the app's prover (another WSL user) could not connect (the
|
||||
# 21:25Z fault of job 1): the server is killed and the socket removed here and again at the end (the rule of
|
||||
# tools/ci/prover-socket-check.sh)
|
||||
for i in $(seq 1 36); do pgrep -x sp1-gpu-server >/dev/null || break; sleep 5; done
|
||||
pkill -f sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock # the root-socket class: the CI check (prover-socket-check.sh) wants pkill -f
|
||||
echo "RESULT gpu_server_before $(stamp) running=$(pgrep -x sp1-gpu-server | wc -l) socket=$(ls /tmp/sp1-cuda-0.sock 2>/dev/null || echo none)"
|
||||
'@
|
||||
$phase = @'
|
||||
# bash phase.sh <label> <cmd file>: runs the host command in the file (one line, env assignments first) with a 1-s
|
||||
# nvidia-smi sampler underneath; every RESULT/STAGE line comes out prefixed "<label>:"; a gpu line closes the phase.
|
||||
set -uo pipefail
|
||||
. "$(dirname "$0")/vars.sh"
|
||||
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
||||
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}"
|
||||
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
L="$1"; CMD="$2"
|
||||
H=/opt/igneum-aggcost/igneum-prove-host
|
||||
export H
|
||||
nvidia-smi --query-gpu=timestamp,index,memory.used,utilization.gpu,power.draw --format=csv,noheader,nounits -l 1 > "$JOB/smi-$L.csv" 2>/dev/null &
|
||||
SMI=$!
|
||||
t0=$(date +%s.%N); START=$(stamp)
|
||||
echo "RESULT phase $L start $START cmd: $(cat "$CMD" | cut -c1-400)"
|
||||
bash "$CMD" > "$JOB/host-$L.log" 2> "$JOB/host-$L.err"
|
||||
RC=$?
|
||||
WALL=$(python3 -c "import time; print(round(time.time() - $t0, 1))")
|
||||
kill $SMI 2>/dev/null; sleep 1
|
||||
grep -E "^(RESULT|STAGE|igneum-prove-host sources|segment proof written)" "$JOB/host-$L.log" | sed "s/^/$L: /"
|
||||
awk -F', *' '{ if ($3+0 > max) max=$3+0; u+=$4; n++ } END { if (n) printf "RESULT phase_gpu LABEL samples=%d memory_used_max_mib=%d util_mean_pct=%.1f\n", n, max, u/n }' "$JOB/smi-$L.csv" | sed "s/LABEL/$L/"
|
||||
echo "RESULT phase $L end $(stamp) exit $RC wall $WALL s (started $START); stderr $(wc -l < "$JOB/host-$L.err") lines"
|
||||
if [ $RC -ne 0 ]; then tail -5 "$JOB/host-$L.err" | sed "s/^/$L stderr: /"; fi
|
||||
'@
|
||||
foreach ($pair in @(@('prep.sh', $prep), @('phase.sh', $phase))) {
|
||||
[IO.File]::WriteAllText((Join-Path $job $pair[0]), ($pair[1] -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
||||
}
|
||||
function Wsl($file) { & wsl.exe -d Ubuntu-24.04 -u root -- bash (WslPath $file) 2>&1 | ForEach-Object { ($_ -replace "`0", '') } }
|
||||
function Cmd($label, $line) {
|
||||
$f = Join-Path $job "cmd-$label.sh"
|
||||
[IO.File]::WriteAllText($f, ($line + "`n"), (New-Object System.Text.UTF8Encoding $false))
|
||||
return (WslPath $f)
|
||||
}
|
||||
function RunPhase($label, $line) {
|
||||
$el = ((Get-Date) - $jobStart).TotalMinutes
|
||||
if ($el -gt 16.5) { "RESULT phase $label skipped: $([math]::Round($el,1)) min into the job (the 20-min window)"; return }
|
||||
$cmdW = Cmd $label $line
|
||||
& wsl.exe -d Ubuntu-24.04 -u root -- bash "$jobW/phase.sh" $label $cmdW 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
||||
}
|
||||
function Fixtures { (Get-Content (Join-Path $job 'fixtures.txt') -Raw).Trim() }
|
||||
function Parent($n) { (Get-Content (Join-Path $job "parent-$n.txt") -Raw).Trim() }
|
||||
# proof groups for --mode aggregate: the four blocks' shard 0 proofs from the source job (one shard a block tonight)
|
||||
function SrcFirst { if ($ShardSourceFirst -gt 0) { $ShardSourceFirst } else { $first } }
|
||||
function SrcParent { if ($ShardSourceParent) { $ShardSourceParent } else { Parent $first } }
|
||||
function AggGroups($src) { $f = SrcFirst; (($f..($f + 3)) | ForEach-Object { "$src/block-$_-shard-0-compressed.bin" }) -join ';' }
|
||||
# --- own miner ---
|
||||
$script:miner = $null
|
||||
# (job 1, 21:10Z: `if (StartMiner ...)` was always true because the function's RESULT strings are part of its output; the
|
||||
# outcome is the script-scope flag $script:minerOk now)
|
||||
function StartMiner($label, $batchLog2) {
|
||||
$script:minerOk = $false
|
||||
if (-not $minerExe) { "RESULT $label own miner: no app miner command line was captured; phase void"; return }
|
||||
$alive = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' } | Measure-Object).Count
|
||||
if ($alive -gt 0) { "RESULT $label own miner: the app's CUDA worker is still running ($alive); the phase would mine twice on the card, so it is void (job 3, 22:48Z)"; return }
|
||||
$a = $minerArgs
|
||||
$extra = "--race off"
|
||||
if ($batchLog2 -and $batchLog2 -ne 22) { $extra = "$extra --batch-log2 $batchLog2" }
|
||||
if ($a -match '--worker-args "([^"]*)"') { $a = $a -replace '--worker-args "([^"]*)"', ('--worker-args "$1 ' + $extra + '"') }
|
||||
elseif ($a -match '--worker-args (\S+)') { $a = $a -replace '--worker-args (\S+)', ('--worker-args "$1 ' + $extra + '"') }
|
||||
else { $a = $a + ' --worker-args "' + $extra + '"' }
|
||||
$out = Join-Path $job "miner-$label.out"; $err = Join-Path $job "miner-$label.err"
|
||||
$script:miner = Start-Process -FilePath $minerExe -ArgumentList $a -WorkingDirectory $app -RedirectStandardOutput $out -RedirectStandardError $err -NoNewWindow -PassThru
|
||||
"RESULT $label own miner started $(Stamp) pid $($script:miner.Id) batch_log2=$batchLog2 args=$a"
|
||||
$w = 0
|
||||
while ($w -lt 150) {
|
||||
Start-Sleep -Seconds 10; $w += 10
|
||||
$s = Get-Content $out -ErrorAction SilentlyContinue | Where-Object { $_ -match 'STATUS' } | Select-Object -Last 1
|
||||
if ($s -and $s -match 'hash=([\d.]+) MH/s' -and [double]$Matches[1] -gt 10) { "RESULT $label own miner hashing after $w s: $s"; $script:minerOk = $true; return }
|
||||
if ($script:miner.HasExited) { "RESULT $label own miner EXITED (code $($script:miner.ExitCode)); err: $((Get-Content $err -ErrorAction SilentlyContinue | Select-Object -Last 3) -join ' / ')"; return }
|
||||
}
|
||||
"RESULT $label own miner not hashing after $w s; last lines: $((Get-Content $out -ErrorAction SilentlyContinue | Select-Object -Last 3) -join ' / ')"
|
||||
}
|
||||
function StopMiner($label) {
|
||||
if (-not $script:miner) { return }
|
||||
$pid0 = $script:miner.Id
|
||||
Get-CimInstance Win32_Process | Where-Object { $_.ParentProcessId -eq $pid0 } | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }
|
||||
Stop-Process -Id $pid0 -Force -ErrorAction SilentlyContinue
|
||||
Start-Sleep -Seconds 3
|
||||
$left = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker*' -or $_.ProcessName -like 'igneum-miner*' } | Measure-Object).Count
|
||||
"RESULT $label own miner stopped $(Stamp); miner/worker processes left: $left"
|
||||
$script:miner = $null
|
||||
}
|
||||
function MinerRate($label, $skip) {
|
||||
$out = Join-Path $job "miner-$label.out"
|
||||
$h = @(); foreach ($l in (Get-Content $out -ErrorAction SilentlyContinue | Where-Object { $_ -match 'STATUS' })) { if ($l -match 'now=([\d.]+) MH/s wall') { $h += [double]$Matches[1] } }
|
||||
if ($h.Count -gt $skip) { $h = $h[$skip..($h.Count - 1)] }
|
||||
if ($h.Count -eq 0) { return "n=0" }
|
||||
$m = $h | Measure-Object -Average -Minimum -Maximum
|
||||
"n=$($h.Count) mean=$([math]::Round($m.Average,2)) min=$([math]::Round($m.Minimum,2)) max=$([math]::Round($m.Maximum,2)) MH/s wall (the now= field of the STATUS lines every 10 s, the first $skip skipped)"
|
||||
}
|
||||
$paused = $false
|
||||
try {
|
||||
Wsl (Join-Path $job 'prep.sh')
|
||||
if (-not (Test-Path (Join-Path $job 'fixtures.txt'))) { "RESULT prep FAILED: no fixtures"; exit 1 }
|
||||
$list = Fixtures
|
||||
$first2 = ($list -split ',')[0..1] -join ','; $last2 = ($list -split ',')[2..3] -join ','
|
||||
if ($PhaseSet -contains 'A0') {
|
||||
RunPhase 'A0' "SP1_PROVER=cuda RUST_LOG=info `$H --mode chain --chain $(($list -split ',')[0]) --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-A0.json"
|
||||
$errf = Join-Path $job 'host-A0.err'
|
||||
"RESULT A0 profile: $((Get-Content $errf -ErrorAction SilentlyContinue | Measure-Object).Count) stderr lines; the first 160 with a level or a duration follow"
|
||||
Get-Content $errf -ErrorAction SilentlyContinue | Where-Object { $_ -match 'INFO|WARN|DEBUG|ERROR|time\.|elapsed|took|ms\b|\bs\b' } | Select-Object -First 160 | ForEach-Object { "PROFILE $_" }
|
||||
}
|
||||
if ($PhaseSet -contains 'A') { RunPhase 'A' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --save-shards --out `$JOB/results-A.json" }
|
||||
if ($PhaseSet -contains 'B0') { RunPhase 'B0' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B0.json" }
|
||||
if ($PhaseSet -contains 'B') { RunPhase 'B' "SP1_WORKER_VERIFY_INTERMEDIATES=false SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B.json" }
|
||||
$needIdle = @('C','C1','C2','G0','D','E','E0','F','G','H') | Where-Object { $PhaseSet -contains $_ }
|
||||
if ($needIdle) {
|
||||
# the 5090 switched off through /api/cards (and on again in the finally block): /api/pause then /api/resume left
|
||||
# the worker off on 0.3.9 (21:25Z, an app defect in the resume path, fixed for 0.3.11)
|
||||
"RESULT card_off $(Stamp) $(CardSwitch $false $null)"; $paused = $true
|
||||
$w = 0; $left = 1
|
||||
while ($w -lt 120 -and $left -gt 0) { Start-Sleep -Seconds 5; $w += 5; $left = CudaWorkers }
|
||||
if ($left -gt 0) { "RESULT card_off_failed $(Stamp) the app's CUDA worker is still running after $w s ($left); /api/pause as the fallback: $(Post '/api/pause' @{})"; $w2 = 0; while ($w2 -lt 60 -and (CudaWorkers) -gt 0) { Start-Sleep -Seconds 5; $w2 += 5 }; $left = CudaWorkers }
|
||||
"RESULT paused $(Stamp) after $w s; CUDA workers left: $left; $(Smi)"
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
if ($PhaseSet -contains 'C') { RunPhase 'C' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-C.json" }
|
||||
if ($PhaseSet -contains 'C1') { RunPhase 'C1' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-C1.json" }
|
||||
if ($PhaseSet -contains 'C2') { RunPhase 'C2' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain `$JOB/block-344-shards4.json --prover 0xCAfc6e74000000000000000000000000000000c2 --save-shards --out `$JOB/results-C2.json" }
|
||||
if ($PhaseSet -contains 'G0') {
|
||||
$c1 = Cmd 'G01' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $first2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G01.json"
|
||||
$c2 = Cmd 'G02' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $last2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G02.json"
|
||||
$p1 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G01 $c1" -RedirectStandardOutput (Join-Path $job 'G01.out') -NoNewWindow -PassThru
|
||||
Start-Sleep -Seconds 2
|
||||
$p2 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G02 $c2" -RedirectStandardOutput (Join-Path $job 'G02.out') -NoNewWindow -PassThru
|
||||
$p1.WaitForExit(); $p2.WaitForExit()
|
||||
foreach ($f in 'G01.out', 'G02.out') { ((Get-Content (Join-Path $job $f) -Raw -ErrorAction SilentlyContinue) -replace "`0", '') -split "`n" | ForEach-Object { $_.TrimEnd("`r") } | Where-Object { $_ } }
|
||||
}
|
||||
if ($PhaseSet -contains 'D') {
|
||||
StartMiner 'D' 22; if ($script:minerOk) { RunPhase 'D' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-D.json"; "RESULT D miner rate $(MinerRate 'D' 2)" }
|
||||
StopMiner 'D'
|
||||
}
|
||||
if ($PhaseSet -contains 'E') {
|
||||
foreach ($b in 20, 18, 16) {
|
||||
$lab = "E$b"
|
||||
StartMiner $lab $b; if ($script:minerOk) { RunPhase $lab "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-$lab.json"; "RESULT $lab miner rate $(MinerRate $lab 2)" }
|
||||
StopMiner $lab
|
||||
}
|
||||
}
|
||||
if ($PhaseSet -contains 'E0') {
|
||||
$g = (0..3 | ForEach-Object { "$srcW/block-344-shard-$_-compressed.bin" }) -join ','
|
||||
StartMiner 'E0' 22; if ($script:minerOk) { RunPhase 'E0' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$g' --parent $(Parent 344) --out `$JOB/results-E0.json"; "RESULT E0 miner rate $(MinerRate 'E0' 2)" }
|
||||
StopMiner 'E0'
|
||||
}
|
||||
if ($PhaseSet -contains 'F') {
|
||||
$pol = (& nvidia-smi compute-policy --set-timeslice=1 2>&1) -join ' '
|
||||
"RESULT F timeslice set: $pol"
|
||||
if ($pol -notmatch 'rror|nsufficient|not supported|Unknown') {
|
||||
StartMiner 'F' 22; if ($script:minerOk) { RunPhase 'F' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-F.json"; "RESULT F miner rate $(MinerRate 'F' 2)" }
|
||||
StopMiner 'F'
|
||||
}
|
||||
"RESULT F timeslice restored: $((& nvidia-smi compute-policy --set-timeslice=0 2>&1) -join ' ')"
|
||||
}
|
||||
if ($PhaseSet -contains 'G') {
|
||||
StartMiner 'G' 22; if ($script:minerOk) {
|
||||
$c1 = Cmd 'G1' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $first2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G1.json"
|
||||
$c2 = Cmd 'G2' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $last2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G2.json"
|
||||
$p1 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G1 $c1" -RedirectStandardOutput (Join-Path $job 'G1.out') -NoNewWindow -PassThru
|
||||
Start-Sleep -Seconds 2
|
||||
$p2 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G2 $c2" -RedirectStandardOutput (Join-Path $job 'G2.out') -NoNewWindow -PassThru
|
||||
$p1.WaitForExit(); $p2.WaitForExit()
|
||||
foreach ($f in 'G1.out', 'G2.out') { ((Get-Content (Join-Path $job $f) -Raw -ErrorAction SilentlyContinue) -replace "`0", '') -split "`n" | ForEach-Object { $_.TrimEnd("`r") } | Where-Object { $_ } }
|
||||
"RESULT G miner rate $(MinerRate 'G' 2)"
|
||||
}
|
||||
StopMiner 'G'
|
||||
}
|
||||
if ($PhaseSet -contains 'H') {
|
||||
# the combination from this job's own phases: the batch-log2 with the shortest chain among D and E (the miner's
|
||||
# rate alongside), the knob when phase B beat B0 by 5% or more; skipped when the job is past 15 min (the window)
|
||||
try {
|
||||
$cands = @(); foreach ($lab in 'D', 'E20', 'E18', 'E16') { $f = Join-Path $job "results-$lab.json"; if (Test-Path $f) { $r = (Get-Content $f -Raw | ConvertFrom-Json); $cands += [pscustomobject]@{ lab = $lab; batch = $(if ($lab -eq 'D') { 22 } else { [int]$lab.Substring(1) }); secs = [double]$r.chain_seconds } } }
|
||||
if ($cands.Count -gt 0) { $bestC = $cands | Sort-Object secs | Select-Object -First 1; $BestBatch = $bestC.batch; "RESULT H choice: $(($cands | ForEach-Object { "$($_.lab)=$([math]::Round($_.secs,1))s" }) -join ' ') -> batch_log2 $BestBatch" }
|
||||
$b0 = Join-Path $job 'results-B0.json'; $b1 = Join-Path $job 'results-B.json'
|
||||
if ((Test-Path $b0) -and (Test-Path $b1)) { $s0 = [double](Get-Content $b0 -Raw | ConvertFrom-Json).aggregate_seconds; $s1 = [double](Get-Content $b1 -Raw | ConvertFrom-Json).aggregate_seconds; if ($s1 -lt 0.95 * $s0) { $BestKnobs = 'SP1_WORKER_VERIFY_INTERMEDIATES=false' }; "RESULT H knobs: B0 $([math]::Round($s0,1)) s, B $([math]::Round($s1,1)) s -> '$BestKnobs'" }
|
||||
} catch { "RESULT H choice error: $_" }
|
||||
$elapsedMin = ((Get-Date) - $jobStart).TotalMinutes
|
||||
if ($elapsedMin -gt 15) { "RESULT H skipped: $([math]::Round($elapsedMin,1)) min into the job (the 20-min window)" }
|
||||
else { StartMiner 'H' $BestBatch }
|
||||
if ($elapsedMin -le 15 -and $script:minerOk) { RunPhase 'H' "$BestKnobs SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-H.json"; "RESULT H miner rate $(MinerRate 'H' 2)" }
|
||||
StopMiner 'H'
|
||||
}
|
||||
} finally {
|
||||
StopMiner 'final'
|
||||
& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
||||
if ($paused) {
|
||||
"RESULT card_on $(Stamp) $(CardSwitch $true $null)"; "RESULT resume $(Stamp) $(Post '/api/resume' @{})"
|
||||
$w3 = 0; while ($w3 -lt 90 -and (CudaWorkers) -eq 0) { Start-Sleep -Seconds 5; $w3 += 5 }
|
||||
if ((CudaWorkers) -eq 0) { "RESULT miner_back_failed $(Stamp) no CUDA worker after $w3 s; /api/start: $(Post '/api/start' @{})" } else { "RESULT miner_back $(Stamp) CUDA worker running after $w3 s" }
|
||||
}
|
||||
"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})"
|
||||
"RESULT end $(Stamp) $(Smi)"
|
||||
}
|
||||
342
tools/proving-v1/pc2-segments.ps1
Normal file
342
tools/proving-v1/pc2-segments.ps1
Normal file
|
|
@ -0,0 +1,342 @@
|
|||
# Proving v1, the segment-aligned prover on PC 2 (6 October 2026, Josh: "proving v1 is active but has proven zero
|
||||
# segments, and cannot with one prover; find a way to solve this"). A signed `run` job (shell powershell, not
|
||||
# elevated; the app keeps mining; the app's own prover is switched OFF for the run through /api/prove and ON again
|
||||
# at the end; the live /opt/igneum host is untouched: this build lands in /opt/igneum-segal).
|
||||
#
|
||||
# What it does, for RUN_MINUTES, exactly what the app's segment path (app/igneum-app/src/prover.rs, pick_segment and
|
||||
# prove_segment) does:
|
||||
# 1. the node's v1 status and the work list (igneum_getAssignedShards, lookback 600) grouped into whole untouched
|
||||
# segments (every block present, every shard open, unpaid, not in our pool); the newest one inside its deadline by
|
||||
# the margin (240 DAA, or 1.5x the last segment's time) is claimed; the app ranks by FNV of (first, key) for the
|
||||
# multi-prover spread, which with one prover changes nothing but the order
|
||||
# 2. the segment statement: executed, pending; fresh when the previous segment is not paid and no verified record of
|
||||
# it waits in the pool, else chained to the previous proof (--prev) when the pool holds it
|
||||
# 3. one export (igneum_exportSegments 0..last), one fixture per block, one host run (--mode chain --save-shards
|
||||
# [--prev]) with a 1-s nvidia-smi sampler underneath
|
||||
# 4. every shard record signed (igneum-miner.exe sign-record) and submitted (igneum_submitProofRecord), then the
|
||||
# segment record (sign-segment-record, igneum_submitSegmentRecord)
|
||||
# 5. the paid state of every submitted segment polled each pass (igneum_getSegmentRecords: carried, paid)
|
||||
# Every number is a RESULT line. The miner's hash rate comes from the app log's "status: ... MH/s" lines (every 30 s),
|
||||
# the 30 minutes before the job against the run. Card keys, when needed, come from settings.json, never /api/state
|
||||
# (that reply is "{}" on 0.3.11 once paid_wei passes u64::MAX; fixed in 6714a45).
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$RunMinutes = 30
|
||||
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
||||
$app = if ($env:IGNEUM_APP_DIR) { $env:IGNEUM_APP_DIR } else { Join-Path $env:LOCALAPPDATA 'igneum\app' }
|
||||
$urlFile = Join-Path $app 'app.url'
|
||||
$base = if (Test-Path $urlFile) { (Get-Content $urlFile -Raw).Trim().TrimEnd('/') } else { '' }
|
||||
function Post($path, $obj) { try { (Invoke-RestMethod -Method Post -Uri "$base$path" -ContentType 'application/json' -Body ($obj | ConvertTo-Json -Compress) -TimeoutSec 10) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
||||
function Rpc($method, $params) {
|
||||
try { (Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$evmPort" -ContentType 'application/json' -Body (@{jsonrpc='2.0'; id=1; method=$method; params=$params} | ConvertTo-Json -Compress -Depth 8) -TimeoutSec 120).result } catch { $script:rpcErr = "$_"; $null }
|
||||
}
|
||||
function RpcRaw($method, $params) {
|
||||
try { Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$evmPort" -ContentType 'application/json' -Body (@{jsonrpc='2.0'; id=1; method=$method; params=$params} | ConvertTo-Json -Compress -Depth 8) -TimeoutSec 120 } catch { $script:rpcErr = "$_"; $null }
|
||||
}
|
||||
function Hex($v) { if ($null -eq $v) { return 0 }; if ($v -is [string] -and $v.StartsWith('0x')) { return [Convert]::ToInt64($v.Substring(2), 16) }; return [int64]$v }
|
||||
$job = $env:IGNEUM_JOB_DIR
|
||||
if (-not $job) { $job = Join-Path $env:TEMP 'igneum-segments' }
|
||||
New-Item -ItemType Directory -Force -Path $job | Out-Null
|
||||
$jobStart = Get-Date
|
||||
"RESULT start $(Stamp) app_dir=$app base_len=$($base.Length) job=$job run_minutes=$RunMinutes"
|
||||
$evmPort = $null
|
||||
foreach ($p in 26790, 26800, 26810) { $evmPort = $p; if (Rpc 'igneum_getProvingStatus' @()) { break }; $evmPort = $null }
|
||||
if (-not $evmPort) { "RESULT refused $(Stamp) no node EVM RPC on 26790/26800/26810: $script:rpcErr"; exit 1 }
|
||||
# the app's miner (the signer) and its label, the payout address
|
||||
$mp = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' } | Select-Object -First 1
|
||||
$minerExe = if ($mp) { $mp.ExecutablePath } else { $null }
|
||||
if (-not $minerExe -or -not (Test-Path $minerExe)) {
|
||||
$cand = Get-ChildItem -Path (Split-Path $app -Parent) -Recurse -Filter 'igneum-miner.exe' -ErrorAction SilentlyContinue | Select-Object -First 1
|
||||
if ($cand) { $minerExe = $cand.FullName }
|
||||
}
|
||||
if (-not $minerExe) { "RESULT refused $(Stamp) no igneum-miner.exe found (the signer)"; exit 1 }
|
||||
$settingsRaw = Get-Content (Join-Path $app 'settings.json') -Raw
|
||||
$payout = [regex]::Match($settingsRaw, '"address":\s*"(0x[0-9a-fA-F]{40})"').Groups[1].Value
|
||||
if ($payout.Length -ne 42) { "RESULT refused $(Stamp) no payout address in settings.json"; exit 1 }
|
||||
$identities = 1; $im = [regex]::Match($settingsRaw, '"nvidia:[^"]*":\s*\{[^}]*"identities":\s*(\d+)'); if ($im.Success) { $identities = [int]$im.Groups[1].Value }
|
||||
$machineId = ''; $mm = [regex]::Match($settingsRaw, '"machine_id":\s*"([0-9a-f]+)"'); if ($mm.Success) { $machineId = $mm.Groups[1].Value }
|
||||
if (-not $machineId) { $mf = Join-Path $app 'machine-id'; if (Test-Path $mf) { $machineId = (Get-Content $mf -Raw).Trim() } }
|
||||
if (-not $machineId -and $job -match '([0-9a-f]{8})') { $machineId = $Matches[1] }
|
||||
$id8 = if ($machineId.Length -ge 8) { $machineId.Substring(0, 8) } else { '1ccfe586' }
|
||||
$label = if ($identities -gt 1) { "win-$id8-1-1" } else { "win-$id8-1" }
|
||||
$chain = 'igneum-devnet' # the app's chain_name with no devnet suffix (IGNEUM_APP_DEVNET_SUFFIX unset on the fleet)
|
||||
$keyHash = (& $minerExe key-hash $label 2>$null | Select-Object -Last 1).Trim()
|
||||
if ($keyHash -and -not $keyHash.StartsWith('0x')) { $keyHash = "0x$keyHash" }
|
||||
"RESULT signer $(Stamp) miner=$minerExe label=$label key=$keyHash chain=$chain payout=$payout identities=$identities"
|
||||
if (-not $keyHash -or $keyHash.Length -lt 64) { "RESULT refused $(Stamp) key-hash gave nothing"; exit 1 }
|
||||
# the live prover off for the run: its sp1-gpu-server would be shared with ours (same socket, its environment)
|
||||
"RESULT prove_off $(Stamp) $(Post '/api/prove' @{on=$false})"
|
||||
Start-Sleep -Seconds 5
|
||||
# --- WSL side: the package built into /opt/igneum-segal (root, the warm target of the live build), the segment runner
|
||||
$pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-segal\igneum-prove-wsl2'
|
||||
if (-not (Test-Path $pkg)) { $pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-segal' }
|
||||
"RESULT package $(Stamp) $pkg exists=$(Test-Path (Join-Path $pkg 'package'))"
|
||||
function WslPath($p) { $w = (& wsl.exe -d Ubuntu-24.04 -u root -- wslpath -a ($p -replace '\\', '/') 2>$null); if ($w) { ($w -replace "`0", '').Trim() } else { '/mnt/c' + ($p.Substring(2) -replace '\\', '/') } }
|
||||
$pkgW = WslPath $pkg; $jobW = WslPath $job
|
||||
$prep = @'
|
||||
set -uo pipefail
|
||||
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
||||
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}"
|
||||
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
PKG="$1"
|
||||
DEST="$HOME/igneum-prove-segal"; LIVE_TARGET="$HOME/igneum-prove/proving/igneum-prove/target"
|
||||
mkdir -p "$DEST"
|
||||
rsync -a --delete --exclude target "$PKG/package/" "$DEST/"
|
||||
find "$DEST" -name target -prune -o -type f -exec touch {} + 2>/dev/null
|
||||
grep -o '"program_id": "0x[0-9a-f]*"' "$DEST/proving/igneum-prove/elf/manifest.json" | sed 's/^/RESULT manifest /'
|
||||
cd "$DEST/proving/igneum-prove"
|
||||
t0=$(date +%s)
|
||||
if ! CARGO_TARGET_DIR="$LIVE_TARGET" cargo build --release -p igneum-prove-export -p igneum-prove-host --features igneum-prove-host/cuda 2>&1 | tail -3; then echo "RESULT build FAILED"; exit 1; fi
|
||||
echo "RESULT build $(stamp) exit 0 in $(( $(date +%s) - t0 )) s"
|
||||
mkdir -p /opt/igneum-segal && cp "$LIVE_TARGET/release/igneum-prove-host" "$LIVE_TARGET/release/igneum-prove-export" /opt/igneum-segal/
|
||||
H=/opt/igneum-segal/igneum-prove-host; X=/opt/igneum-segal/igneum-prove-export
|
||||
echo "RESULT installed $(sha256sum $H | cut -c1-16) host, $(sha256sum $X | cut -c1-16) export; live /opt/igneum untouched: $(sha256sum /opt/igneum/igneum-prove-host | cut -c1-16)"
|
||||
$H --mode id | sed 's/^/RESULT segal-host /'
|
||||
echo "RESULT gpu_server_before $(stamp) running=$(pgrep -x sp1-gpu-server | wc -l) socket=$(ls -la /tmp/sp1-cuda-0.sock 2>/dev/null || echo none)"
|
||||
# the root-socket rule (tools/ci/prover-socket-check.sh): this run is root; the app's prover is off, so its server goes too
|
||||
pkill -f sp1-gpu-server; sleep 1; rm -f /tmp/sp1-cuda-*.sock
|
||||
echo "RESULT socket_start $(stamp) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"
|
||||
'@
|
||||
# bash seg.sh <job dir> <first> <last> <payout> [prev file]: unwrap the export, cut the blocks, run the chain with the sampler
|
||||
$seg = @'
|
||||
set -uo pipefail
|
||||
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
||||
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}"
|
||||
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||
JOB="$1"; FIRST="$2"; LAST="$3"; PAYOUT="$4"; PREV="${5:-}"
|
||||
H=/opt/igneum-segal/igneum-prove-host; X=/opt/igneum-segal/igneum-prove-export
|
||||
D="$JOB/seg-$FIRST"; mkdir -p "$D"
|
||||
t0=$(date +%s.%N)
|
||||
python3 -c "import json; d=json.load(open('$D/seq.json')); json.dump(d['result'], open('$D/export.json','w'))" || { echo "RESULT seg $FIRST unwrap FAILED"; exit 1; }
|
||||
LIST=""
|
||||
for n in $(seq $FIRST $LAST); do
|
||||
if ! $X "$D/export.json" $n "$D/block-$n.json" --source "PC 2 live devnet, segment-aligned prover" >"$D/export-$n.log" 2>&1; then echo "RESULT seg $FIRST cut $n FAILED: $(tail -1 "$D/export-$n.log")"; exit 1; fi
|
||||
LIST="$LIST${LIST:+,}$D/block-$n.json"
|
||||
done
|
||||
CUT=$(python3 -c "import time; print(round(time.time() - $t0, 1))")
|
||||
echo "RESULT seg $FIRST cut $(stamp) $LAST blocks in $CUT s"
|
||||
nvidia-smi --query-gpu=timestamp,index,memory.used,utilization.gpu,power.draw --format=csv,noheader,nounits -l 1 > "$D/smi.csv" 2>/dev/null &
|
||||
SMI=$!
|
||||
t1=$(date +%s.%N)
|
||||
PREVARG=""; [ -n "$PREV" ] && PREVARG="--prev $PREV"
|
||||
SP1_PROVER=cuda RUST_LOG=off $H --mode chain --chain "$LIST" --prover "$PAYOUT" --save-shards $PREVARG --out "$D/chain-results.json" >"$D/chain.log" 2>"$D/chain.err"
|
||||
RC=$?
|
||||
WALL=$(python3 -c "import time; print(round(time.time() - $t1, 1))")
|
||||
kill $SMI 2>/dev/null; sleep 1
|
||||
grep -E "^RESULT (chain block|chain:|chain prev|setup)" "$D/chain.log" | sed "s/^/seg $FIRST: /"
|
||||
awk -F', *' '{ if ($3+0 > max) max=$3+0; u+=$4; n++ } END { if (n) printf "RESULT seg FIRST gpu samples=%d memory_used_max_mib=%d util_mean_pct=%.1f\n", n, max, u/n }' "$D/smi.csv" | sed "s/FIRST/$FIRST/"
|
||||
echo "RESULT seg $FIRST chain $(stamp) exit $RC wall $WALL s"
|
||||
if [ $RC -ne 0 ]; then tail -4 "$D/chain.err" | sed "s/^/seg $FIRST stderr: /"; exit 1; fi
|
||||
# the submit bodies: one JSON-RPC request per shard record and one for the segment, the proof bytes as hex
|
||||
python3 - "$D" <<'PY'
|
||||
import json, sys, os
|
||||
d = sys.argv[1]
|
||||
r = json.load(open(os.path.join(d, 'chain-results.json')))
|
||||
recs = []
|
||||
for b in r['blocks']:
|
||||
for s in b.get('shard_records', []):
|
||||
recs.append(s)
|
||||
json.dump(recs, open(os.path.join(d, 'shard-records.json'), 'w'))
|
||||
print('RESULT seg %s records %d shard records, segment chain_len %s, proof %s bytes' % (r['first'], len(recs), r['segment_chain_len'], r['segment_proof_bytes']))
|
||||
PY
|
||||
'@
|
||||
# bash body.sh <proof file> <record hex> <method> <out file>: the JSON-RPC body with the proof as hex
|
||||
$body = @'
|
||||
set -u
|
||||
python3 - "$1" "$2" "$3" "$4" <<'PY'
|
||||
import json, sys, binascii
|
||||
proof = '0x' + binascii.hexlify(open(sys.argv[1], 'rb').read()).decode()
|
||||
json.dump({'jsonrpc': '2.0', 'id': 1, 'method': sys.argv[3], 'params': [{'record': sys.argv[2], 'proof': proof}]}, open(sys.argv[4], 'w'))
|
||||
PY
|
||||
'@
|
||||
# bash unhex.sh <hex file> <out file>: a hex string (0x...) to bytes
|
||||
$unhex = @'
|
||||
set -u
|
||||
python3 -c "import sys,binascii; h=open(sys.argv[1]).read().strip(); h=h[2:] if h.startswith('0x') else h; open(sys.argv[2],'wb').write(binascii.unhexlify(h))" "$1" "$2"
|
||||
'@
|
||||
foreach ($pair in @(@('prep.sh', $prep), @('seg.sh', $seg), @('body.sh', $body), @('unhex.sh', $unhex))) {
|
||||
[IO.File]::WriteAllText((Join-Path $job $pair[0]), ($pair[1] -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
||||
}
|
||||
function Wsl { param([string[]]$a) & wsl.exe -d Ubuntu-24.04 -u root -- bash @a 2>&1 | ForEach-Object { ($_ -replace "`0", '') } }
|
||||
Wsl @("$jobW/prep.sh", $pkgW)
|
||||
if (-not (Test-Path (Join-Path $job 'prep.sh'))) { "RESULT refused $(Stamp) prep.sh missing"; exit 1 }
|
||||
# --- the app log's miner rate before the run (baseline): "status: ... <x> MH/s" every 30 s
|
||||
$logDir = Join-Path $app 'logs'
|
||||
function RateLines($since, $until) {
|
||||
$out = @()
|
||||
$files = @(Get-ChildItem -Path $logDir, (Join-Path (Split-Path $app -Parent) 'logs'), $app -Filter '*.log' -ErrorAction SilentlyContinue | Where-Object { $_.LastWriteTime -gt (Get-Date).AddHours(-3) } | Sort-Object LastWriteTime -Descending | Select-Object -First 4)
|
||||
if ($files.Count -eq 0) { "RESULT rate_files none under $logDir, $(Join-Path (Split-Path $app -Parent) 'logs'), $app" }
|
||||
foreach ($f in $files) {
|
||||
foreach ($l in (Get-Content $f.FullName -ErrorAction SilentlyContinue | Where-Object { $_ -match '^(\d{10}) status: .* ([\d.]+) MH/s' })) {
|
||||
$t = [int64]$Matches[1]; if ($t -ge $since -and $t -le $until) { $out += [double]$Matches[2] }
|
||||
}
|
||||
}
|
||||
return $out
|
||||
}
|
||||
function Epoch($d) { [int64](($d.ToUniversalTime()) - (Get-Date '1970-01-01')).TotalSeconds }
|
||||
$t0 = Epoch $jobStart
|
||||
$before = RateLines ($t0 - 1800) $t0
|
||||
if ($before.Count) { $m = $before | Measure-Object -Average -Minimum -Maximum; "RESULT rate_before $(Stamp) n=$($before.Count) mean=$([math]::Round($m.Average,2)) min=$($m.Minimum) max=$($m.Maximum) MH/s (the app log's status lines, the 30 min before the job)" } else { "RESULT rate_before $(Stamp) no status lines found in $logDir" }
|
||||
# --- the loop
|
||||
$submittedSegments = @{} # first -> @{last; aggWei; at}
|
||||
$attempted = @{}
|
||||
$lastSegSecs = 0
|
||||
$n = 1; $start = 0; $unproven = 600
|
||||
$passes = 0; $claimed = 0; $submitted = 0; $shardsAccepted = 0; $shardsRefused = 0
|
||||
$runStart = Get-Date
|
||||
function Candidates {
|
||||
$st = Rpc 'igneum_getProvingStatus' @()
|
||||
if (-not $st -or -not $st.v1 -or -not $st.v1.active -or -not $st.v1.start) { return @() }
|
||||
$script:start = Hex $st.v1.start; $script:n = [math]::Max(1, (Hex $st.v1.segmentBlocks)); $script:unproven = Hex $st.v1.unprovenDaa
|
||||
$tipDaa = Hex $st.tipDaa
|
||||
$work = Rpc 'igneum_getAssignedShards' @(@($keyHash), 600)
|
||||
if (-not $work) { return @() }
|
||||
$by = @{}
|
||||
foreach ($w in $work) { [int64]$num = Hex $w.number; if ($num -lt $script:start) { continue }; if (-not $by.ContainsKey($num)) { $by[$num] = @() }; $by[$num] += $w }
|
||||
"RESULT worklist $(Stamp) entries=$(@($work).Count) blocks=$($by.Count) start=$($script:start) n=$($script:n) tipDaa=$tipDaa"
|
||||
$need = [math]::Max(240, [math]::Ceiling($lastSegSecs * 1.5))
|
||||
$segs = @()
|
||||
$seen = @{}
|
||||
foreach ($num in ($by.Keys | Sort-Object)) {
|
||||
# [int64] throughout: [math]::Floor gives a double, and a double never matches an int64 hashtable key (job b,
|
||||
# 07:20Z to 07:50Z: every segment "not whole", 88 passes, nothing claimed)
|
||||
[int64]$k = [math]::Floor(($num - $script:start) / $script:n); [int64]$first = $script:start + $k * $script:n; [int64]$last = $first + $script:n - 1
|
||||
if ($seen.ContainsKey($first)) { continue }; $seen[$first] = $true
|
||||
if ($attempted.ContainsKey($first)) { continue }
|
||||
$whole = $true; $shards = @(); $lastDaa = 0; $wei = 0
|
||||
for ([int64]$b = $first; $b -le $last; $b++) {
|
||||
if (-not $by.ContainsKey($b)) { $whole = $false; break }
|
||||
$es = $by[$b] | Sort-Object { $_.shard } -Unique
|
||||
foreach ($e in $es) {
|
||||
$inPool = ($e.pool -and (@($e.pool)).Count -gt 0)
|
||||
if (-not $e.open -or $null -ne $e.paid -or $inPool) { $whole = $false; break }
|
||||
$shards += @{ number = $b; hash = $e.hash; shard = [int]$e.shard }
|
||||
if ($b -eq $last) { $lastDaa = Hex $e.daaScore }
|
||||
}
|
||||
if (-not $whole) { break }
|
||||
}
|
||||
if ($whole -and $shards.Count -gt 0) {
|
||||
$deadline = $lastDaa + $script:unproven
|
||||
if ($deadline -ge ($tipDaa + 1 + $need)) { $segs += @{ first = $first; last = $last; lastDaa = $lastDaa; deadline = $deadline; shards = $shards; margin = [int64]($deadline - $tipDaa - 1) } }
|
||||
}
|
||||
}
|
||||
# newest first (the most time before the deadline); the app ranks by FNV(first, key) for the spread
|
||||
return @($segs | Sort-Object { $_.first } -Descending)
|
||||
}
|
||||
while (((Get-Date) - $runStart).TotalMinutes -lt $RunMinutes) {
|
||||
$passes++
|
||||
# paid segments
|
||||
foreach ($f in @($submittedSegments.Keys)) {
|
||||
$rec = Rpc 'igneum_getSegmentRecords' @(('0x{0:x}' -f $f))
|
||||
if ($rec) {
|
||||
$carried = (@($rec.carried)).Count
|
||||
if ($rec.paid) { "RESULT paid $(Stamp) segment $f..$($submittedSegments[$f].last) paid wei=$($rec.paid.wei) ($([math]::Round((Hex $rec.paid.wei) / 1e18, 4)) IGN) carrier=$(Hex $rec.paid.carrierNumber) payout=$($rec.paid.payout) after $([math]::Round(((Get-Date) - $submittedSegments[$f].at).TotalSeconds)) s"; $submittedSegments.Remove($f) }
|
||||
elseif ($carried -gt 0) { "RESULT carried $(Stamp) segment $f carried=$carried valid=$(@($rec.carried | ForEach-Object { $_.valid }) -join ',') rejected=$(@($rec.carried | ForEach-Object { $_.rejected }) -join ';')" }
|
||||
}
|
||||
}
|
||||
$cands = Candidates
|
||||
if (-not $cands -or $cands.Count -eq 0) { "RESULT pass $passes $(Stamp) no whole segment inside the margin (need=$([math]::Max(240, [math]::Ceiling($lastSegSecs * 1.5))) DAA); waiting 20 s"; Start-Sleep -Seconds 20; continue }
|
||||
$picked = $null; $prevFile = $null; $expected = ''
|
||||
$tries = 0
|
||||
foreach ($c in $cands) {
|
||||
if ($tries -ge 3) { break }; $tries++
|
||||
$stmt = Rpc 'igneum_getSegmentStatement' @(('0x{0:x}' -f $c.first))
|
||||
if (-not $stmt -or -not $stmt.executed -or $stmt.status.status -ne 'pending') { $attempted[$c.first] = $true; "RESULT skip $(Stamp) segment $($c.first): executed=$($stmt.executed) status=$($stmt.status.status)"; continue }
|
||||
if ($null -eq $stmt.previous) {
|
||||
if ($c.first -ge ($start + $n)) {
|
||||
$pr = Rpc 'igneum_getSegmentRecords' @(('0x{0:x}' -f ($c.first - $n)))
|
||||
$waiting = $false
|
||||
if ($pr -and $pr.pool) { foreach ($e in @($pr.pool)) { if ($e.verified -eq $true -and $null -eq $e.includedIn) { $waiting = $true } } }
|
||||
if ($waiting -or ($pr -and $pr.paid)) { "RESULT skip $(Stamp) segment $($c.first): the previous segment has a record waiting or paid; a fresh chain would be refused"; continue }
|
||||
}
|
||||
$picked = $c; $expected = $stmt.publicValuesFresh; break
|
||||
}
|
||||
if ($stmt.previous.proofInPool -ne $true) { "RESULT skip $(Stamp) segment $($c.first): previous paid, its proof not in this pool"; continue }
|
||||
$got = Rpc 'igneum_getSegmentProofBytes' @($stmt.previous.first, $stmt.previous.keyHash)
|
||||
if (-not $got -or -not $got.proof) { continue }
|
||||
$hexFile = Join-Path $job "prev-$($c.first).hex"; $binFile = Join-Path $job "prev-$($c.first).bin"
|
||||
[IO.File]::WriteAllText($hexFile, $got.proof)
|
||||
Wsl @("$jobW/unhex.sh", (WslPath $hexFile), (WslPath $binFile))
|
||||
if (-not (Test-Path $binFile)) { continue }
|
||||
$picked = $c; $prevFile = WslPath $binFile; $expected = $stmt.publicValuesContinuing; break
|
||||
}
|
||||
if (-not $picked) { "RESULT pass $passes $(Stamp) $($cands.Count) candidates, none usable this pass; waiting 20 s"; Start-Sleep -Seconds 20; continue }
|
||||
$first = $picked.first; $last = $picked.last
|
||||
$attempted[$first] = $true; $claimed++
|
||||
$tipNow = Hex (Rpc 'eth_blockNumber' @())
|
||||
"RESULT claim $(Stamp) segment $first..$last ($($picked.shards.Count) shards, $(if ($prevFile) { 'continuing' } else { 'fresh' })), margin=$($picked.margin) DAA, tip=$tipNow, candidates=$($cands.Count)"
|
||||
$segStart = Get-Date
|
||||
$d = Join-Path $job "seg-$first"; New-Item -ItemType Directory -Force -Path $d | Out-Null
|
||||
# 3. the export (curl.exe streams the reply to a file; Invoke-WebRequest's Content is a string there)
|
||||
$t = Get-Date
|
||||
$bodyTxt = (@{jsonrpc='2.0'; id=1; method='igneum_exportSegments'; params=@('0x0', ('0x{0:x}' -f $last))} | ConvertTo-Json -Compress)
|
||||
$bodyFile = Join-Path $d 'export-request.json'; $seqFile = Join-Path $d 'seq.json'
|
||||
[IO.File]::WriteAllText($bodyFile, $bodyTxt, (New-Object System.Text.UTF8Encoding $false))
|
||||
& curl.exe -s -S -m 600 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bodyFile" -o $seqFile 2>&1 | ForEach-Object { "curl: $_" }
|
||||
if (-not (Test-Path $seqFile) -or (Get-Item $seqFile).Length -lt 1000) { "RESULT seg $first export FAILED"; continue }
|
||||
"RESULT seg $first export $(Stamp) $((Get-Item $seqFile).Length) bytes in $([math]::Round(((Get-Date) - $t).TotalSeconds,1)) s"
|
||||
$segArgs = @("$jobW/seg.sh", $jobW, "$first", "$last", $payout); if ($prevFile) { $segArgs += $prevFile }
|
||||
Wsl $segArgs
|
||||
Remove-Item $seqFile -ErrorAction SilentlyContinue
|
||||
$resFile = Join-Path $d 'chain-results.json'
|
||||
if (-not (Test-Path $resFile)) { "RESULT seg $first FAILED: no chain results"; continue }
|
||||
$res = Get-Content $resFile -Raw | ConvertFrom-Json
|
||||
$recs = Get-Content (Join-Path $d 'shard-records.json') -Raw | ConvertFrom-Json
|
||||
# 4. the shard records
|
||||
$okShards = 0
|
||||
foreach ($r in $recs) {
|
||||
$sg = (& $minerExe sign-record $label $chain $r.block_hash "$($r.number)" "$($r.shard)" $payout $r.statement $r.proof_sha256 2>&1 | Select-Object -Last 1)
|
||||
$signed = $null; try { $signed = $sg | ConvertFrom-Json } catch {}
|
||||
if (-not $signed -or -not $signed.record) { "RESULT seg $first shard $($r.number)/$($r.shard) sign FAILED: $sg"; $shardsRefused++; continue }
|
||||
$proofWin = $r.proof_file -replace '^/mnt/c/', 'C:/' -replace '/', '\'
|
||||
$bf = Join-Path $d "body-$($r.number)-$($r.shard).json"
|
||||
Wsl @("$jobW/body.sh", $r.proof_file, $signed.record, 'igneum_submitProofRecord', (WslPath $bf))
|
||||
$reply = (& curl.exe -s -S -m 120 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bf" 2>&1)
|
||||
$rr = $null; try { $rr = ($reply | ConvertFrom-Json).result } catch {}
|
||||
if ($rr -and $rr.accepted) { $okShards++; $shardsAccepted++ } else { $shardsRefused++; "RESULT seg $first shard $($r.number)/$($r.shard) refused: $(if ($rr) { $rr.reason } else { "$reply".Substring(0, [math]::Min(300, "$reply".Length)) })" }
|
||||
Remove-Item $bf -ErrorAction SilentlyContinue
|
||||
}
|
||||
"RESULT seg $first shards $(Stamp) accepted $okShards of $($recs.Count) (prove $([math]::Round($res.shard_prove_seconds_total,1)) s, aggregation $([math]::Round($res.aggregate_prove_seconds_total,1)) s, chain $([math]::Round($res.chain_seconds,1)) s)"
|
||||
if ($okShards -ne $recs.Count) { "RESULT seg $first FAILED: not every shard record accepted; the segment record is not submitted"; continue }
|
||||
# 5. the segment record: the statement against the node's (every field but provers, bytes 236..268 of the 340)
|
||||
$pv = $res.segment_public_values
|
||||
function Strip($h) { $h = $h -replace '^0x', ''; if ($h.Length -eq 680) { $h.Substring(0, 472) + $h.Substring(536) } else { $h } }
|
||||
if ((Strip $pv) -ne (Strip $expected)) { "RESULT seg $first FAILED: statement differs from the node's native one; ours $($pv.Substring(0,66)) node $($expected.Substring(0, [math]::Min(66, $expected.Length)))"; continue }
|
||||
$lastHash = ($picked.shards | Where-Object { $_.number -eq $last } | Select-Object -First 1).hash
|
||||
$sg = (& $minerExe sign-segment-record $label $chain "$first" "$last" $lastHash $payout $pv $res.segment_proof_sha256 2>&1 | Select-Object -Last 1)
|
||||
$signed = $null; try { $signed = $sg | ConvertFrom-Json } catch {}
|
||||
if (-not $signed -or -not $signed.record) { "RESULT seg $first segment sign FAILED: $sg"; continue }
|
||||
$bf = Join-Path $d 'body-segment.json'
|
||||
Wsl @("$jobW/body.sh", $res.segment_proof_file, $signed.record, 'igneum_submitSegmentRecord', (WslPath $bf))
|
||||
$reply = (& curl.exe -s -S -m 120 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bf" 2>&1)
|
||||
$rr = $null; try { $rr = ($reply | ConvertFrom-Json).result } catch {}
|
||||
Remove-Item $bf -ErrorAction SilentlyContinue
|
||||
$segSecs = [math]::Round(((Get-Date) - $segStart).TotalSeconds, 1)
|
||||
if ($rr -and $rr.accepted) {
|
||||
$submitted++; $lastSegSecs = $segSecs
|
||||
$stmt2 = Rpc 'igneum_getSegmentStatement' @(('0x{0:x}' -f $first))
|
||||
$aggWei = if ($stmt2) { Hex $stmt2.aggregatorWei } else { 0 }
|
||||
$submittedSegments[$first] = @{ last = $last; aggWei = $aggWei; at = (Get-Date) }
|
||||
"RESULT seg $first submitted $(Stamp) segment $first..$last record accepted (new=$($rr.new), chain_len $($res.segment_chain_len)), aggregator share $([math]::Round($aggWei / 1e18, 4)) IGN, end to end $segSecs s (export+cut+chain+sign+submit)"
|
||||
} else {
|
||||
"RESULT seg $first segment refused $(Stamp): $(if ($rr) { $rr.reason } else { "$reply".Substring(0, [math]::Min(300, "$reply".Length)) }); end to end $segSecs s"
|
||||
}
|
||||
foreach ($b in $first..$last) { Remove-Item (Join-Path $d "block-$b.json") -ErrorAction SilentlyContinue }
|
||||
Remove-Item (Join-Path $d 'export.json') -ErrorAction SilentlyContinue
|
||||
}
|
||||
# --- the end: paid state once more, the rate during the run, the sockets, the prover back on
|
||||
Start-Sleep -Seconds 30
|
||||
$paidTotal = 0; $paidCount = 0
|
||||
foreach ($f in @($submittedSegments.Keys)) {
|
||||
$rec = Rpc 'igneum_getSegmentRecords' @(('0x{0:x}' -f $f))
|
||||
if ($rec -and $rec.paid) { $paidCount++; $paidTotal += (Hex $rec.paid.wei); "RESULT paid $(Stamp) segment $f..$($submittedSegments[$f].last) paid wei=$($rec.paid.wei) carrier=$(Hex $rec.paid.carrierNumber)"; $submittedSegments.Remove($f) }
|
||||
else { "RESULT unpaid $(Stamp) segment $f..$($submittedSegments[$f].last) carried=$(if ($rec) { (@($rec.carried)).Count } else { '?' }) pool=$(if ($rec) { (@($rec.pool)).Count } else { '?' }) $(if ($rec -and $rec.pool) { ($rec.pool | ForEach-Object { "verified=$($_.verified) included=$($_.includedIn) note=$($_.note)" }) -join '; ' })" }
|
||||
}
|
||||
$st = Rpc 'igneum_getProvingStatus' @()
|
||||
if ($st) { "RESULT node_v1 $(Stamp) paidSegments=$($st.v1.paidSegments) paidSegmentWei=$($st.v1.paidSegmentWei) window pending=$($st.v1.segmentsInWindow.pending) proven=$($st.v1.segmentsInWindow.proven) unproven=$($st.v1.segmentsInWindow.unproven) pool entries=$($st.v1.pool.entries) verified=$($st.v1.pool.verified) failed=$($st.v1.pool.failed)" }
|
||||
$t1 = Epoch (Get-Date)
|
||||
$during = RateLines (Epoch $runStart) $t1
|
||||
if ($during.Count) { $m = $during | Measure-Object -Average -Minimum -Maximum; "RESULT rate_during $(Stamp) n=$($during.Count) mean=$([math]::Round($m.Average,2)) min=$($m.Minimum) max=$($m.Maximum) MH/s (the app log's status lines over the run)" } else { "RESULT rate_during $(Stamp) no status lines found" }
|
||||
"RESULT summary $(Stamp) passes=$passes claimed=$claimed submitted=$submitted paid=$paidCount paid_wei=$paidTotal shards_accepted=$shardsAccepted shards_refused=$shardsRefused last_segment_s=$lastSegSecs run_min=$([math]::Round(((Get-Date) - $runStart).TotalMinutes,1))"
|
||||
& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -f sp1-gpu-server; sleep 1; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
||||
"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})"
|
||||
"RESULT end $(Stamp)"
|
||||
Loading…
Reference in a new issue