620 lines
34 KiB
Rust
620 lines
34 KiB
Rust
//! `igneum-pool recheck`: the same-work test's pool reader (review B F03, R02/R03; the founder's Test and Acceptance
|
|
//! Standard, 8 October 2026). The pool's own share verifier (`IgneumEngine::epoch_for` and `hash_bound`, the path
|
|
//! every live share takes in `verify::check`) is driven over the P01 driver's job lines and every nonce's lane hash is
|
|
//! compared to the CPU reference list; the evidence JSON carries the same fields as `tools/ci/p01-vectors.py` (agree,
|
|
//! disagree, missing, the first ten disagreements, the manifest sha, the verdict) plus the accepted-share verdict of a
|
|
//! nonce sample through `verify::check` and, with two jobs in a phase, the day boundary between them.
|
|
//!
|
|
//! igneum-pool recheck --jobs <file: one P01 job line per line> --reference <file: "<nonce> <hash16>" per line>
|
|
//! [--state <igsd1 file>] [--dataset-log2 N] [--phase N] [--manifest <sha>] [--pairing <text>]
|
|
//! [--sample 64] [--threads N] --out <evidence.json>
|
|
//! igneum-pool recheck --self-test
|
|
//!
|
|
//! The job line is pool.rs's (the miner's `job` to its worker): `job <seq> <prehash hex> <share_target64 hex16>
|
|
//! <start nonce> <nonces> <epoch seed bytes hex> <day bytes hex> class=<c> era=<era hex>`; the day bytes are
|
|
//! `"igneum-day/" || day_le64` (igneum_pow::bind::day_bytes), the class a generator number or `vN`.
|
|
|
|
use crate::verify::{check, Code, JobKey};
|
|
use kaspa_consensus_core::igneum::{install_pow_genesis, pow_genesis_day_index, ProgramClass};
|
|
use kaspa_hashes::Hash;
|
|
use kaspa_pow::igneum::{DayStateProvider, EpochRef, EpochSeeds, IgneumEngine};
|
|
use serde_json::{json, Value};
|
|
use std::collections::BTreeMap;
|
|
use std::sync::Arc;
|
|
use std::time::Instant;
|
|
|
|
/// One P01 job line, parsed.
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
pub struct Job {
|
|
pub seq: u64,
|
|
pub prehash: [u8; 32],
|
|
pub share_target64: u64,
|
|
pub start: u64,
|
|
pub count: u64,
|
|
pub seeds: EpochSeeds,
|
|
pub line: String,
|
|
}
|
|
|
|
fn hex_to<const N: usize>(s: &str) -> Result<[u8; N], String> {
|
|
let s = s.trim_start_matches("0x");
|
|
let b = hex::decode(s).map_err(|e| format!("{s}: {e}"))?;
|
|
<[u8; N]>::try_from(b.as_slice()).map_err(|_| format!("{s}: {} bytes, {N} wanted", b.len()))
|
|
}
|
|
|
|
fn num(s: &str) -> Result<u64, String> {
|
|
if let Some(h) = s.strip_prefix("0x") { u64::from_str_radix(h, 16).map_err(|e| format!("{s}: {e}")) } else { s.parse::<u64>().map_err(|e| format!("{s}: {e}")) }
|
|
}
|
|
|
|
/// Parses a job line of the P01 driver's shape.
|
|
pub fn parse_job(line: &str) -> Result<Job, String> {
|
|
let p: Vec<&str> = line.split_whitespace().collect();
|
|
if p.len() < 8 || p[0] != "job" {
|
|
return Err(format!("not a job line ({} fields): {}", p.len(), line.chars().take(80).collect::<String>()));
|
|
}
|
|
let seq = num(p[1])?;
|
|
let prehash = hex_to::<32>(p[2])?;
|
|
let share_target64 = u64::from_str_radix(p[3].trim_start_matches("0x"), 16).map_err(|e| format!("share target {}: {e}", p[3]))?;
|
|
let start = num(p[4])?;
|
|
let count = num(p[5])?;
|
|
let epoch = Hash::from_bytes(hex_to::<32>(p[6])?);
|
|
let day_bytes = hex_to::<19>(p[7])?;
|
|
if &day_bytes[..11] != b"igneum-day/" {
|
|
return Err("day bytes do not start with igneum-day/".into());
|
|
}
|
|
let day = u64::from_le_bytes(day_bytes[11..19].try_into().unwrap());
|
|
let mut class: Option<ProgramClass> = None;
|
|
let mut era = kaspa_hashes::ZERO_HASH;
|
|
let mut shadow_reps = 0u16;
|
|
for kv in &p[8..] {
|
|
if let Some(c) = kv.strip_prefix("class=") {
|
|
let n: u32 = c.trim_start_matches('v').parse().map_err(|_| format!("class {c}"))?;
|
|
class = Some(ProgramClass::from_generator(n).ok_or_else(|| format!("class {c} is not one this pool's node names (its enum ends before generator {n})"))?);
|
|
} else if let Some(e) = kv.strip_prefix("era=") {
|
|
era = Hash::from_bytes(hex_to::<32>(e)?);
|
|
} else if let Some(r) = kv.strip_prefix("shadow_reps=") {
|
|
shadow_reps = r.parse().map_err(|_| format!("shadow_reps {r}"))?;
|
|
}
|
|
}
|
|
let class = class.ok_or("job line names no class=")?;
|
|
Ok(Job { seq, prehash, share_target64, start, count, seeds: EpochSeeds { epoch, day, class, era, shadow_reps }, line: line.to_string() })
|
|
}
|
|
|
|
/// The reference list: `<nonce> <hash16>` per line (hex or decimal nonce), as `igneum-pow hash-bound` writes it.
|
|
pub fn read_reference(text: &str) -> BTreeMap<u64, u64> {
|
|
let mut m = BTreeMap::new();
|
|
for l in text.lines() {
|
|
let p: Vec<&str> = l.split_whitespace().collect();
|
|
if p.len() < 2 || p[0].starts_with('#') {
|
|
continue;
|
|
}
|
|
let Ok(n) = num(p[0]) else { continue };
|
|
let Ok(h) = u64::from_str_radix(p[1].trim_start_matches("0x"), 16) else { continue };
|
|
m.insert(n, h);
|
|
}
|
|
m
|
|
}
|
|
|
|
/// The class v5/v6 day state from a file (the node's `igsd1` stream export): one stream for every block asked, the
|
|
/// engine checks that the stream's block is the seed block.
|
|
pub struct FileStateProvider(pub Vec<u8>);
|
|
|
|
impl DayStateProvider for FileStateProvider {
|
|
fn state_stream(&self, _block: Hash) -> Result<Vec<u8>, String> {
|
|
Ok(self.0.clone())
|
|
}
|
|
fn name(&self) -> &'static str {
|
|
"a state stream file (recheck --state)"
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug, Default)]
|
|
pub struct JobResult {
|
|
pub seq: u64,
|
|
pub start: u64,
|
|
pub count: u64,
|
|
pub agree: u64,
|
|
pub disagree: u64,
|
|
pub missing: u64,
|
|
pub first_disagreements: Vec<(u64, u64, u64)>,
|
|
pub sample_ok: u64,
|
|
pub sample_codes: BTreeMap<String, u64>,
|
|
pub elapsed_s: f64,
|
|
}
|
|
|
|
/// Every nonce of the job through the pool's engine against the reference, on `threads` threads; the first ten
|
|
/// disagreements kept; `sample` nonces spread over the range through `verify::check` as a member's share would be.
|
|
pub fn run_job(epoch: &Arc<EpochRef>, job: &Job, reference: &BTreeMap<u64, u64>, threads: usize, sample: u64) -> JobResult {
|
|
let t0 = Instant::now();
|
|
let threads = threads.max(1);
|
|
let per = job.count.div_ceil(threads as u64);
|
|
let parts: Vec<(u64, u64, Vec<(u64, u64, u64)>)> = std::thread::scope(|s| {
|
|
let handles: Vec<_> = (0..threads as u64)
|
|
.map(|t| {
|
|
let (epoch, job) = (epoch.clone(), job);
|
|
let lo = job.start + t * per;
|
|
let hi = (lo + per).min(job.start + job.count);
|
|
s.spawn(move || {
|
|
let (mut agree, mut missing, mut dis) = (0u64, 0u64, Vec::new());
|
|
for n in lo..hi {
|
|
let h = epoch.hash_bound(&job.prehash, n);
|
|
match reference.get(&n) {
|
|
None => missing += 1,
|
|
Some(r) if *r == h => agree += 1,
|
|
Some(r) => {
|
|
if dis.len() < 10 {
|
|
dis.push((n, *r, h));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
(agree, missing, dis)
|
|
})
|
|
})
|
|
.collect();
|
|
handles.into_iter().map(|h| h.join().expect("recheck thread")).collect()
|
|
});
|
|
let mut r = JobResult { seq: job.seq, start: job.start, count: job.count, ..Default::default() };
|
|
for (a, m, d) in parts {
|
|
r.agree += a;
|
|
r.missing += m;
|
|
for x in d {
|
|
if r.first_disagreements.len() < 10 {
|
|
r.first_disagreements.push(x);
|
|
}
|
|
}
|
|
}
|
|
r.disagree = job.count - r.agree - r.missing;
|
|
r.first_disagreements.sort();
|
|
// the accepted-share sample: the reference hash claimed as a member would claim it, judged by verify::check
|
|
let key = JobKey { prehash: job.prehash, vote_key_hash: Hash::from_bytes([0x5au8; 32]), target64: 0, share_target64: job.share_target64, seeds: job.seeds };
|
|
let step = (job.count / sample.max(1)).max(1);
|
|
let mut n = job.start;
|
|
let mut taken = 0;
|
|
while taken < sample && n < job.start + job.count {
|
|
if let Some(claimed) = reference.get(&n) {
|
|
let v = check(epoch, &key, n, *claimed, key.vote_key_hash);
|
|
if v.code == Code::Ok {
|
|
r.sample_ok += 1;
|
|
}
|
|
*r.sample_codes.entry(v.code.as_str().to_string()).or_insert(0) += 1;
|
|
taken += 1;
|
|
}
|
|
n += step;
|
|
}
|
|
r.elapsed_s = t0.elapsed().as_secs_f64();
|
|
r
|
|
}
|
|
|
|
/// The phase's evidence in the P01 driver's shape.
|
|
pub fn evidence(phase: u64, manifest: &str, pairing: &str, jobs: &[Job], results: &[JobResult], reference_len: usize) -> Value {
|
|
let agree: u64 = results.iter().map(|r| r.agree).sum();
|
|
let disagree: u64 = results.iter().map(|r| r.disagree).sum();
|
|
let missing: u64 = results.iter().map(|r| r.missing).sum();
|
|
let count: u64 = results.iter().map(|r| r.count).sum();
|
|
let mut first: Vec<&(u64, u64, u64)> = results.iter().flat_map(|r| r.first_disagreements.iter()).collect();
|
|
first.sort();
|
|
first.truncate(10);
|
|
let sample_ok: u64 = results.iter().map(|r| r.sample_ok).sum();
|
|
let sample_n: u64 = results.iter().map(|r| r.sample_codes.values().sum::<u64>()).sum();
|
|
let mut codes: BTreeMap<String, u64> = BTreeMap::new();
|
|
for r in results {
|
|
for (c, n) in &r.sample_codes {
|
|
*codes.entry(c.clone()).or_insert(0) += n;
|
|
}
|
|
}
|
|
let boundary = if jobs.len() >= 2 {
|
|
let b = jobs[1].start;
|
|
json!({ "nonce": b, "day_before": jobs[0].seeds.day, "day_after": jobs[1].seeds.day, "class_before": jobs[0].seeds.class.name(), "class_after": jobs[1].seeds.class.name(), "agree_before": results[0].agree, "agree_after": results[1].agree, "note": "the day changes at this nonce: the two jobs hash the same prehash over the day before and the day from it" })
|
|
} else {
|
|
Value::Null
|
|
};
|
|
let verdict = if disagree == 0 && missing == 0 && sample_ok == sample_n { "agree" } else { "disagree" };
|
|
json!({
|
|
"reader": "igneum-pool recheck (the pool's share verifier: IgneumEngine::epoch_for + hash_bound, verify::check for the sample)",
|
|
"phase": phase,
|
|
"manifest_sha": manifest,
|
|
"pairing": pairing,
|
|
"jobs": jobs.iter().map(|j| json!({ "seq": j.seq, "start": j.start, "count": j.count, "epoch_seed": j.seeds.epoch.to_string(), "day": j.seeds.day, "class": j.seeds.class.name(), "generator": j.seeds.class.generator_version(), "era": j.seeds.era.to_string(), "share_target64": format!("{:016x}", j.share_target64), "line": j.line })).collect::<Vec<_>>(),
|
|
"reference_entries": reference_len,
|
|
"count": count, "agree": agree, "disagree": disagree, "missing": missing,
|
|
"first_disagreements": first.iter().map(|(n, r, p)| json!({ "nonce": n, "reference": format!("{r:016x}"), "pool": format!("{p:016x}") })).collect::<Vec<_>>(),
|
|
"boundary": boundary,
|
|
"accepted_share_sample": { "n": sample_n, "ok": sample_ok, "codes": codes },
|
|
"per_job": results.iter().map(|r| json!({ "seq": r.seq, "agree": r.agree, "disagree": r.disagree, "missing": r.missing, "elapsed_s": r.elapsed_s })).collect::<Vec<_>>(),
|
|
"verdict": verdict,
|
|
})
|
|
}
|
|
|
|
fn arg(args: &[String], name: &str) -> Option<String> {
|
|
args.iter().position(|a| a == name).and_then(|i| args.get(i + 1).cloned())
|
|
}
|
|
|
|
/// A pack's `program.json` fields the driver reads (`pack_fields`): the seed bytes, the day bytes, the class, the era
|
|
/// bytes, the program id, the generator.
|
|
pub struct PackFields {
|
|
pub seed_bytes: String,
|
|
pub day_bytes: String,
|
|
pub class: Value,
|
|
pub era: String,
|
|
pub program_id: Value,
|
|
pub generator: Value,
|
|
pub pack: String,
|
|
/// `dataset.log2_words`: the dataset size the pack was built for; the reader installs it (a wrong size is every
|
|
/// hash wrong, the same-work context's packs say 28)
|
|
pub log2_words: Option<u32>,
|
|
}
|
|
|
|
pub fn pack_fields(dir: &std::path::Path) -> Result<PackFields, String> {
|
|
let j: Value = serde_json::from_slice(&std::fs::read(dir.join("program.json")).map_err(|e| format!("{}: {e}", dir.join("program.json").display()))?).map_err(|e| format!("{}: {e}", dir.display()))?;
|
|
Ok(PackFields {
|
|
seed_bytes: j.get("seed_bytes").and_then(|v| v.as_str()).unwrap_or("").to_string(),
|
|
day_bytes: j.get("dataset").and_then(|d| d.get("day_bytes")).and_then(|v| v.as_str()).ok_or("program.json has no dataset.day_bytes")?.to_string(),
|
|
class: j.get("program_class").cloned().unwrap_or(json!("?")),
|
|
era: j.get("era_seed_bytes").and_then(|v| v.as_str()).unwrap_or("").to_string(),
|
|
program_id: j.get("program_id").cloned().unwrap_or(json!("?")),
|
|
generator: j.get("generator").cloned().unwrap_or(json!("?")),
|
|
pack: dir.file_name().and_then(|n| n.to_str()).unwrap_or("?").to_string(),
|
|
log2_words: j.get("dataset").and_then(|d| d.get("log2_words")).and_then(|v| v.as_u64()).map(|v| v as u32),
|
|
})
|
|
}
|
|
|
|
/// A segment of a phase: `[start, end)` on one pack and one reference, as the driver's `segments_for` cuts them.
|
|
pub struct Segment {
|
|
pub start: u64,
|
|
pub end: u64,
|
|
pub pack: std::path::PathBuf,
|
|
pub reference: std::path::PathBuf,
|
|
}
|
|
|
|
/// The driver's `segments_for`: phase 1 on D, phase 3 on D1, phase 2 split at the boundary nonce.
|
|
pub fn segments_for(jc: &Value, base: &std::path::Path, phase: u64) -> Result<Vec<Segment>, String> {
|
|
let w = jc.get("range_log2").and_then(|v| v.as_u64()).unwrap_or(20);
|
|
if !(1..=3).contains(&phase) {
|
|
return Err("--phase must be 1, 2 or 3".into());
|
|
}
|
|
let pth = |s: &str| if std::path::Path::new(s).is_absolute() { std::path::PathBuf::from(s) } else { base.join(s) };
|
|
let packs = jc.get("packs").ok_or("job context has no packs")?;
|
|
let refs = jc.get("references").ok_or("job context has no references")?;
|
|
let get = |o: &Value, k: &str| o.get(k).and_then(|v| v.as_str()).map(pth).ok_or_else(|| format!("job context: no {k}"));
|
|
let (s0, e0) = ((phase - 1) << w, phase << w);
|
|
Ok(match phase {
|
|
1 => vec![Segment { start: s0, end: e0, pack: get(packs, "D")?, reference: get(refs, "D")? }],
|
|
3 => vec![Segment { start: s0, end: e0, pack: get(packs, "D1")?, reference: get(refs, "D1")? }],
|
|
_ => {
|
|
let b = jc.get("boundary_nonce").and_then(|v| v.as_u64().or_else(|| v.as_str().and_then(|s| s.parse().ok()))).ok_or("job context has no boundary_nonce")?;
|
|
if !(s0 < b && b < e0) {
|
|
return Err(format!("the boundary nonce {b} is not inside phase 2 [{s0}, {e0})"));
|
|
}
|
|
vec![Segment { start: s0, end: b, pack: get(packs, "D")?, reference: get(refs, "D")? }, Segment { start: b, end: e0, pack: get(packs, "D1")?, reference: get(refs, "D1")? }]
|
|
}
|
|
})
|
|
}
|
|
|
|
/// The driver's job line for a segment (`job <seq> <prehash> ffff.. <start> <count> <seed bytes> <day bytes> class=<c> era=<era>`).
|
|
pub fn job_line(seq: u64, prehash: &str, seg: &Segment, f: &PackFields) -> String {
|
|
let class = match &f.class {
|
|
Value::Number(n) => n.to_string(),
|
|
Value::String(s) => s.clone(),
|
|
other => other.to_string(),
|
|
};
|
|
format!("job {seq} {prehash} ffffffffffffffff {} {} {} {} class={class} era={}", seg.start, seg.end - seg.start, f.seed_bytes, f.day_bytes, f.era)
|
|
}
|
|
|
|
/// The CLI: exit 0 on PASS, 1 on FAIL, 2 on BLOCKED (the engine refused the context; the evidence says why) or a bad
|
|
/// argument. The evidence JSON carries the driver's keys so the six readers' files read alike.
|
|
pub fn cli(args: &[String]) -> i32 {
|
|
if args.first().map(|a| a.as_str()) == Some("--self-test") {
|
|
return self_test();
|
|
}
|
|
let (Some(jc_path), Some(phase), Some(out)) = (arg(args, "--job-context"), arg(args, "--phase").and_then(|s| s.parse::<u64>().ok()), arg(args, "--out")) else {
|
|
eprintln!("usage: igneum-pool recheck --job-context <job-context.json> --phase 1|2|3 [--state <igsd1>] [--dataset-log2 N] [--genesis-day D] [--manifest <sha>] [--sample 64] [--threads N] --out <evidence.json>");
|
|
return 2;
|
|
};
|
|
let t0 = Instant::now();
|
|
let jc_abs = std::fs::canonicalize(&jc_path).unwrap_or_else(|_| std::path::PathBuf::from(&jc_path));
|
|
let jc: Value = match std::fs::read(&jc_abs).map_err(|e| e.to_string()).and_then(|b| serde_json::from_slice(&b).map_err(|e| e.to_string())) {
|
|
Ok(v) => v,
|
|
Err(e) => {
|
|
eprintln!("{jc_path}: {e}");
|
|
return 2;
|
|
}
|
|
};
|
|
let base = jc_abs.parent().map(|p| p.to_path_buf()).unwrap_or_default();
|
|
let segs = match segments_for(&jc, &base, phase) {
|
|
Ok(s) => s,
|
|
Err(e) => {
|
|
eprintln!("recheck: {e}");
|
|
return 2;
|
|
}
|
|
};
|
|
let prehash = jc.get("prehash").and_then(|v| v.as_str()).unwrap_or("0000000000000000000000000000000000000000000000000000000000000001").to_string();
|
|
let manifest = arg(args, "--manifest").or_else(|| jc.get("manifest").and_then(|v| v.as_str()).map(|s| s.to_string())).unwrap_or_default();
|
|
let sample = arg(args, "--sample").and_then(|s| s.parse().ok()).unwrap_or(64);
|
|
let threads = arg(args, "--threads").and_then(|s| s.parse().ok()).unwrap_or_else(|| std::thread::available_parallelism().map(|n| n.get()).unwrap_or(4));
|
|
let state_note = match arg(args, "--state") {
|
|
Some(p) => match std::fs::read(&p) {
|
|
Ok(b) => {
|
|
let n = b.len();
|
|
kaspa_pow::igneum::install_day_state_provider(Arc::new(FileStateProvider(b)));
|
|
format!("day state from {p} ({n} bytes)")
|
|
}
|
|
Err(e) => {
|
|
eprintln!("{p}: {e}");
|
|
return 2;
|
|
}
|
|
},
|
|
None => "no --state: the engine's own provider, or none".to_string(),
|
|
};
|
|
let dataset_flag = arg(args, "--dataset-log2").and_then(|s| s.parse::<u32>().ok());
|
|
let genesis_day_flag = arg(args, "--genesis-day").and_then(|s| s.parse::<u64>().ok());
|
|
let engine = IgneumEngine::new();
|
|
let mut fields = Vec::new();
|
|
let mut jobs = Vec::new();
|
|
let mut results = Vec::new();
|
|
let mut reference: BTreeMap<u64, u64> = BTreeMap::new();
|
|
let mut blocked: Option<String> = None;
|
|
for (i, seg) in segs.iter().enumerate() {
|
|
let f = match pack_fields(&seg.pack) {
|
|
Ok(f) => f,
|
|
Err(e) => {
|
|
eprintln!("recheck: {e}");
|
|
return 2;
|
|
}
|
|
};
|
|
let text = match std::fs::read_to_string(&seg.reference) {
|
|
Ok(t) => t,
|
|
Err(e) => {
|
|
eprintln!("recheck: {}: {e}", seg.reference.display());
|
|
return 2;
|
|
}
|
|
};
|
|
for (n, h) in read_reference(&text) {
|
|
if seg.start <= n && n < seg.end {
|
|
reference.insert(n, h);
|
|
}
|
|
}
|
|
let line = job_line(i as u64 + 1, &prehash, seg, &f);
|
|
// the pack's geometry, before its epoch is built: the dataset size (dataset.log2_words, the flag overriding)
|
|
// AND the genesis day index, the first segment's day (the same-work lane's class, 8 October 2026 22:10 UK:
|
|
// a standalone process holds the genesis day at 0 and the growth rule doubled the cache fourteen times by
|
|
// day 20730, every item wrong; the live miner installs both from the template, the reader from the pack)
|
|
if i == 0
|
|
&& let Ok(j0) = parse_job(&line)
|
|
{
|
|
let l = dataset_flag.or(f.log2_words).unwrap_or_else(kaspa_consensus_core::igneum::pow_genesis_dataset_log2);
|
|
let g = genesis_day_flag.unwrap_or(j0.seeds.day);
|
|
if (g, l) != (pow_genesis_day_index(), kaspa_consensus_core::igneum::pow_genesis_dataset_log2()) {
|
|
install_pow_genesis(g, l);
|
|
}
|
|
eprintln!("recheck: genesis day {g}, dataset 2^{l} words (the pack's geometry; --genesis-day and --dataset-log2 override)");
|
|
}
|
|
let job = match parse_job(&line) {
|
|
Ok(j) => j,
|
|
Err(e) => {
|
|
blocked = Some(format!("the pack's job line is not one this pool can name: {e}"));
|
|
fields.push(f);
|
|
break;
|
|
}
|
|
};
|
|
match engine.epoch_for_bounded(&job.seeds) {
|
|
Ok((epoch, _)) => {
|
|
let r = run_job(&epoch, &job, &reference, threads, sample);
|
|
eprintln!("recheck: phase {phase} segment [{}, {}): agree {} disagree {} missing {} sample ok {} of {} in {:.1} s", seg.start, seg.end, r.agree, r.disagree, r.missing, r.sample_ok, r.sample_codes.values().sum::<u64>(), r.elapsed_s);
|
|
results.push(r);
|
|
jobs.push(job);
|
|
}
|
|
Err(e) => {
|
|
blocked = Some(format!("the pool's engine refused the context at segment [{}, {}): {e:?}", seg.start, seg.end));
|
|
jobs.push(job);
|
|
}
|
|
}
|
|
fields.push(f);
|
|
}
|
|
let ev = driver_evidence(&jc, &jc_abs, phase, &manifest, &prehash, &segs, &fields, &jobs, &results, &reference, blocked.as_deref(), &state_note, t0.elapsed().as_secs_f64());
|
|
if let Err(e) = std::fs::write(&out, serde_json::to_vec_pretty(&ev).unwrap()) {
|
|
eprintln!("{out}: {e}");
|
|
return 2;
|
|
}
|
|
let v = ev["verdict"].as_str().unwrap_or("?");
|
|
println!("igneum-pool recheck: {v}: phase {phase} answered {} agree {} disagree {} missing {} in {:.1} s -> {out}{}", ev["answered"], ev["agree"], ev["disagree"], ev["missing"], t0.elapsed().as_secs_f64(), blocked.map(|b| format!(" ({b})")).unwrap_or_default());
|
|
match v {
|
|
"PASS" => 0,
|
|
"FAIL" => 1,
|
|
_ => 2,
|
|
}
|
|
}
|
|
|
|
/// The evidence in the P01 driver's shape (`p01-vectors.py`), the pool's columns in the gpu's place.
|
|
#[allow(clippy::too_many_arguments)]
|
|
pub fn driver_evidence(jc: &Value, jc_abs: &std::path::Path, phase: u64, manifest: &str, prehash: &str, segs: &[Segment], fields: &[PackFields], jobs: &[Job], results: &[JobResult], reference: &BTreeMap<u64, u64>, blocked: Option<&str>, state_note: &str, seconds: f64) -> Value {
|
|
let start = segs.first().map(|s| s.start).unwrap_or(0);
|
|
let end = segs.last().map(|s| s.end).unwrap_or(0);
|
|
let count = end - start;
|
|
let agree: u64 = results.iter().map(|r| r.agree).sum();
|
|
let disagree: u64 = results.iter().map(|r| r.disagree).sum();
|
|
let answered: u64 = results.iter().map(|r| r.count).sum();
|
|
let missing: u64 = results.iter().map(|r| r.missing).sum::<u64>() + (count - answered);
|
|
let mut first: Vec<&(u64, u64, u64)> = results.iter().flat_map(|r| r.first_disagreements.iter()).collect();
|
|
first.sort();
|
|
first.truncate(10);
|
|
let mut first_missing: Vec<u64> = Vec::new();
|
|
for r in results {
|
|
if r.missing > 0 {
|
|
for n in r.start..r.start + r.count {
|
|
if !reference.contains_key(&n) {
|
|
first_missing.push(n);
|
|
if first_missing.len() >= 10 {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
let sample_ok: u64 = results.iter().map(|r| r.sample_ok).sum();
|
|
let sample_n: u64 = results.iter().map(|r| r.sample_codes.values().sum::<u64>()).sum();
|
|
let mut codes: BTreeMap<String, u64> = BTreeMap::new();
|
|
for r in results {
|
|
for (c, n) in &r.sample_codes {
|
|
*codes.entry(c.clone()).or_insert(0) += n;
|
|
}
|
|
}
|
|
let f0 = fields.first();
|
|
let mut ev = json!({
|
|
"case": "POW-01", "profile": "P01",
|
|
"pack": fields.iter().map(|f| f.pack.clone()).collect::<Vec<_>>().join(", "),
|
|
"program_id": f0.map(|f| f.program_id.clone()).unwrap_or(Value::Null), "generator": f0.map(|f| f.generator.clone()).unwrap_or(Value::Null), "class": f0.map(|f| f.class.clone()).unwrap_or(Value::Null),
|
|
"worker": "igneum-pool recheck", "worker_args": ["the pool's share verifier: IgneumEngine::epoch_for + hash_bound per nonce; verify::check for the accepted-share sample"],
|
|
"device_line": format!("cpu, {} threads; {state_note}", std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1)),
|
|
"manifest_sha": manifest, "prehash": prehash,
|
|
"nonces": {"start": start, "count": count}, "answered": answered, "agree": agree, "disagree": disagree, "missing": missing,
|
|
"first_disagreements": first.iter().map(|(n, r, p)| json!({"nonce": n, "pool": format!("{p:016x}"), "gpu": format!("{p:016x}"), "cpu": format!("{r:016x}")})).collect::<Vec<_>>(),
|
|
"first_missing": first_missing, "worker_errors": blocked.map(|b| vec![b.to_string()]).unwrap_or_default(),
|
|
"seconds": (seconds * 10.0).round() / 10.0, "at": chrono_now(),
|
|
"job_context": jc_abs.display().to_string(), "phase": phase, "object": jc.get("object").cloned().unwrap_or(Value::Null),
|
|
"segments": segs.iter().zip(fields.iter()).map(|(s, f)| json!({"start": s.start, "end": s.end, "program_id": f.program_id, "day_bytes": f.day_bytes, "class": f.class, "pack": f.pack})).collect::<Vec<_>>(),
|
|
"jobs": jobs.iter().map(|j| json!({"seq": j.seq, "epoch_seed": j.seeds.epoch.to_string(), "day": j.seeds.day, "class": j.seeds.class.name(), "era": j.seeds.era.to_string(), "line": j.line})).collect::<Vec<_>>(),
|
|
"accepted_share_sample": {"n": sample_n, "ok": sample_ok, "codes": codes},
|
|
});
|
|
let mut switched = true;
|
|
if segs.len() == 2 && fields.len() == 2 {
|
|
let b = segs[1].start;
|
|
let (fb, fa) = (&fields[0], &fields[1]);
|
|
let hb = results.first().map(|r| r.disagree == 0 && r.missing == 0).unwrap_or(false);
|
|
let ha = results.get(1).map(|r| r.disagree == 0 && r.missing == 0).unwrap_or(false);
|
|
switched = fb.day_bytes != fa.day_bytes && hb && ha;
|
|
ev["boundary"] = json!({"nonce": b, "program_id_before": fb.program_id, "program_id_after": fa.program_id, "day_bytes_before": fb.day_bytes, "day_bytes_after": fa.day_bytes,
|
|
"reference_before": reference.get(&(b - 1)).map(|h| format!("{h:016x}")), "reference_after": reference.get(&b).map(|h| format!("{h:016x}")),
|
|
"agree_before": results.first().map(|r| r.agree), "agree_after": results.get(1).map(|r| r.agree), "switched": switched});
|
|
}
|
|
ev["verdict"] = json!(if blocked.is_some() { "BLOCKED" } else if agree == count && disagree == 0 && missing == 0 && sample_ok == sample_n && switched { "PASS" } else { "FAIL" });
|
|
if let Some(b) = blocked {
|
|
ev["blocked"] = json!(b);
|
|
}
|
|
ev
|
|
}
|
|
|
|
fn chrono_now() -> String {
|
|
let t = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs();
|
|
// civil date from the epoch second (Howard Hinnant's algorithm), UTC
|
|
let days = t / 86_400;
|
|
let (h, m, s) = ((t % 86_400) / 3600, (t % 3600) / 60, t % 60);
|
|
let z = days as i64 + 719_468;
|
|
let era = z.div_euclid(146_097);
|
|
let doe = z - era * 146_097;
|
|
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
|
|
let y = yoe + era * 400;
|
|
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
|
let mp = (5 * doy + 2) / 153;
|
|
let d = doy - (153 * mp + 2) / 5 + 1;
|
|
let mo = if mp < 10 { mp + 3 } else { mp - 9 };
|
|
let y = if mo <= 2 { y + 1 } else { y };
|
|
format!("{y:04}-{mo:02}-{d:02}T{h:02}:{m:02}:{s:02}Z")
|
|
}
|
|
|
|
/// The reader shown to fire: a reference the pool's own engine wrote agrees in full; one flipped hash and one missing
|
|
/// nonce in it are counted and named, the verdict disagree, the first disagreement the flipped nonce.
|
|
fn self_test() -> i32 {
|
|
let engine = IgneumEngine::new();
|
|
let seeds = EpochSeeds::v2(Hash::from_bytes([0x33u8; 32]), 0);
|
|
let (sb, db) = IgneumEngine::seed_bytes(&seeds);
|
|
let prehash = [0x44u8; 32];
|
|
let line = format!("job 1 {} ffffffffffffffff 1000 256 {} {} class=2 era={}", hex::encode(prehash), hex::encode(sb), hex::encode(db), hex::encode([0u8; 32]));
|
|
let job = parse_job(&line).expect("the self-test's own line parses");
|
|
assert_eq!(job.seeds, seeds);
|
|
let epoch = engine.epoch_for(&seeds);
|
|
let mut reference: BTreeMap<u64, u64> = (1000..1256).map(|n| (n, epoch.hash_bound(&prehash, n))).collect();
|
|
let good = run_job(&epoch, &job, &reference, 4, 16);
|
|
let mut fails = 0;
|
|
if !(good.agree == 256 && good.disagree == 0 && good.missing == 0 && good.sample_ok == 16) {
|
|
eprintln!("self-test: a full reference should agree: {good:?}");
|
|
fails += 1;
|
|
}
|
|
reference.insert(1100, reference[&1100] ^ 1);
|
|
reference.remove(&1200);
|
|
let bad = run_job(&epoch, &job, &reference, 4, 16);
|
|
if !(bad.agree == 254 && bad.disagree == 1 && bad.missing == 1 && bad.first_disagreements.first().map(|d| d.0) == Some(1100)) {
|
|
eprintln!("self-test: the flipped nonce and the missing one must be counted: {bad:?}");
|
|
fails += 1;
|
|
}
|
|
let ev = evidence(1, "selftest", "the self-test", &[job], &[bad], reference.len());
|
|
if ev["verdict"] != "disagree" || ev["first_disagreements"][0]["nonce"] != 1100 {
|
|
eprintln!("self-test: the evidence must say disagree and name nonce 1100: {ev}");
|
|
fails += 1;
|
|
}
|
|
let text = (1000..1256).map(|n| format!("{n} {:016x}", reference.get(&n).copied().unwrap_or(0))).collect::<Vec<_>>().join("\n");
|
|
if read_reference(&text).len() != 256 {
|
|
eprintln!("self-test: the reference reader");
|
|
fails += 1;
|
|
}
|
|
// the job-context form: two packs (day 0 and day 1 of the self-test's class 2 seeds), two references the engine
|
|
// itself wrote, range_log2 4, the boundary at 24 inside phase 2; phase 2 PASSes with the boundary switched, and a
|
|
// context whose stream the engine refuses reads BLOCKED (a class v5 line over no state)
|
|
let d = std::env::temp_dir().join(format!("igneum-pool-recheck-{}", std::process::id()));
|
|
let _ = std::fs::remove_dir_all(&d);
|
|
let mut refs = Vec::new();
|
|
for (day, name) in [(0u64, "D"), (1u64, "D1")] {
|
|
let seeds = EpochSeeds::v2(Hash::from_bytes([0x33u8; 32]), day);
|
|
let (sb, db) = IgneumEngine::seed_bytes(&seeds);
|
|
let pack = d.join(format!("pack-{name}"));
|
|
std::fs::create_dir_all(&pack).unwrap();
|
|
std::fs::write(pack.join("program.json"), serde_json::to_vec(&json!({"seed_bytes": hex::encode(sb), "dataset": {"day_bytes": hex::encode(db), "log2_words": kaspa_consensus_core::igneum::pow_genesis_dataset_log2()}, "program_class": 2, "era_seed_bytes": hex::encode([0u8; 32]), "program_id": format!("0x{}", day + 1), "generator": 2})).unwrap()).unwrap();
|
|
let epoch = engine.epoch_for(&seeds);
|
|
let text = (0..48u64).map(|n| format!("{n} {:016x}", epoch.hash_bound(&[0x44u8; 32], n))).collect::<Vec<_>>().join("\n");
|
|
std::fs::write(d.join(format!("ref-{name}.txt")), text).unwrap();
|
|
refs.push(name);
|
|
}
|
|
let jc = d.join("job-context.json");
|
|
std::fs::write(&jc, serde_json::to_vec(&json!({"object": "self-test", "range_log2": 4, "boundary_nonce": 24, "prehash": hex::encode([0x44u8; 32]), "manifest": "selftest", "packs": {"D": "pack-D", "D1": "pack-D1"}, "references": {"D": "ref-D.txt", "D1": "ref-D1.txt"}})).unwrap()).unwrap();
|
|
for phase in 1..=3u64 {
|
|
let out = d.join(format!("pool-{phase}.json"));
|
|
let rc = cli(&["--job-context".into(), jc.display().to_string(), "--phase".into(), phase.to_string(), "--sample".into(), "4".into(), "--threads".into(), "2".into(), "--out".into(), out.display().to_string()]);
|
|
let ev: Value = serde_json::from_slice(&std::fs::read(&out).unwrap()).unwrap();
|
|
let want_start = (phase - 1) << 4;
|
|
if !(rc == 0 && ev["verdict"] == "PASS" && ev["nonces"] == json!({"start": want_start, "count": 16}) && ev["agree"] == 16) {
|
|
eprintln!("self-test: phase {phase} of the job context was not a PASS on its range: rc={rc} {ev}");
|
|
fails += 1;
|
|
}
|
|
if phase == 2 && !(ev["boundary"]["nonce"] == 24 && ev["boundary"]["switched"] == true && ev["boundary"]["program_id_before"] == "0x1" && ev["boundary"]["program_id_after"] == "0x2" && ev["segments"].as_array().map(|a| a.len()) == Some(2)) {
|
|
eprintln!("self-test: phase 2 did not switch at the boundary nonce 24 with the boundary block: {}", ev["boundary"]);
|
|
fails += 1;
|
|
}
|
|
}
|
|
// a refusal is a BLOCKED verdict in the file, exit 2, never a crash: class 5 over no day state
|
|
let pack = d.join("pack-v5");
|
|
std::fs::create_dir_all(&pack).unwrap();
|
|
let seeds5 = EpochSeeds { epoch: Hash::from_bytes([0x55u8; 32]), day: 0, class: ProgramClass::V5, era: Hash::from_bytes([0x55u8; 32]), shadow_reps: 0 };
|
|
let (sb, db) = IgneumEngine::seed_bytes(&seeds5);
|
|
std::fs::write(pack.join("program.json"), serde_json::to_vec(&json!({"seed_bytes": hex::encode(sb), "dataset": {"day_bytes": hex::encode(db)}, "program_class": 5, "era_seed_bytes": hex::encode([0x55u8; 32]), "program_id": "0x5", "generator": 5})).unwrap()).unwrap();
|
|
let jc5 = d.join("job-context-v5.json");
|
|
std::fs::write(&jc5, serde_json::to_vec(&json!({"object": "self-test v5", "range_log2": 4, "prehash": hex::encode([0x44u8; 32]), "packs": {"D": "pack-v5", "D1": "pack-v5"}, "references": {"D": "ref-D.txt", "D1": "ref-D.txt"}})).unwrap()).unwrap();
|
|
let out = d.join("pool-v5.json");
|
|
let rc = cli(&["--job-context".into(), jc5.display().to_string(), "--phase".into(), "1".into(), "--threads".into(), "2".into(), "--out".into(), out.display().to_string()]);
|
|
let ev: Value = serde_json::from_slice(&std::fs::read(&out).unwrap()).unwrap();
|
|
if !(rc == 2 && ev["verdict"] == "BLOCKED" && ev["blocked"].as_str().is_some_and(|b| b.contains("refused the context"))) {
|
|
eprintln!("self-test: the engine's refusal must read BLOCKED with its text, exit 2: rc={rc} {}", ev["verdict"]);
|
|
fails += 1;
|
|
}
|
|
let _ = std::fs::remove_dir_all(&d);
|
|
if fails == 0 {
|
|
println!("recheck self-test: known-pass (256 of 256 agree, 16 of 16 accepted), known-failed (one flipped hash at nonce 1100, one missing at 1200: disagree 1, missing 1), the job-context form (three phases on their ranges, phase 2 switched at nonce 24 with the boundary block) and the BLOCKED verdict on an engine refusal all fire");
|
|
0
|
|
} else {
|
|
1
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn a_job_line_of_the_p01_shape_parses_and_the_reader_fires_on_both_cases() {
|
|
assert_eq!(self_test(), 0);
|
|
// the hash lane's day bytes of the same-work context: igneum-day/20730 and 20731
|
|
let d = hex_to::<19>("69676e65756d2d6461792ffa50000000000000").unwrap();
|
|
assert_eq!(u64::from_le_bytes(d[11..19].try_into().unwrap()), 20730);
|
|
let d = hex_to::<19>("69676e65756d2d6461792ffb50000000000000").unwrap();
|
|
assert_eq!(u64::from_le_bytes(d[11..19].try_into().unwrap()), 20731);
|
|
assert!(parse_job("job 1 00 ff 0 1 00 00 class=2").is_err(), "short fields are refused");
|
|
}
|
|
}
|