diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06cc74820..a5cd62109 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,8 @@ jobs: - name: igneum-pow tests (release) working-directory: igneum-pow run: cargo test --release + - name: pack loader seed rule (packfile.h on a known-good and a known-mismatched pack) + run: bash proto-cuda/nvrtc/emu/packfile-test.sh - name: igneum-census build (release) working-directory: igneum-census run: cargo build --release diff --git a/app/igneum-app/src/watchdog.rs b/app/igneum-app/src/watchdog.rs index 27f30c837..369ad8222 100644 --- a/app/igneum-app/src/watchdog.rs +++ b/app/igneum-app/src/watchdog.rs @@ -27,6 +27,12 @@ pub const HEALTHY_RESET_S: f64 = 300.0; pub const MINER_GAVE_UP_CODE: i32 = 43; /// No `watch` reading from our node for this long: restart it. pub const NODE_SILENT_S: f64 = 120.0; +/// `igneum-miner` exit code when its worker refused the program pack it was started with and the miner could not +/// rebuild it (or rebuilt it `PACK_REBUILD_CAP` times this epoch): the app exports the pack from the node again +/// before the next start, at most `PACK_REBUILD_CAP` times per epoch, then shows the card. +pub const PACK_OUT_OF_DATE_CODE: i32 = 44; +/// Pack rebuilds per epoch seed before the app stops restarting the miner on it. +pub const PACK_REBUILD_CAP: u32 = 3; /// Node restarts inside this window grow the delay before the next one. pub const NODE_WINDOW_S: f64 = 600.0; @@ -280,10 +286,114 @@ fn kv_f64(line: &str, key: &str) -> Option { kv(line, key)?.trim_end_matches('s').parse().ok() } +/// The decision after a pack refusal (exit `PACK_OUT_OF_DATE_CODE`, or a `PACK OUT OF DATE` / `error 0 pack` line +/// from the miner): rebuild the pack before the restart, or stop for this epoch. 5 October 2026: the app restarted +/// two workers every 20 to 40 s for an hour on a pack neither it nor the miner had re-exported in between. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PackAction { + /// Export the pack from the node again (the n-th time this epoch) and start the miner on it now. + Rebuild { n: u32, cap: u32 }, + /// The cap for this epoch is reached: show the card with this reason and wait for the next epoch (or a reset). + GiveUp { n: u32, reason: String }, +} + +/// Pack rebuilds per epoch seed (the epoch the exported pack names in its seeds.txt). +#[derive(Debug, Default)] +pub struct PackRebuilds { + epoch: Option, + n: u32, +} + +impl PackRebuilds { + pub fn new() -> Self { + Self::default() + } + + /// The miner exited over its pack while the exported pack is for `epoch`; `why` is the miner's reason. + pub fn decide(&mut self, epoch: &str, why: &str) -> PackAction { + if self.epoch.as_deref() != Some(epoch) { + self.epoch = Some(epoch.to_string()); + self.n = 0; + } + if self.n >= PACK_REBUILD_CAP { + return PackAction::GiveUp { n: self.n, reason: format!("the program pack still fails after {} rebuilds this epoch ({why}); the worker and the pack disagree", self.n) }; + } + self.n += 1; + PackAction::Rebuild { n: self.n, cap: PACK_REBUILD_CAP } + } + + pub fn count(&self) -> u32 { + self.n + } +} + +/// A miner line that names a pack refusal: the worker's own `error 0 pack : ` (relayed as +/// `worker error: ...`) or the miner's `PACK OUT OF DATE : `. Returns the reason, in plain words. +pub fn pack_refusal(text: &str) -> Option { + if let Some(i) = text.find("PACK OUT OF DATE") { + let rest = text[i + "PACK OUT OF DATE".len()..].trim_start_matches(':').trim(); + return Some(rest.split_once(": ").map(|(_, why)| why).unwrap_or(rest).to_string()); + } + if let Some(i) = text.find("error 0 pack ") { + let rest = &text[i + "error 0 pack ".len()..]; + let (_, why) = rest.split_once(": ")?; + return Some(why.trim().to_string()); + } + None +} + +/// The epoch seed hex an exported pack names (`epoch_seed_hex ` in its seeds.txt), 16 chars. +pub fn pack_epoch_of(seeds_txt: &str) -> Option { + seeds_txt.lines().find_map(|l| l.strip_prefix("epoch_seed_hex ")).map(|h| h.trim().chars().take(16).collect()) +} + #[cfg(test)] mod tests { use super::*; + // The refusal as PC 1 and PC 2 logged it on 5 October 2026 (epoch 34), the miner's own line, and non-refusals. + const REFUSAL: &str = "! 1791224840.037 worker error: error 0 pack packs\\devnet: the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT (wrong seeds.txt for this pack?)"; + const MINER_LINE: &str = "1791224840.100 PACK OUT OF DATE packs\\devnet: the worker refused its program pack (the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT); rebuilding the program pack before the restart"; + + #[test] + fn pack_refusal_reads_both_lines_and_nothing_else() { + assert_eq!(pack_refusal(REFUSAL).unwrap(), "the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT (wrong seeds.txt for this pack?)"); + assert!(pack_refusal(MINER_LINE).unwrap().starts_with("the worker refused its program pack")); + assert_eq!(pack_refusal("! 1791224651.247 worker error: error 5838 epoch seed mismatch: this worker holds epoch bed7ab62cbece66c"), None); + assert_eq!(pack_refusal(STATUS_OK), None); + assert_eq!(pack_epoch_of("epoch_seed_hex 009858237e118f69abc8d096e9b1af21c24539eaecdfd1b896588825660a69ec\nday_seed_hex 69676e65\n").as_deref(), Some("009858237e118f69")); + assert_eq!(pack_epoch_of("day_seed_hex 69676e65\n"), None); + } + + /// Known-good: a refusal rebuilds the pack, once per refusal, and a new epoch starts the count again. + #[test] + fn pack_rebuilds_known_good() { + let mut p = PackRebuilds::new(); + assert_eq!(p.decide("009858237e118f69", "x"), PackAction::Rebuild { n: 1, cap: PACK_REBUILD_CAP }); + assert_eq!(p.decide("009858237e118f69", "x"), PackAction::Rebuild { n: 2, cap: PACK_REBUILD_CAP }); + assert_eq!(p.decide("5e5d0c3b2a19f8e7", "x"), PackAction::Rebuild { n: 1, cap: PACK_REBUILD_CAP }); + assert_eq!(p.count(), 1); + } + + /// Known-mismatched: a pack the worker refuses after every rebuild stops at the cap with the reason in plain + /// words, and stays stopped for that epoch. + #[test] + fn pack_rebuilds_known_mismatched_gives_up_at_the_cap() { + let mut p = PackRebuilds::new(); + for n in 1..=PACK_REBUILD_CAP { + assert_eq!(p.decide("009858237e118f69", "the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT"), PackAction::Rebuild { n, cap: PACK_REBUILD_CAP }); + } + match p.decide("009858237e118f69", "the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT") { + PackAction::GiveUp { n, reason } => { + assert_eq!(n, PACK_REBUILD_CAP); + assert_eq!(reason, "the program pack still fails after 3 rebuilds this epoch (the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT); the worker and the pack disagree"); + } + a => panic!("{a:?}"), + } + assert!(matches!(p.decide("009858237e118f69", "x"), PackAction::GiveUp { .. })); + assert_eq!(p.decide("5e5d0c3b2a19f8e7", "x"), PackAction::Rebuild { n: 1, cap: PACK_REBUILD_CAP }); + } + // Lines as igneum-miner 0.3.x prints them (devnet v4, 4 October 2026; identities and labels shortened). const STATUS_OK: &str = "1791138616.597 STATUS 'win-1' [worker]: 70s jobs=486 accepted=3 rejected=0 mismatched=0 extra=0 rate=0.04 blocks/s hash=123.90 MH/s wall (124.20 MH/s inside jobs) now=124.10 MH/s wall (124.30 MH/s inside jobs, 70 jobs, seed walk 0 calls) template_age=0.31s synced=true idle=0.2% (last 10s: 0.1%) queued=2 restarts=0 faults=0 identities=8 accepted_by_identity=1/0/1/0/0/1/0/0"; const STATUS_ZERO: &str = "1791138626.597 STATUS 'win-1' [worker]: 80s jobs=486 accepted=3 rejected=0 mismatched=0 extra=0 rate=0.04 blocks/s hash=108.41 MH/s wall (124.20 MH/s inside jobs) now=0.00 MH/s wall (0.00 MH/s inside jobs, 0 jobs, seed walk 0 calls) template_age=0.31s synced=true idle=12.5% (last 10s: 100.0%) queued=2 restarts=0 faults=0 identities=8 accepted_by_identity=1/0/1/0/0/1/0/0"; diff --git a/igneum-pow/src/lib.rs b/igneum-pow/src/lib.rs index a35f37a4c..2c51a6208 100644 --- a/igneum-pow/src/lib.rs +++ b/igneum-pow/src/lib.rs @@ -26,6 +26,7 @@ pub mod bind; pub mod emit; pub mod generator; pub mod memhard; +pub mod packcheck; pub mod seed; pub mod verify; diff --git a/igneum-pow/src/packcheck.rs b/igneum-pow/src/packcheck.rs new file mode 100644 index 000000000..fdcd185ab --- /dev/null +++ b/igneum-pow/src/packcheck.rs @@ -0,0 +1,304 @@ +//! A program pack on disk, read the way the one-click workers read it (`proto-cuda/nvrtc/packfile.h`, `pf_load`), +//! and checked against the seeds the node is on. +//! +//! The rule (5 October 2026, the epoch 34 incident on both PCs): a pack's `IGNEUM_SEEDW_INIT` is the seed words of +//! the program's ATTEMPT, `attempt_words(epoch_seed, IGNEUM_PROGRAM_ATTEMPT)`, not the words of the bare seed. The +//! generator retries a rejected candidate with `seed || k_le32` (spec 01 section 1.4.6), so from attempt 1 on the +//! bare-seed words and the pack's words differ. The workers derived the expected words from the bare seed, refused +//! every pack of a retried program ("the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT") and the miner +//! and the app restarted them forever. Epoch 34 (seed `009858237e11...`) was the first live epoch whose program is +//! a later attempt. This module is the one place that rule is written in Rust; the miner checks every pack it +//! writes with it before a worker sees the pack, and the tests pin the attempt vectors the C side also pins. + +use crate::generator::attempt_words; +use crate::seed::seed_words_from_bytes; +use std::fmt; +use std::path::Path; + +/// What a pack says about itself. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackIdentity { + pub epoch_hex: String, + pub day_hex: String, + pub attempt: u32, + pub seedw: [u32; 8], + pub keyw: [u32; 8], +} + +/// Why a pack is not the one a worker should mine with. `Display` is the plain-words line the logs carry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PackFault { + /// program.h or seeds.txt is missing or does not parse. + Unreadable(String), + /// A well-formed pack for other seeds than the node's: the pack is stale (or the node moved on). + OutOfDate { pack_epoch: String, pack_day: String, want_epoch: String, want_day: String }, + /// The files of one pack contradict each other (seeds.txt against program.h, or the init words against the + /// seeds and the attempt): a half-written or hand-edited pack, or a worker and an exporter on different rules. + Disagree(String), +} + +impl fmt::Display for PackFault { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + PackFault::Unreadable(w) => write!(f, "program pack unreadable: {w}"), + PackFault::OutOfDate { pack_epoch, pack_day, want_epoch, want_day } => write!( + f, + "program pack out of date: the pack is for epoch {} day {}, the node is on epoch {} day {}", + short(pack_epoch), + day_label(pack_day), + short(want_epoch), + day_label(want_day) + ), + PackFault::Disagree(w) => write!(f, "program pack and its seeds disagree: {w}"), + } + } +} + +impl std::error::Error for PackFault {} + +fn short(hex: &str) -> &str { + if hex.len() >= 16 { + &hex[..16] + } else { + hex + } +} + +/// The day bytes are `igneum-day/` followed by the little-endian day index (`bind::day_bytes`); print the index +/// when the hex has that shape, else the hex. +fn day_label(hex: &str) -> String { + const PREFIX: &str = "69676e65756d2d6461792f"; // "igneum-day/" + if let Some(rest) = hex.strip_prefix(PREFIX) { + if let Some(bytes) = unhex(rest) { + let mut v = 0u64; + for (i, b) in bytes.iter().enumerate().take(8) { + v |= (*b as u64) << (8 * i); + } + return v.to_string(); + } + } + hex.to_string() +} + +pub fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn unhex(s: &str) -> Option> { + if s.len() % 2 != 0 { + return None; + } + (0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).ok()).collect() +} + +/// `#define NAME ` in a header; the value as text, trimmed, with a trailing `//` comment removed. +fn define(text: &str, name: &str) -> Option { + for line in text.lines() { + let t = line.trim_start(); + let Some(rest) = t.strip_prefix("#define ") else { continue }; + let rest = rest.trim_start(); + let Some(after) = rest.strip_prefix(name) else { continue }; + if !after.starts_with(|c: char| c.is_whitespace()) { + continue; + } + let v = after.trim(); + let v = v.split("//").next().unwrap_or("").trim(); + return Some(v.to_string()); + } + None +} + +fn define_str(text: &str, name: &str) -> Option { + let v = define(text, name)?; + let v = v.strip_prefix('"')?.strip_suffix('"')?; + Some(v.to_string()) +} + +fn define_u32(text: &str, name: &str) -> Option { + let v = define(text, name)?; + let v = v.trim_end_matches('u'); + if let Some(h) = v.strip_prefix("0x") { + u32::from_str_radix(h, 16).ok() + } else { + v.parse().ok() + } +} + +fn define_words(text: &str, name: &str) -> Option<[u32; 8]> { + let v = define(text, name)?; + let inner = v.trim().strip_prefix('{')?.strip_suffix('}')?; + let mut out = [0u32; 8]; + let mut n = 0; + for part in inner.split(',') { + let p = part.trim().trim_end_matches('u'); + if p.is_empty() { + continue; + } + if n >= 8 { + return None; + } + out[n] = if let Some(h) = p.strip_prefix("0x") { u32::from_str_radix(h, 16).ok()? } else { p.parse().ok()? }; + n += 1; + } + (n == 8).then_some(out) +} + +/// One `key value` line of seeds.txt. +fn seeds_line(text: &str, key: &str) -> Option { + text.lines().find_map(|l| l.strip_prefix(key).and_then(|r| r.strip_prefix(' ')).map(|v| v.trim().to_string())) +} + +/// Checks the texts of a pack (program.h, and seeds.txt when it exists) against the seeds a worker will be asked +/// to mine with. Pure: the miner and the tests call it with file contents. +pub fn verify_pack_texts(program_h: &str, seeds_txt: Option<&str>, want_epoch: &[u8], want_day: &[u8]) -> Result { + let seedw = define_words(program_h, "IGNEUM_SEEDW_INIT").ok_or_else(|| PackFault::Unreadable("program.h has no IGNEUM_SEEDW_INIT with 8 words".into()))?; + let keyw = define_words(program_h, "IGNEUM_KEY_INIT").ok_or_else(|| PackFault::Unreadable("program.h has no IGNEUM_KEY_INIT with 8 words".into()))?; + let attempt = define_u32(program_h, "IGNEUM_PROGRAM_ATTEMPT").unwrap_or(0); + let mut epoch_hex = define_str(program_h, "IGNEUM_SEED_BYTES_HEX").unwrap_or_default(); + let mut day_hex = define_str(program_h, "IGNEUM_DAY_BYTES_HEX").unwrap_or_default(); + if let Some(s) = seeds_txt { + let e = seeds_line(s, "epoch_seed_hex").ok_or_else(|| PackFault::Unreadable("seeds.txt has no epoch_seed_hex line".into()))?; + let d = seeds_line(s, "day_seed_hex").ok_or_else(|| PackFault::Unreadable("seeds.txt has no day_seed_hex line".into()))?; + if !epoch_hex.is_empty() && !epoch_hex.eq_ignore_ascii_case(&e) { + return Err(PackFault::Disagree(format!("seeds.txt names epoch {} but program.h was generated for epoch {} (a pack half rewritten?)", short(&e), short(&epoch_hex)))); + } + if !day_hex.is_empty() && !day_hex.eq_ignore_ascii_case(&d) { + return Err(PackFault::Disagree(format!("seeds.txt names day {} but program.h was generated for day {}", day_label(&d), day_label(&day_hex)))); + } + epoch_hex = e.to_ascii_lowercase(); + day_hex = d.to_ascii_lowercase(); + } + if epoch_hex.is_empty() || day_hex.is_empty() { + return Err(PackFault::Unreadable("no seeds: neither seeds.txt nor IGNEUM_SEED_BYTES_HEX / IGNEUM_DAY_BYTES_HEX in program.h".into())); + } + let epoch_bytes = unhex(&epoch_hex).filter(|b| b.len() == 32).ok_or_else(|| PackFault::Unreadable("the epoch seed is not 32 bytes of hex".into()))?; + let day_bytes = unhex(&day_hex).ok_or_else(|| PackFault::Unreadable("the day seed hex is malformed".into()))?; + // The pack's own consistency first: a pack that contradicts itself is never "out of date", it is broken + let want_w = attempt_words(&epoch_bytes, attempt); + if want_w != seedw { + return Err(PackFault::Disagree(format!( + "IGNEUM_SEEDW_INIT is not attempt {attempt} of the epoch seed {} (the words of attempt {attempt} are {:08x} {:08x} ..., the pack has {:08x} {:08x} ...)", + short(&epoch_hex), + want_w[0], + want_w[1], + seedw[0], + seedw[1] + ))); + } + let want_k = seed_words_from_bytes(&day_bytes); + if want_k != keyw { + return Err(PackFault::Disagree(format!("IGNEUM_KEY_INIT is not the key of the day seed {} ", day_label(&day_hex)))); + } + let want_epoch_hex = hex(want_epoch); + let want_day_hex = hex(want_day); + if epoch_hex != want_epoch_hex || day_hex != want_day_hex { + return Err(PackFault::OutOfDate { pack_epoch: epoch_hex, pack_day: day_hex, want_epoch: want_epoch_hex, want_day: want_day_hex }); + } + Ok(PackIdentity { epoch_hex, day_hex, attempt, seedw, keyw }) +} + +/// [`verify_pack_texts`] over a pack directory. +pub fn verify_pack_dir(dir: &Path, want_epoch: &[u8], want_day: &[u8]) -> Result { + let program_h = std::fs::read_to_string(dir.join("program.h")).map_err(|e| PackFault::Unreadable(format!("cannot read {}/program.h: {e}", dir.display())))?; + let seeds = std::fs::read_to_string(dir.join("seeds.txt")).ok(); + verify_pack_texts(&program_h, seeds.as_deref(), want_epoch, want_day) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::emit::program_header; + use crate::generator::generate_from_seed_bytes; + use crate::verify::Epoch; + + // The two live devnet epochs of 5 October 2026 (epoch 33 mined, epoch 34 refused by the one-click workers) + const EPOCH_33: &str = "bed7ab62cbece66cf791485336d81d90fa1452ffed28ecd8a7416960ef64164c"; + const EPOCH_34: &str = "009858237e118f69abc8d096e9b1af21c24539eaecdfd1b896588825660a69ec"; + // "igneum-day/" || le64(20731) + const DAY_20731: &str = "69676e65756d2d6461792ffb50000000000000"; + + fn bytes(h: &str) -> Vec { + unhex(h).unwrap() + } + + /// The attempt vectors the C side pins too (proto-cuda/nvrtc/emu/packfile-test.c): a change to either + /// derivation fails on one side first. + #[test] + fn attempt_words_vectors_shared_with_the_workers() { + let e = bytes(EPOCH_34); + assert_eq!(attempt_words(&e, 0), [0x06af2a61, 0x4d67274e, 0x4ebda738, 0xad1dea73, 0x6233cd8c, 0x50371601, 0x39d0b873, 0x6af024a2]); + assert_eq!(attempt_words(&e, 1), [0x0dcff56b, 0x6b1beb0d, 0x234dc70c, 0xe4016fa9, 0x72397152, 0xb558aa79, 0x3ffb3299, 0x72b9962e]); + // epoch 33's bare words, as the CUDA worker printed them on 5 October ("seed words be5983a6 f750dab7 ...") + assert_eq!(attempt_words(&bytes(EPOCH_33), 0)[..2], [0xbe5983a6, 0xf750dab7]); + } + + /// The incident: epoch 34's program is a later attempt, epoch 33's is the bare seed. A worker that derives the + /// words from the bare seed accepts 33 and refuses 34. + #[test] + fn epoch_34_program_is_a_later_attempt() { + let p34 = generate_from_seed_bytes("epoch 34", &bytes(EPOCH_34)); + assert!(p34.attempt >= 1, "epoch 34 must be a retried program for the incident to reproduce; attempt {}", p34.attempt); + assert_eq!(p34.seed, attempt_words(&bytes(EPOCH_34), p34.attempt)); + assert_ne!(p34.seed, attempt_words(&bytes(EPOCH_34), 0)); + let p33 = generate_from_seed_bytes("epoch 33", &bytes(EPOCH_33)); + assert_eq!(p33.attempt, 0); + } + + fn pack_texts(epoch_hex: &str, day_hex: &str) -> (String, String, u32) { + let (e, d) = (bytes(epoch_hex), bytes(day_hex)); + let epoch = Epoch::from_seed_bytes(&e, &d, "test"); + let h = program_header(&epoch.program, "test day", &epoch.dataset); + let s = format!("epoch_seed_hex {epoch_hex}\nday_seed_hex {day_hex}\nday_index 20731\n"); + (h, s, epoch.program.attempt) + } + + /// Known-good: the pack of a retried program verifies against its own seeds, with its attempt. + #[test] + fn known_good_pack_of_a_later_attempt_verifies() { + let (h, s, attempt) = pack_texts(EPOCH_34, DAY_20731); + assert!(attempt >= 1); + let id = verify_pack_texts(&h, Some(&s), &bytes(EPOCH_34), &bytes(DAY_20731)).expect("the pack verifies"); + assert_eq!(id.attempt, attempt); + assert_eq!(id.epoch_hex, EPOCH_34); + assert_eq!(id.seedw, attempt_words(&bytes(EPOCH_34), attempt)); + // without seeds.txt program.h's own bytes carry the pack + assert!(verify_pack_texts(&h, None, &bytes(EPOCH_34), &bytes(DAY_20731)).is_ok()); + } + + /// Known-mismatched: a well-formed pack for the previous epoch is "out of date" against the new one, in plain + /// words with both epochs named. + #[test] + fn known_mismatched_pack_is_out_of_date() { + let (h, s, _) = pack_texts(EPOCH_33, DAY_20731); + let err = verify_pack_texts(&h, Some(&s), &bytes(EPOCH_34), &bytes(DAY_20731)).unwrap_err(); + assert!(matches!(err, PackFault::OutOfDate { .. }), "{err}"); + assert_eq!(err.to_string(), "program pack out of date: the pack is for epoch bed7ab62cbece66c day 20731, the node is on epoch 009858237e118f69 day 20731"); + } + + /// A pack that contradicts itself is "disagree", never "out of date": seeds.txt of one epoch with program.h of + /// another (a half rewritten directory), or init words that are not the attempt's words (a worker on the old + /// rule would have produced this verdict for every retried program). + #[test] + fn inconsistent_pack_disagrees() { + let (h33, _, _) = pack_texts(EPOCH_33, DAY_20731); + let s34 = format!("epoch_seed_hex {EPOCH_34}\nday_seed_hex {DAY_20731}\n"); + let err = verify_pack_texts(&h33, Some(&s34), &bytes(EPOCH_34), &bytes(DAY_20731)).unwrap_err(); + assert!(matches!(err, PackFault::Disagree(_)), "{err}"); + assert!(err.to_string().starts_with("program pack and its seeds disagree: seeds.txt names epoch 009858237e118f69"), "{err}"); + + let (h34, s, _) = pack_texts(EPOCH_34, DAY_20731); + let bare = attempt_words(&bytes(EPOCH_34), 0); + let edited = h34.lines().map(|l| if l.starts_with("#define IGNEUM_SEEDW_INIT") { format!("#define IGNEUM_SEEDW_INIT {{ {} }}", bare.iter().map(|w| format!("0x{w:08x}")).collect::>().join(", ")) } else { l.to_string() }).collect::>().join("\n"); + let err = verify_pack_texts(&edited, Some(&s), &bytes(EPOCH_34), &bytes(DAY_20731)).unwrap_err(); + assert!(err.to_string().contains("IGNEUM_SEEDW_INIT is not attempt"), "{err}"); + // and a pack with no attempt line at all is read as attempt 0 (the packs before generator version 2) + let no_attempt = h34.lines().filter(|l| !l.starts_with("#define IGNEUM_PROGRAM_ATTEMPT")).collect::>().join("\n"); + assert!(verify_pack_texts(&no_attempt, Some(&s), &bytes(EPOCH_34), &bytes(DAY_20731)).is_err()); + } + + #[test] + fn day_label_reads_the_index() { + assert_eq!(day_label(DAY_20731), "20731"); + assert_eq!(day_label("abcd"), "abcd"); + } +} diff --git a/proto-cuda/nvrtc/emu/packfile-test.c b/proto-cuda/nvrtc/emu/packfile-test.c new file mode 100644 index 000000000..c2a4ffb11 --- /dev/null +++ b/proto-cuda/nvrtc/emu/packfile-test.c @@ -0,0 +1,115 @@ +// packfile-test.c: the pack loader's seed rule (packfile.h pf_load) on a known-good and a known-mismatched pack. +// C99, no GPU, no NVRTC: `emu/packfile-test.sh` compiles and runs it on the Mac in a second, and CI runs it too. +// +// 5 October 2026, epoch 34 on both PCs: pf_load derived the expected IGNEUM_SEEDW_INIT from the bare epoch seed, but +// a pack carries the words of its program's ATTEMPT (igneum-pow attempt_words: seed || k_le32 for k > 0), so every +// pack of a retried program was refused and the workers restarted for an hour. These cases pin the rule: +// 1. the attempt vectors of epoch 34's seed, the same constants igneum-pow/src/packcheck.rs pins (one vector, two +// implementations: a change on either side fails here or there first); +// 2. known-good: a pack of attempt 1 with seeds.txt loads, and so does the checked-in attempt-0 pack; +// 3. known-mismatched: the same pack with the bare seed's words under IGNEUM_PROGRAM_ATTEMPT 1 (what the old +// rule expected) is refused in plain words; a seeds.txt of another epoch is refused as a disagreement. +#include "../packfile.h" +#include +#include + +#define EPOCH_34 "009858237e118f69abc8d096e9b1af21c24539eaecdfd1b896588825660a69ec" +#define EPOCH_33 "bed7ab62cbece66cf791485336d81d90fa1452ffed28ecd8a7416960ef64164c" +#define DAY_20731 "69676e65756d2d6461792ffb50000000000000" + +static int failures = 0; +#define CHECK(cond, what) do { if (cond) printf("ok %s\n", what); else { printf("FAIL %s (%s:%d)\n", what, __FILE__, __LINE__); failures++; } } while (0) + +static void write_file(const char* dir, const char* name, const char* text) { + char path[1024]; + FILE* f; + snprintf(path, sizeof(path), "%s/%s", dir, name); + f = fopen(path, "wb"); + if (!f) { perror(path); exit(2); } + fputs(text, f); + fclose(f); +} + +static void words_hex(const uint32_t w[8], char* out) { + int i; + out[0] = 0; + for (i = 0; i < 8; ++i) sprintf(out + strlen(out), "%s0x%08x", i ? ", " : "", w[i]); +} + +// A program.h with only what pf_load reads, for the given seeds, attempt and init words. +static void write_program_h(const char* dir, const char* epochHex, const char* dayHex, uint32_t attempt, const uint32_t seedw[8], const uint32_t keyw[8]) { + char sw[200], kw[200], text[2000]; + words_hex(seedw, sw); words_hex(keyw, kw); + snprintf(text, sizeof(text), + "// test pack\n#pragma once\n#define IGNEUM_SEED_STRING \"test\"\n#define IGNEUM_SEED_BYTES_HEX \"%s\"\n#define IGNEUM_GENERATOR 2\n" + "#define IGNEUM_PROGRAM_ATTEMPT %u\n#define IGNEUM_DAY_BYTES_HEX \"%s\"\n#define IGNEUM_DATASET_LOG2 28\n#define IGNEUM_DATASET_MODE 1\n" + "#define IGNEUM_SEEDW_INIT { %s }\n#define IGNEUM_KEY_INIT { %s }\n#define IGNEUM_CACHE_LOG2_WORDS 26\n#define IGNEUM_CACHE_SEGMENTS 4096u\n", + epochHex, (unsigned)attempt, dayHex, sw, kw); + write_file(dir, "program.h", text); +} + +int main(int argc, char** argv) { + const char* checked_in = argc > 1 ? argv[1] : NULL; // proto-cuda/packs/igneum-devnet-v4-epoch0 (attempt 0, no seeds.txt) + char dir[512]; + uint8_t e34[32], day[64]; + size_t n = 0, dn = 0; + uint32_t bare[8], att1[8], keyw[8]; + static const uint32_t want_bare[8] = { 0x06af2a61, 0x4d67274e, 0x4ebda738, 0xad1dea73, 0x6233cd8c, 0x50371601, 0x39d0b873, 0x6af024a2 }; + static const uint32_t want_att1[8] = { 0x0dcff56b, 0x6b1beb0d, 0x234dc70c, 0xe4016fa9, 0x72397152, 0xb558aa79, 0x3ffb3299, 0x72b9962e }; + PfPack pk; + char err[512]; + + pf_unhex(EPOCH_34, e34, 32, &n); + pf_unhex(DAY_20731, day, sizeof(day), &dn); + CHECK(n == 32 && dn == 19, "the fixture seeds unhex (32 epoch bytes, 19 day bytes)"); + + // 1. the attempt vectors shared with igneum-pow + pf_program_words(e34, 32, 0, bare); + pf_program_words(e34, 32, 1, att1); + pf_seed_words_from_bytes(day, dn, keyw); + CHECK(memcmp(bare, want_bare, 32) == 0, "attempt 0 of epoch 34 = the bare seed words (06af2a61 4d67274e ...)"); + CHECK(memcmp(att1, want_att1, 32) == 0, "attempt 1 of epoch 34 = words of seed || 01000000 (0dcff56b 6b1beb0d ...)"); + CHECK(memcmp(bare, att1, 32) != 0, "the two attempts differ"); + + // 2. known-good: a pack of attempt 1 with seeds.txt, as igneum-miner writes it + snprintf(dir, sizeof(dir), "%s/igneum-packfile-test-%d", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp", (int)getpid()); + mkdir(dir, 0755); + write_program_h(dir, EPOCH_34, DAY_20731, 1, att1, keyw); + write_file(dir, "seeds.txt", "epoch_seed_hex " EPOCH_34 "\nday_seed_hex " DAY_20731 "\nday_index 20731\n"); + err[0] = 0; + CHECK(pf_load(dir, &pk, err, sizeof(err)) == 1, "known-good: the attempt-1 pack loads"); + if (err[0]) printf(" (%s)\n", err); + CHECK(pk.attempt == 1 && memcmp(pk.seedw, att1, 32) == 0 && strcmp(pk.epochHex, EPOCH_34) == 0, "known-good: attempt, words and epoch hex read back"); + if (checked_in) { + err[0] = 0; + CHECK(pf_load(checked_in, &pk, err, sizeof(err)) == 1, "known-good: the checked-in attempt-0 pack loads from program.h's own bytes"); + if (err[0]) printf(" (%s)\n", err); + CHECK(pk.attempt == 0, "the checked-in pack is attempt 0"); + } + + // 3. known-mismatched: the bare words under attempt 1 (the old rule's expectation) are refused, in plain words + write_program_h(dir, EPOCH_34, DAY_20731, 1, bare, keyw); + err[0] = 0; + CHECK(pf_load(dir, &pk, err, sizeof(err)) == 0, "known-mismatched: bare words under attempt 1 are refused"); + CHECK(strstr(err, "program pack and its seeds disagree: IGNEUM_SEEDW_INIT is not attempt 1 of the epoch seed 009858237e118f69") == err, "the refusal names the attempt and the epoch in plain words"); + printf(" (%s)\n", err); + // and a seeds.txt of another epoch against this program.h is a disagreement, not a load + write_program_h(dir, EPOCH_34, DAY_20731, 1, att1, keyw); + write_file(dir, "seeds.txt", "epoch_seed_hex " EPOCH_33 "\nday_seed_hex " DAY_20731 "\n"); + err[0] = 0; + CHECK(pf_load(dir, &pk, err, sizeof(err)) == 0, "known-mismatched: seeds.txt of epoch 33 with program.h of epoch 34 is refused"); + CHECK(strstr(err, "seeds.txt epoch_seed_hex differs from program.h") != NULL, "the refusal says the files disagree"); + // a pack with no IGNEUM_PROGRAM_ATTEMPT line is attempt 0 (packs before generator version 2) + { + char text[2000], sw[200], kw[200]; + words_hex(bare, sw); words_hex(keyw, kw); + snprintf(text, sizeof(text), "#define IGNEUM_SEED_BYTES_HEX \"%s\"\n#define IGNEUM_DAY_BYTES_HEX \"%s\"\n#define IGNEUM_DATASET_LOG2 28\n#define IGNEUM_DATASET_MODE 1\n#define IGNEUM_SEEDW_INIT { %s }\n#define IGNEUM_KEY_INIT { %s }\n#define IGNEUM_CACHE_LOG2_WORDS 26\n#define IGNEUM_CACHE_SEGMENTS 4096u\n", EPOCH_34, DAY_20731, sw, kw); + write_file(dir, "program.h", text); + write_file(dir, "seeds.txt", "epoch_seed_hex " EPOCH_34 "\nday_seed_hex " DAY_20731 "\n"); + err[0] = 0; + CHECK(pf_load(dir, &pk, err, sizeof(err)) == 1 && pk.attempt == 0, "no attempt line reads as attempt 0 and the bare words load"); + } + + printf("%s: %d failure(s)\n", argv[0], failures); + return failures ? 1 : 0; +} diff --git a/proto-cuda/nvrtc/emu/packfile-test.sh b/proto-cuda/nvrtc/emu/packfile-test.sh new file mode 100755 index 000000000..5ccd2b51d --- /dev/null +++ b/proto-cuda/nvrtc/emu/packfile-test.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# The pack loader's seed rule (packfile.h) on a known-good and a known-mismatched pack: emu/packfile-test.c, C99, +# no GPU. Runs on the Mac in a second and in CI. Usage: emu/packfile-test.sh +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../../.." && pwd)" +OUT="${TMPDIR:-/tmp}/igneum-packfile-test" +mkdir -p "$OUT" +CC="${CC:-cc}" +"$CC" -std=c99 -Wall -Wextra -Wno-unused-function -O1 -o "$OUT/packfile-test" "$HERE/packfile-test.c" +"$OUT/packfile-test" "$ROOT/proto-cuda/packs/igneum-devnet-v4-epoch0" diff --git a/proto-cuda/nvrtc/packfile.h b/proto-cuda/nvrtc/packfile.h index fa407f93c..c711f6d7b 100644 --- a/proto-cuda/nvrtc/packfile.h +++ b/proto-cuda/nvrtc/packfile.h @@ -23,6 +23,7 @@ typedef struct { // program.h uint32_t datasetLog2, cacheLog2Words, cacheSegments, datasetMode, generator; + uint32_t attempt; // IGNEUM_PROGRAM_ATTEMPT: seedw are the words of this attempt of the epoch seed (0 = bare seed) uint32_t seedw[8], keyw[8]; char seedString[600]; // seeds.txt (or program.h): the seeds as the worker protocol carries them @@ -208,6 +209,20 @@ static void pf_seed_words_from_bytes(const uint8_t* b, size_t n, uint32_t out[8] } } +// attempt_words of igneum-pow/src/generator.rs: the words of attempt k of a program seed are +// seed_words_from_bytes(seed || k_le32) for k > 0 and the bare seed's words for k = 0. The generator retries a +// rejected candidate with the next attempt, so a pack's IGNEUM_SEEDW_INIT is the words of IGNEUM_PROGRAM_ATTEMPT, +// not of the bare seed. 5 October 2026: pf_load derived the expected words from the bare seed and refused every +// pack of a retried program (epoch 34, both PCs, the miner restarting the worker for an hour); this is the rule. +static void pf_program_words(const uint8_t* b, size_t n, uint32_t attempt, uint32_t out[8]) { + uint8_t buf[256 + 4]; + if (attempt == 0) { pf_seed_words_from_bytes(b, n, out); return; } + if (n > 256) n = 256; + memcpy(buf, b, n); + buf[n] = (uint8_t)attempt; buf[n + 1] = (uint8_t)(attempt >> 8); buf[n + 2] = (uint8_t)(attempt >> 16); buf[n + 3] = (uint8_t)(attempt >> 24); + pf_seed_words_from_bytes(buf, n + 4, out); +} + static uint64_t pf_fnv1a64(const void* p, size_t n) { const uint8_t* b = (const uint8_t*)p; uint64_t h = 0xcbf29ce484222325ull; @@ -253,6 +268,7 @@ static int pf_load(const char* dir, PfPack* pk, char* err, size_t cap) { if (!pf_define_u32(prog, "IGNEUM_CACHE_LOG2_WORDS", &pk->cacheLog2Words)) { free(prog); return pf_fail(err, cap, "program.h has no IGNEUM_CACHE_LOG2_WORDS"); } if (!pf_define_u32(prog, "IGNEUM_CACHE_SEGMENTS", &pk->cacheSegments)) { free(prog); return pf_fail(err, cap, "program.h has no IGNEUM_CACHE_SEGMENTS"); } if (!pf_define_u32(prog, "IGNEUM_GENERATOR", &pk->generator)) pk->generator = 1; + if (!pf_define_u32(prog, "IGNEUM_PROGRAM_ATTEMPT", &pk->attempt)) pk->attempt = 0; if (pf_define_words(prog, "IGNEUM_SEEDW_INIT", pk->seedw, 8) != 8) { free(prog); return pf_fail(err, cap, "program.h has no IGNEUM_SEEDW_INIT with 8 words"); } if (pf_define_words(prog, "IGNEUM_KEY_INIT", pk->keyw, 8) != 8) { free(prog); return pf_fail(err, cap, "program.h has no IGNEUM_KEY_INIT with 8 words"); } if (!pf_define_str(prog, "IGNEUM_SEED_STRING", pk->seedString, sizeof(pk->seedString))) strncpy(pk->seedString, "(no IGNEUM_SEED_STRING)", sizeof(pk->seedString) - 1); @@ -276,12 +292,18 @@ static int pf_load(const char* dir, PfPack* pk, char* err, size_t cap) { if (!ehex[0] || !dhex[0]) return pf_fail(err, cap, "no seeds: neither seeds.txt nor IGNEUM_SEED_BYTES_HEX / IGNEUM_DAY_BYTES_HEX in program.h (a pack from igneum-pow export --seed has no byte seeds)"); if (strlen(ehex) != 64) return pf_fail(err, cap, "epoch seed is not 64 hex characters"); strcpy(pk->epochHex, ehex); strcpy(pk->dayHex, dhex); - // The seed words derived from the bytes must be the pack's own words: otherwise the pack and the seeds disagree + // The seed words derived from the bytes AND the attempt must be the pack's own words: otherwise the pack and + // its seeds disagree (a half rewritten directory, or an exporter on another rule) { uint32_t w[8]; + char m[400]; if (!pf_unhex(ehex, bytes, 32, &blen) || blen != 32) return pf_fail(err, cap, "epoch seed hex is malformed"); - pf_seed_words_from_bytes(bytes, 32, w); - if (memcmp(w, pk->seedw, 32) != 0) return pf_fail(err, cap, "the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT (wrong seeds.txt for this pack?)"); + pf_program_words(bytes, 32, pk->attempt, w); + if (memcmp(w, pk->seedw, 32) != 0) { + snprintf(m, sizeof(m), "program pack and its seeds disagree: IGNEUM_SEEDW_INIT is not attempt %u of the epoch seed %.16s (attempt %u gives %08x %08x ..., the pack has %08x %08x ...); run igneum-miner export-pack again", + (unsigned)pk->attempt, ehex, (unsigned)pk->attempt, w[0], w[1], pk->seedw[0], pk->seedw[1]); + return pf_fail(err, cap, m); + } if (!pf_unhex(dhex, bytes, sizeof(bytes), &blen)) return pf_fail(err, cap, "day seed hex is malformed"); pf_seed_words_from_bytes(bytes, blen, w); if (memcmp(w, pk->keyw, 32) != 0) return pf_fail(err, cap, "the day seed bytes do not give the pack's IGNEUM_KEY_INIT (wrong seeds.txt for this pack?)"); diff --git a/proto-cuda/nvrtc/worker.cpp b/proto-cuda/nvrtc/worker.cpp index 11597fd2a..6143e1e7c 100644 --- a/proto-cuda/nvrtc/worker.cpp +++ b/proto-cuda/nvrtc/worker.cpp @@ -394,6 +394,18 @@ static bool rtcCompile(Ctx& c, const std::string& src, const char* name, const s // --------------------------------------------------------------------------------------------- // A resident pair: one pack compiled, its cache and dataset on the device, self-tested +static bool hexEq(const std::string& a, const std::string& b) { + if (a.size() != b.size()) return false; + for (size_t i = 0; i < a.size(); ++i) if (std::tolower((unsigned char)a[i]) != std::tolower((unsigned char)b[i])) return false; + return true; +} + +struct Pair; +// A pair is the pair of a job when the job's seeds (the hex the node sent) are the pair's seeds. The derived seed +// words are no identity: a retried program's words are its attempt's words, not the bare seed's (packfile.h, +// 5 October 2026), so comparing words refused every job of a retried program. +static bool pairIs(const Pair* p, const std::string& epochHex, const std::string& dayHex); + struct Pair { std::string dir, epochHex, dayHex, seedString; uint32_t sw[8] = {0}, kw[8] = {0}; @@ -411,6 +423,10 @@ struct Pair { double raceMs = 0; }; +static bool pairIs(const Pair* p, const std::string& epochHex, const std::string& dayHex) { + return p && hexEq(p->epochHex, epochHex) && hexEq(p->dayHex, dayHex); +} + // The job loop and a race take turns on the card: a variant is timed with no job running (exclusive numbers), and // mining resumes between variants. Held per chunk by the job loop, per variant by the race. static std::mutex gpuMutex; @@ -1039,8 +1055,7 @@ static int runServe(Ctx& c, const Options& o, Pair* cur) { pf_seed_words_from_bytes(daySeed, dl, kw); double t0 = wallMs(); bool switched = false; - if ((std::memcmp(sw, cur->sw, 32) != 0 || std::memcmp(kw, cur->kw, 32) != 0) && !task && - !(prepared && std::memcmp(sw, prepared->sw, 32) == 0 && std::memcmp(kw, prepared->kw, 32) == 0)) { + if (!pairIs(cur, f[6], f[7]) && !task && !pairIs(prepared, f[6], f[7])) { // Self-heal: a job on seeds this worker has no pair for and no prepare in flight (a prepare failed, or // the miner never sent one). The miner writes a pack per pair under its --prepare-packs root; find it by // seeds.txt and build it now, in the foreground. The miner only re-sends prepare for the pair after this one. @@ -1056,14 +1071,14 @@ static int runServe(Ctx& c, const Options& o, Pair* cur) { else emit("error " + jobId + " could not build " + dir + ": " + berr); } } - if (std::memcmp(sw, cur->sw, 32) != 0 || std::memcmp(kw, cur->kw, 32) != 0) { - if (prepared && std::memcmp(sw, prepared->sw, 32) == 0 && std::memcmp(kw, prepared->kw, 32) == 0) { + if (!pairIs(cur, f[6], f[7])) { + if (pairIs(prepared, f[6], f[7])) { if (old) releasePair(c, old); old = cur; cur = prepared; prepared = nullptr; switched = true; info(fmt("switched to the prepared pair epoch %.16s day %s in %.2f ms", cur->epochHex.c_str(), cur->dayHex.c_str(), wallMs() - t0)); - } else if (std::memcmp(sw, cur->sw, 32) != 0) { + } else if (!hexEq(cur->epochHex, f[6])) { emit(fmt("need %s %s", f[6].c_str(), f[7].c_str())); // the miner prepares this pair (4 October 2026) - emit(fmt("error %s epoch seed mismatch: this worker holds epoch %.16s (seed words %08x %08x ...)%s, the job's epoch seed %.16s gives %08x %08x ...; send prepare with a pack directory", + emit(fmt("error %s epoch seed mismatch: this worker holds epoch %.16s (program words %08x %08x ...)%s, the job is for epoch %.16s (bare seed words %08x %08x ...); send prepare with a pack directory", jobId.c_str(), cur->epochHex.c_str(), cur->sw[0], cur->sw[1], prepared ? " plus one prepared pair" : "", f[6].c_str(), sw[0], sw[1])); continue; } else { diff --git a/proto-opencl/host.c b/proto-opencl/host.c index 4ec8f6f6f..8d944fdda 100644 --- a/proto-opencl/host.c +++ b/proto-opencl/host.c @@ -962,6 +962,23 @@ typedef struct { int checked; } ServePair; +static int hexEq(const char* a, const char* b) { + size_t i; + if (strlen(a) != strlen(b)) return 0; + for (i = 0; a[i]; ++i) if (tolower((unsigned char)a[i]) != tolower((unsigned char)b[i])) return 0; + return 1; +} + +/* A pair is the pair of a job when the job's seeds (the hex the node sent) are the pair's seeds. The derived seed + * words are no identity: a retried program's words are its attempt's words, not the bare seed's (packfile.h, + * 5 October 2026), so comparing words refused every job of a retried program. The compiled-in placeholder pack + * (no --pack, no prepared pair) has no seed hex; it keeps the word comparison. */ +static int pairIs(const ServePair* p, const char* epochHex, const char* dayHex, const uint32_t sw[8], const uint32_t kw[8]) { + if (!p) return 0; + if (p->epochHex[0]) return hexEq(p->epochHex, epochHex) && hexEq(p->dayHex, dayHex); + return memcmp(sw, p->sw, 32) == 0 && memcmp(kw, p->kw, 32) == 0; +} + static void releasePair(ServePair* p) { if (!p) return; if (p->ds) { clReleaseMemObject(p->ds); ++gMemReleased; } @@ -1266,15 +1283,15 @@ static int runServe(Device* dv, const DeviceInfo* di, const Options* o) { seedWordsFromBytes(epochSeed, 32, sw); seedWordsFromBytes(daySeed, dayLen, kw); t0 = wallMs(); - if (memcmp(sw, cur->sw, 32) != 0 || memcmp(kw, cur->kw, 32) != 0) { - if (prepared && memcmp(sw, prepared->sw, 32) == 0 && memcmp(kw, prepared->kw, 32) == 0) { + if (!pairIs(cur, f[6], f[7], sw, kw)) { + if (pairIs(prepared, f[6], f[7], sw, kw)) { /* The prepared pair: switch now, release the old one after this job */ if (old) releasePair(old); old = cur; cur = prepared; prepared = NULL; switched = 1; printf("info switched to the prepared pair epoch %.16s day %s in %.2f ms\n", cur->epochHex, cur->dayHex, wallMs() - t0); fflush(stdout); - } else if (memcmp(sw, cur->sw, 32) != 0) { + } else if (cur->epochHex[0] ? !hexEq(cur->epochHex, f[6]) : memcmp(sw, cur->sw, 32) != 0) { printf("need %s %s\n", f[6], f[7]); /* the miner prepares this pair (4 October 2026) */ - printf("error %s epoch seed mismatch: this worker holds %s%s (seed words %08x %08x ...)%s, the job's epoch seed %.16s gives %08x %08x ...; send prepare with a pack directory, or run igneum-miner export-pack and rebuild\n", + printf("error %s epoch seed mismatch: this worker holds %s%s (program words %08x %08x ...)%s, the job is for epoch %.16s (bare seed words %08x %08x ...); send prepare with a pack directory, or run igneum-miner export-pack and rebuild\n", jobId, cur->epochHex[0] ? "prepared epoch " : "pack \"" IGNEUM_SEED_STRING "\"", cur->epochHex[0] ? cur->epochHex : "", cur->sw[0], cur->sw[1], prepared ? " plus one prepared pair" : "", f[6], sw[0], sw[1]); fflush(stdout); continue; } else { diff --git a/relay/lib/parse.mjs b/relay/lib/parse.mjs index a8134f61e..5224f7d8e 100644 --- a/relay/lib/parse.mjs +++ b/relay/lib/parse.mjs @@ -4,6 +4,9 @@ export const kv = (text, k) => { const m = new RegExp(`(?:^|[\\s(])${k}=([^\\s,) export const kvNum = (text, k) => { const v = kv(text, k); return v === null ? null : Number(v); }; export const lastMatch = (lines, re) => { for (let i = lines.length - 1; i >= 0; i--) { const m = re.exec(lines[i]); if (m) return m; } return null; }; export const FAULT = /WORKER MISMATCH|worker error|worker exited|panicked|CUDA error|submit error/; +// A program pack the worker refused at start or the miner found stale (5 October 2026, epoch 34 on both PCs: the +// bare restart count hid an hour-long loop). The card says "pack mismatch, rebuilding" instead of a bare count. +export const PACK_MISMATCH = /error 0 pack |PACK OUT OF DATE|program pack out of date|program pack and its seeds disagree/; // A card's hash is "now" only while its worker's STATUS line (every 30 s, uploaded every 60 s) is this fresh; older // than this the card is marked stale and its hash leaves the machine total (4 October 2026: PC 2 showed 117 MH/s "now" @@ -59,6 +62,9 @@ export function parseMinerTail(tail) { const faults = lines.filter(l => FAULT.test(l)); c.faults = faults.length; if (faults.length) c.fault = faults[faults.length - 1].replace(/^\d+(\.\d+)? /, '').slice(0, 200); + const packs = lines.filter(l => PACK_MISMATCH.test(l)); + c.pack_mismatch = packs.length > 0; + if (packs.length) c.fault = `pack mismatch, rebuilding (${packs.length} refusal${packs.length === 1 ? '' : 's'} in the tail${c.restarts ? ', ' + c.restarts + ' restarts' : ''})`; const acc = lastMatch(lines, /^(\d+(?:\.\d+)?) ACCEPTED block/); if (acc) c.last_accepted_at = new Date(Number(acc[1]) * 1000).toISOString(); return c; diff --git a/relay/test/parse.test.mjs b/relay/test/parse.test.mjs index f5aa00494..f3f5e806f 100644 --- a/relay/test/parse.test.mjs +++ b/relay/test/parse.test.mjs @@ -1,7 +1,7 @@ // node --test relay/test/parse.test.mjs relay/test/auth.test.mjs (no dependencies; CI runs both in the site job) import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { parseLabel, parseMinerTail, parseAppTail, parseCardsLine, markStale, STALE_S } from '../lib/parse.mjs'; +import { parseLabel, parseMinerTail, parseAppTail, parseCardsLine, markStale, STALE_S, PACK_MISMATCH } from '../lib/parse.mjs'; test('labels: every vendor the app names, the app log, the node log, the legacy launchers', () => { assert.deepEqual(parseLabel('miner-nvidia-ae432dc7-1'), { id: 'ae432dc7', platform: 'win', stream: 'miner', vendor: 'nvidia', card: 1 }); @@ -104,3 +104,32 @@ test('app tail: the cards line gives kind and state per card, and the hot-plug e assert.deepEqual(a.hotplug.map(h => h.text), ['New card: gfx1201, mining', 'Card removed: gfx1201; its worker stopped']); assert.equal(parseAppTail('1791140830 status: x | node y | up 1h').gpus, undefined); }); + +test('pack mismatch: the refused-pack tail of 5 October 2026 reads "pack mismatch, rebuilding", a healthy tail does not', () => { + // PC 2, nvidia-1ccfe586-1, 18:27 to 18:28 UTC: the worker refused packs\devnet at every start and the miner restarted it + const refused = [ + "1791224818.418 epoch seed 009858237e118f69abc8d096e9b1af21c24539eaecdfd1b896588825660a69ec day 20731 (daa 122513): CPU program and cache ready in 179 ms", + "! 1791224818.419 worker error: error 0 pack packs\\devnet: the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT (wrong seeds.txt for this pack?)", + '! 1791224818.933 worker exited (code Some(1)); restarting it in 21 s (restart 1)', + "1791224840.037 STATUS 'nvidia-1ccfe586-1' [worker]: 22s jobs=0 accepted=0 rejected=0 fee=0 mismatched=0 extra=0 rate=0.00 blocks/s hash=0.00 MH/s wall (0.00 MH/s inside jobs) now=0.00 MH/s wall (0.00 MH/s inside jobs, 0 jobs, seed walk 0 calls) template_age=0.45s synced=true idle=100.0% (last 22s: 100.0%) queued=2 restarts=1 faults=0 identities=8 accepted_by_identity=0/0/0/0/0/0/0/0", + "! 1791224840.037 worker error: error 0 pack packs\\devnet: the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT (wrong seeds.txt for this pack?)", + '! 1791224840.548 worker exited (code Some(1)); restarting it in 41 s (restart 2)', + ].join('\n'); + const c = parseMinerTail(refused); + assert.equal(c.pack_mismatch, true); + assert.equal(c.fault, 'pack mismatch, rebuilding (2 refusals in the tail, 1 restarts)'); + assert.equal(c.restarts, 1); + // the miner's own line (0.3.11 miners) counts too + const own = parseMinerTail('1791224840.100 PACK OUT OF DATE packs\\devnet: the worker refused its program pack; rebuilding the program pack before the restart'); + assert.equal(own.pack_mismatch, true); + assert.ok(own.fault.startsWith('pack mismatch, rebuilding')); + // known-good: a mining tail with a job error that is not a refusal keeps the plain fault text + const healthy = [ + "1791140579.479 STATUS 'other-37ba0461-1' [worker]: 61s jobs=37 accepted=2 rejected=0 mismatched=0 extra=0 rate=0.02 blocks/s hash=1.28 MH/s wall (1.55 MH/s inside jobs) now=1.53 MH/s wall", + '! 1791140580.000 worker error: error 17 epoch seed mismatch: this worker holds epoch bed7ab62cbece66c', + ].join('\n'); + const h = parseMinerTail(healthy); + assert.equal(h.pack_mismatch, false); + assert.ok(h.fault.includes('worker error: error 17')); + assert.equal(PACK_MISMATCH.test('1 ACCEPTED block x'), false); +});