Merge commit '4163143' into ca2-v3
# Conflicts: # proto-cuda/nvrtc/packfile.h
This commit is contained in:
igneum-labs
commit
d28a32e403
12 changed files with 710 additions and 14 deletions
2
.github/workflows/ci.yml
vendored
2
.github/workflows/ci.yml
vendored
|
|
@ -25,6 +25,8 @@ jobs:
|
|||
- name: igneum-pow tests (release)
|
||||
working-directory: igneum-pow
|
||||
run: cargo test --release
|
||||
- name: pack loader seed rule (packfile.h on a known-good and a known-mismatched pack)
|
||||
run: bash proto-cuda/nvrtc/emu/packfile-test.sh
|
||||
- name: igneum-census build (release)
|
||||
working-directory: igneum-census
|
||||
run: cargo build --release
|
||||
|
|
|
|||
|
|
@ -395,6 +395,8 @@ struct MinerSlot {
|
|||
error_at: Option<Instant>,
|
||||
/// the per-card watchdog (src/watchdog.rs): one restart, then faulted
|
||||
watch: crate::watchdog::CardWatch,
|
||||
/// pack refusals (src/watchdog.rs): the pack is exported again before the restart, capped per epoch
|
||||
pack_rebuilds: crate::watchdog::PackRebuilds,
|
||||
}
|
||||
|
||||
pub struct Engine {
|
||||
|
|
@ -1172,6 +1174,7 @@ impl Engine {
|
|||
last_status: None,
|
||||
error_at: None,
|
||||
watch: crate::watchdog::CardWatch::new(),
|
||||
pack_rebuilds: crate::watchdog::PackRebuilds::new(),
|
||||
});
|
||||
}
|
||||
if self.miners.is_empty() {
|
||||
|
|
@ -2282,6 +2285,7 @@ impl Engine {
|
|||
if !p.alive() {
|
||||
let code = p.exit_code.unwrap_or(-1);
|
||||
let tail = tail_of(&p.log_path, 3);
|
||||
let long_tail = if code == crate::watchdog::PACK_OUT_OF_DATE_CODE { tail_of(&p.log_path, 40) } else { Vec::new() };
|
||||
self.miners[i].proc = None;
|
||||
let t = self.secs(now);
|
||||
let verdict = self.miners[i].watch.event(t, crate::watchdog::Event::Exited(code));
|
||||
|
|
@ -2291,6 +2295,37 @@ impl Engine {
|
|||
self.miners[i].needs_rebuild = true;
|
||||
}
|
||||
self.miners[i].restart_at = Some(now);
|
||||
} else if code == crate::watchdog::PACK_OUT_OF_DATE_CODE {
|
||||
// The worker refused its program pack and the miner could not rebuild it (or hit its own
|
||||
// cap): export the pack from the node again before the next start, not a blind restart;
|
||||
// at most PACK_REBUILD_CAP times per epoch, then the card shows the reason
|
||||
let why = long_tail.iter().rev().find_map(|l| crate::watchdog::pack_refusal(l)).unwrap_or_else(|| "the worker refused its program pack".into());
|
||||
let epoch = std::fs::read_to_string(self.shared.runtime.app_dir.join("packs").join("devnet").join("seeds.txt")).ok().and_then(|s| crate::watchdog::pack_epoch_of(&s)).unwrap_or_default();
|
||||
let name = self.st().mining.cards.get(card_idx).map(|c| c.name.clone()).unwrap_or_else(|| self.miners[i].label.clone());
|
||||
match self.miners[i].pack_rebuilds.decide(&epoch, &why) {
|
||||
crate::watchdog::PackAction::Rebuild { n, cap } => {
|
||||
self.shared.event("build", "program pack out of date, rebuilding");
|
||||
self.shared.log(&format!("{name}: program pack out of date, rebuilding (export {n} of {cap} for epoch {epoch}): {why}"));
|
||||
self.miners[i].prepared = false; // prepare_worker exports the pack again before the start
|
||||
self.miners[i].restart_at = Some(now);
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
||||
c.state = "restarting".into();
|
||||
c.hash_now = 0.0;
|
||||
c.message = "program pack out of date, rebuilding".into();
|
||||
}
|
||||
}
|
||||
crate::watchdog::PackAction::GiveUp { n: _, reason } => {
|
||||
self.shared.event("error", &format!("{name}: {reason}"));
|
||||
self.shared.log(&format!("{name}: {reason}; next try in 10 minutes or at the next hour"));
|
||||
self.miners[i].prepared = false;
|
||||
self.miners[i].restart_at = Some(now + Duration::from_secs(600));
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
||||
c.state = "failed".into();
|
||||
c.hash_now = 0.0;
|
||||
c.message = reason;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if verdict != crate::watchdog::Action::None {
|
||||
// exit 43: the miner gave up on its worker; once more, then the card is faulted
|
||||
self.watchdog_verdict(i, verdict, &tail);
|
||||
|
|
@ -2368,6 +2403,25 @@ impl Engine {
|
|||
}
|
||||
}
|
||||
|
||||
/// The strip, the log and the card for a program pack the worker refused or the miner found stale: the miner
|
||||
/// rebuilds the pack and restarts the worker itself (or exits 44 for the app to export it). One strip line per
|
||||
/// 30 s: the miner prints the refusal on stderr and its own line on stdout.
|
||||
fn pack_notice(&mut self, i: usize, card: usize, card_name: &str, why: &str) {
|
||||
let now = Instant::now();
|
||||
self.shared.log(&format!("{card_name}: program pack out of date, rebuilding: {why}"));
|
||||
if now.duration_since(self.last_error_event) >= Duration::from_secs(30) {
|
||||
self.last_error_event = now;
|
||||
self.shared.event("build", "program pack out of date, rebuilding");
|
||||
}
|
||||
self.miners[i].error_at = Some(now);
|
||||
let t = self.secs(now);
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart("program pack out of date, rebuilding"));
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
||||
c.hash_now = 0.0;
|
||||
c.message = "program pack out of date, rebuilding".into();
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies a watchdog verdict to slot `i` (its process already stopped or gone): a restart now with the reason on
|
||||
/// the card, or the card marked faulted with the reason in the UI and the log while the other cards keep mining.
|
||||
fn watchdog_verdict(&mut self, i: usize, verdict: crate::watchdog::Action, tail: &[String]) {
|
||||
|
|
@ -2681,6 +2735,10 @@ impl Engine {
|
|||
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
||||
c.message = "the node is not answering; the miner retries".into();
|
||||
}
|
||||
} else if let Some(why) = crate::watchdog::pack_refusal(text) {
|
||||
// the worker refused its program pack; the miner rebuilds the pack and restarts the worker itself
|
||||
// (or exits 44 for us to export it): the strip and the card name the condition in plain words
|
||||
self.pack_notice(i, card, &card_name, &why);
|
||||
} else if text.contains("WORKER MISMATCH") || text.contains("worker error") || text.contains("worker exited") || text.contains("worker killed by a guard") || text.contains("panicked") || text.contains("CUDA error") || text.contains("submit error") {
|
||||
let now = Instant::now();
|
||||
if now.duration_since(self.last_error_event) >= Duration::from_secs(30) {
|
||||
|
|
@ -2703,6 +2761,12 @@ impl Engine {
|
|||
}
|
||||
return;
|
||||
}
|
||||
if text.contains("PACK OUT OF DATE") {
|
||||
// the miner found its pack stale or refused (stdout): it rebuilds the pack before the restart
|
||||
let why = crate::watchdog::pack_refusal(text).unwrap_or_else(|| short(text, 160));
|
||||
self.pack_notice(i, card, &card_name, &why);
|
||||
return;
|
||||
}
|
||||
if text.contains(" WORKER FAULT ") {
|
||||
// the miner's guards (interval, job time, cpu re-check, stall) killed the worker; it restarts it itself
|
||||
let reason = crate::watchdog::fault_reason(text).unwrap_or_else(|| short(text, 160));
|
||||
|
|
|
|||
|
|
@ -27,6 +27,12 @@ pub const HEALTHY_RESET_S: f64 = 300.0;
|
|||
pub const MINER_GAVE_UP_CODE: i32 = 43;
|
||||
/// No `watch` reading from our node for this long: restart it.
|
||||
pub const NODE_SILENT_S: f64 = 120.0;
|
||||
/// `igneum-miner` exit code when its worker refused the program pack it was started with and the miner could not
|
||||
/// rebuild it (or rebuilt it `PACK_REBUILD_CAP` times this epoch): the app exports the pack from the node again
|
||||
/// before the next start, at most `PACK_REBUILD_CAP` times per epoch, then shows the card.
|
||||
pub const PACK_OUT_OF_DATE_CODE: i32 = 44;
|
||||
/// Pack rebuilds per epoch seed before the app stops restarting the miner on it.
|
||||
pub const PACK_REBUILD_CAP: u32 = 3;
|
||||
/// Node restarts inside this window grow the delay before the next one.
|
||||
pub const NODE_WINDOW_S: f64 = 600.0;
|
||||
|
||||
|
|
@ -280,10 +286,114 @@ fn kv_f64(line: &str, key: &str) -> Option<f64> {
|
|||
kv(line, key)?.trim_end_matches('s').parse().ok()
|
||||
}
|
||||
|
||||
/// The decision after a pack refusal (exit `PACK_OUT_OF_DATE_CODE`, or a `PACK OUT OF DATE` / `error 0 pack` line
|
||||
/// from the miner): rebuild the pack before the restart, or stop for this epoch. 5 October 2026: the app restarted
|
||||
/// two workers every 20 to 40 s for an hour on a pack neither it nor the miner had re-exported in between.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum PackAction {
|
||||
/// Export the pack from the node again (the n-th time this epoch) and start the miner on it now.
|
||||
Rebuild { n: u32, cap: u32 },
|
||||
/// The cap for this epoch is reached: show the card with this reason and wait for the next epoch (or a reset).
|
||||
GiveUp { n: u32, reason: String },
|
||||
}
|
||||
|
||||
/// Pack rebuilds per epoch seed (the epoch the exported pack names in its seeds.txt).
|
||||
#[derive(Debug, Default)]
|
||||
pub struct PackRebuilds {
|
||||
epoch: Option<String>,
|
||||
n: u32,
|
||||
}
|
||||
|
||||
impl PackRebuilds {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// The miner exited over its pack while the exported pack is for `epoch`; `why` is the miner's reason.
|
||||
pub fn decide(&mut self, epoch: &str, why: &str) -> PackAction {
|
||||
if self.epoch.as_deref() != Some(epoch) {
|
||||
self.epoch = Some(epoch.to_string());
|
||||
self.n = 0;
|
||||
}
|
||||
if self.n >= PACK_REBUILD_CAP {
|
||||
return PackAction::GiveUp { n: self.n, reason: format!("the program pack still fails after {} rebuilds this epoch ({why}); the worker and the pack disagree", self.n) };
|
||||
}
|
||||
self.n += 1;
|
||||
PackAction::Rebuild { n: self.n, cap: PACK_REBUILD_CAP }
|
||||
}
|
||||
|
||||
pub fn count(&self) -> u32 {
|
||||
self.n
|
||||
}
|
||||
}
|
||||
|
||||
/// A miner line that names a pack refusal: the worker's own `error 0 pack <dir>: <why>` (relayed as
|
||||
/// `worker error: ...`) or the miner's `PACK OUT OF DATE <dir>: <why>`. Returns the reason, in plain words.
|
||||
pub fn pack_refusal(text: &str) -> Option<String> {
|
||||
if let Some(i) = text.find("PACK OUT OF DATE") {
|
||||
let rest = text[i + "PACK OUT OF DATE".len()..].trim_start_matches(':').trim();
|
||||
return Some(rest.split_once(": ").map(|(_, why)| why).unwrap_or(rest).to_string());
|
||||
}
|
||||
if let Some(i) = text.find("error 0 pack ") {
|
||||
let rest = &text[i + "error 0 pack ".len()..];
|
||||
let (_, why) = rest.split_once(": ")?;
|
||||
return Some(why.trim().to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The epoch seed hex an exported pack names (`epoch_seed_hex <hex>` in its seeds.txt), 16 chars.
|
||||
pub fn pack_epoch_of(seeds_txt: &str) -> Option<String> {
|
||||
seeds_txt.lines().find_map(|l| l.strip_prefix("epoch_seed_hex ")).map(|h| h.trim().chars().take(16).collect())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// The refusal as PC 1 and PC 2 logged it on 5 October 2026 (epoch 34), the miner's own line, and non-refusals.
|
||||
const REFUSAL: &str = "! 1791224840.037 worker error: error 0 pack packs\\devnet: the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT (wrong seeds.txt for this pack?)";
|
||||
const MINER_LINE: &str = "1791224840.100 PACK OUT OF DATE packs\\devnet: the worker refused its program pack (the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT); rebuilding the program pack before the restart";
|
||||
|
||||
#[test]
|
||||
fn pack_refusal_reads_both_lines_and_nothing_else() {
|
||||
assert_eq!(pack_refusal(REFUSAL).unwrap(), "the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT (wrong seeds.txt for this pack?)");
|
||||
assert!(pack_refusal(MINER_LINE).unwrap().starts_with("the worker refused its program pack"));
|
||||
assert_eq!(pack_refusal("! 1791224651.247 worker error: error 5838 epoch seed mismatch: this worker holds epoch bed7ab62cbece66c"), None);
|
||||
assert_eq!(pack_refusal(STATUS_OK), None);
|
||||
assert_eq!(pack_epoch_of("epoch_seed_hex 009858237e118f69abc8d096e9b1af21c24539eaecdfd1b896588825660a69ec\nday_seed_hex 69676e65\n").as_deref(), Some("009858237e118f69"));
|
||||
assert_eq!(pack_epoch_of("day_seed_hex 69676e65\n"), None);
|
||||
}
|
||||
|
||||
/// Known-good: a refusal rebuilds the pack, once per refusal, and a new epoch starts the count again.
|
||||
#[test]
|
||||
fn pack_rebuilds_known_good() {
|
||||
let mut p = PackRebuilds::new();
|
||||
assert_eq!(p.decide("009858237e118f69", "x"), PackAction::Rebuild { n: 1, cap: PACK_REBUILD_CAP });
|
||||
assert_eq!(p.decide("009858237e118f69", "x"), PackAction::Rebuild { n: 2, cap: PACK_REBUILD_CAP });
|
||||
assert_eq!(p.decide("5e5d0c3b2a19f8e7", "x"), PackAction::Rebuild { n: 1, cap: PACK_REBUILD_CAP });
|
||||
assert_eq!(p.count(), 1);
|
||||
}
|
||||
|
||||
/// Known-mismatched: a pack the worker refuses after every rebuild stops at the cap with the reason in plain
|
||||
/// words, and stays stopped for that epoch.
|
||||
#[test]
|
||||
fn pack_rebuilds_known_mismatched_gives_up_at_the_cap() {
|
||||
let mut p = PackRebuilds::new();
|
||||
for n in 1..=PACK_REBUILD_CAP {
|
||||
assert_eq!(p.decide("009858237e118f69", "the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT"), PackAction::Rebuild { n, cap: PACK_REBUILD_CAP });
|
||||
}
|
||||
match p.decide("009858237e118f69", "the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT") {
|
||||
PackAction::GiveUp { n, reason } => {
|
||||
assert_eq!(n, PACK_REBUILD_CAP);
|
||||
assert_eq!(reason, "the program pack still fails after 3 rebuilds this epoch (the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT); the worker and the pack disagree");
|
||||
}
|
||||
a => panic!("{a:?}"),
|
||||
}
|
||||
assert!(matches!(p.decide("009858237e118f69", "x"), PackAction::GiveUp { .. }));
|
||||
assert_eq!(p.decide("5e5d0c3b2a19f8e7", "x"), PackAction::Rebuild { n: 1, cap: PACK_REBUILD_CAP });
|
||||
}
|
||||
|
||||
// Lines as igneum-miner 0.3.x prints them (devnet v4, 4 October 2026; identities and labels shortened).
|
||||
const STATUS_OK: &str = "1791138616.597 STATUS 'win-1' [worker]: 70s jobs=486 accepted=3 rejected=0 mismatched=0 extra=0 rate=0.04 blocks/s hash=123.90 MH/s wall (124.20 MH/s inside jobs) now=124.10 MH/s wall (124.30 MH/s inside jobs, 70 jobs, seed walk 0 calls) template_age=0.31s synced=true idle=0.2% (last 10s: 0.1%) queued=2 restarts=0 faults=0 identities=8 accepted_by_identity=1/0/1/0/0/1/0/0";
|
||||
const STATUS_ZERO: &str = "1791138626.597 STATUS 'win-1' [worker]: 80s jobs=486 accepted=3 rejected=0 mismatched=0 extra=0 rate=0.04 blocks/s hash=108.41 MH/s wall (124.20 MH/s inside jobs) now=0.00 MH/s wall (0.00 MH/s inside jobs, 0 jobs, seed walk 0 calls) template_age=0.31s synced=true idle=12.5% (last 10s: 100.0%) queued=2 restarts=0 faults=0 identities=8 accepted_by_identity=1/0/1/0/0/1/0/0";
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ pub mod bind;
|
|||
pub mod emit;
|
||||
pub mod generator;
|
||||
pub mod memhard;
|
||||
pub mod packcheck;
|
||||
pub mod seed;
|
||||
pub mod verify;
|
||||
|
||||
|
|
|
|||
304
igneum-pow/src/packcheck.rs
Normal file
304
igneum-pow/src/packcheck.rs
Normal file
|
|
@ -0,0 +1,304 @@
|
|||
//! A program pack on disk, read the way the one-click workers read it (`proto-cuda/nvrtc/packfile.h`, `pf_load`),
|
||||
//! and checked against the seeds the node is on.
|
||||
//!
|
||||
//! The rule (5 October 2026, the epoch 34 incident on both PCs): a pack's `IGNEUM_SEEDW_INIT` is the seed words of
|
||||
//! the program's ATTEMPT, `attempt_words(epoch_seed, IGNEUM_PROGRAM_ATTEMPT)`, not the words of the bare seed. The
|
||||
//! generator retries a rejected candidate with `seed || k_le32` (spec 01 section 1.4.6), so from attempt 1 on the
|
||||
//! bare-seed words and the pack's words differ. The workers derived the expected words from the bare seed, refused
|
||||
//! every pack of a retried program ("the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT") and the miner
|
||||
//! and the app restarted them forever. Epoch 34 (seed `009858237e11...`) was the first live epoch whose program is
|
||||
//! a later attempt. This module is the one place that rule is written in Rust; the miner checks every pack it
|
||||
//! writes with it before a worker sees the pack, and the tests pin the attempt vectors the C side also pins.
|
||||
|
||||
use crate::generator::attempt_words;
|
||||
use crate::seed::seed_words_from_bytes;
|
||||
use std::fmt;
|
||||
use std::path::Path;
|
||||
|
||||
/// What a pack says about itself.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct PackIdentity {
|
||||
pub epoch_hex: String,
|
||||
pub day_hex: String,
|
||||
pub attempt: u32,
|
||||
pub seedw: [u32; 8],
|
||||
pub keyw: [u32; 8],
|
||||
}
|
||||
|
||||
/// Why a pack is not the one a worker should mine with. `Display` is the plain-words line the logs carry.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum PackFault {
|
||||
/// program.h or seeds.txt is missing or does not parse.
|
||||
Unreadable(String),
|
||||
/// A well-formed pack for other seeds than the node's: the pack is stale (or the node moved on).
|
||||
OutOfDate { pack_epoch: String, pack_day: String, want_epoch: String, want_day: String },
|
||||
/// The files of one pack contradict each other (seeds.txt against program.h, or the init words against the
|
||||
/// seeds and the attempt): a half-written or hand-edited pack, or a worker and an exporter on different rules.
|
||||
Disagree(String),
|
||||
}
|
||||
|
||||
impl fmt::Display for PackFault {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
PackFault::Unreadable(w) => write!(f, "program pack unreadable: {w}"),
|
||||
PackFault::OutOfDate { pack_epoch, pack_day, want_epoch, want_day } => write!(
|
||||
f,
|
||||
"program pack out of date: the pack is for epoch {} day {}, the node is on epoch {} day {}",
|
||||
short(pack_epoch),
|
||||
day_label(pack_day),
|
||||
short(want_epoch),
|
||||
day_label(want_day)
|
||||
),
|
||||
PackFault::Disagree(w) => write!(f, "program pack and its seeds disagree: {w}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for PackFault {}
|
||||
|
||||
fn short(hex: &str) -> &str {
|
||||
if hex.len() >= 16 {
|
||||
&hex[..16]
|
||||
} else {
|
||||
hex
|
||||
}
|
||||
}
|
||||
|
||||
/// The day bytes are `igneum-day/` followed by the little-endian day index (`bind::day_bytes`); print the index
|
||||
/// when the hex has that shape, else the hex.
|
||||
fn day_label(hex: &str) -> String {
|
||||
const PREFIX: &str = "69676e65756d2d6461792f"; // "igneum-day/"
|
||||
if let Some(rest) = hex.strip_prefix(PREFIX) {
|
||||
if let Some(bytes) = unhex(rest) {
|
||||
let mut v = 0u64;
|
||||
for (i, b) in bytes.iter().enumerate().take(8) {
|
||||
v |= (*b as u64) << (8 * i);
|
||||
}
|
||||
return v.to_string();
|
||||
}
|
||||
}
|
||||
hex.to_string()
|
||||
}
|
||||
|
||||
pub fn hex(bytes: &[u8]) -> String {
|
||||
bytes.iter().map(|b| format!("{b:02x}")).collect()
|
||||
}
|
||||
|
||||
fn unhex(s: &str) -> Option<Vec<u8>> {
|
||||
if s.len() % 2 != 0 {
|
||||
return None;
|
||||
}
|
||||
(0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).ok()).collect()
|
||||
}
|
||||
|
||||
/// `#define NAME <rest of line>` in a header; the value as text, trimmed, with a trailing `//` comment removed.
|
||||
fn define(text: &str, name: &str) -> Option<String> {
|
||||
for line in text.lines() {
|
||||
let t = line.trim_start();
|
||||
let Some(rest) = t.strip_prefix("#define ") else { continue };
|
||||
let rest = rest.trim_start();
|
||||
let Some(after) = rest.strip_prefix(name) else { continue };
|
||||
if !after.starts_with(|c: char| c.is_whitespace()) {
|
||||
continue;
|
||||
}
|
||||
let v = after.trim();
|
||||
let v = v.split("//").next().unwrap_or("").trim();
|
||||
return Some(v.to_string());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn define_str(text: &str, name: &str) -> Option<String> {
|
||||
let v = define(text, name)?;
|
||||
let v = v.strip_prefix('"')?.strip_suffix('"')?;
|
||||
Some(v.to_string())
|
||||
}
|
||||
|
||||
fn define_u32(text: &str, name: &str) -> Option<u32> {
|
||||
let v = define(text, name)?;
|
||||
let v = v.trim_end_matches('u');
|
||||
if let Some(h) = v.strip_prefix("0x") {
|
||||
u32::from_str_radix(h, 16).ok()
|
||||
} else {
|
||||
v.parse().ok()
|
||||
}
|
||||
}
|
||||
|
||||
fn define_words(text: &str, name: &str) -> Option<[u32; 8]> {
|
||||
let v = define(text, name)?;
|
||||
let inner = v.trim().strip_prefix('{')?.strip_suffix('}')?;
|
||||
let mut out = [0u32; 8];
|
||||
let mut n = 0;
|
||||
for part in inner.split(',') {
|
||||
let p = part.trim().trim_end_matches('u');
|
||||
if p.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if n >= 8 {
|
||||
return None;
|
||||
}
|
||||
out[n] = if let Some(h) = p.strip_prefix("0x") { u32::from_str_radix(h, 16).ok()? } else { p.parse().ok()? };
|
||||
n += 1;
|
||||
}
|
||||
(n == 8).then_some(out)
|
||||
}
|
||||
|
||||
/// One `key value` line of seeds.txt.
|
||||
fn seeds_line(text: &str, key: &str) -> Option<String> {
|
||||
text.lines().find_map(|l| l.strip_prefix(key).and_then(|r| r.strip_prefix(' ')).map(|v| v.trim().to_string()))
|
||||
}
|
||||
|
||||
/// Checks the texts of a pack (program.h, and seeds.txt when it exists) against the seeds a worker will be asked
|
||||
/// to mine with. Pure: the miner and the tests call it with file contents.
|
||||
pub fn verify_pack_texts(program_h: &str, seeds_txt: Option<&str>, want_epoch: &[u8], want_day: &[u8]) -> Result<PackIdentity, PackFault> {
|
||||
let seedw = define_words(program_h, "IGNEUM_SEEDW_INIT").ok_or_else(|| PackFault::Unreadable("program.h has no IGNEUM_SEEDW_INIT with 8 words".into()))?;
|
||||
let keyw = define_words(program_h, "IGNEUM_KEY_INIT").ok_or_else(|| PackFault::Unreadable("program.h has no IGNEUM_KEY_INIT with 8 words".into()))?;
|
||||
let attempt = define_u32(program_h, "IGNEUM_PROGRAM_ATTEMPT").unwrap_or(0);
|
||||
let mut epoch_hex = define_str(program_h, "IGNEUM_SEED_BYTES_HEX").unwrap_or_default();
|
||||
let mut day_hex = define_str(program_h, "IGNEUM_DAY_BYTES_HEX").unwrap_or_default();
|
||||
if let Some(s) = seeds_txt {
|
||||
let e = seeds_line(s, "epoch_seed_hex").ok_or_else(|| PackFault::Unreadable("seeds.txt has no epoch_seed_hex line".into()))?;
|
||||
let d = seeds_line(s, "day_seed_hex").ok_or_else(|| PackFault::Unreadable("seeds.txt has no day_seed_hex line".into()))?;
|
||||
if !epoch_hex.is_empty() && !epoch_hex.eq_ignore_ascii_case(&e) {
|
||||
return Err(PackFault::Disagree(format!("seeds.txt names epoch {} but program.h was generated for epoch {} (a pack half rewritten?)", short(&e), short(&epoch_hex))));
|
||||
}
|
||||
if !day_hex.is_empty() && !day_hex.eq_ignore_ascii_case(&d) {
|
||||
return Err(PackFault::Disagree(format!("seeds.txt names day {} but program.h was generated for day {}", day_label(&d), day_label(&day_hex))));
|
||||
}
|
||||
epoch_hex = e.to_ascii_lowercase();
|
||||
day_hex = d.to_ascii_lowercase();
|
||||
}
|
||||
if epoch_hex.is_empty() || day_hex.is_empty() {
|
||||
return Err(PackFault::Unreadable("no seeds: neither seeds.txt nor IGNEUM_SEED_BYTES_HEX / IGNEUM_DAY_BYTES_HEX in program.h".into()));
|
||||
}
|
||||
let epoch_bytes = unhex(&epoch_hex).filter(|b| b.len() == 32).ok_or_else(|| PackFault::Unreadable("the epoch seed is not 32 bytes of hex".into()))?;
|
||||
let day_bytes = unhex(&day_hex).ok_or_else(|| PackFault::Unreadable("the day seed hex is malformed".into()))?;
|
||||
// The pack's own consistency first: a pack that contradicts itself is never "out of date", it is broken
|
||||
let want_w = attempt_words(&epoch_bytes, attempt);
|
||||
if want_w != seedw {
|
||||
return Err(PackFault::Disagree(format!(
|
||||
"IGNEUM_SEEDW_INIT is not attempt {attempt} of the epoch seed {} (the words of attempt {attempt} are {:08x} {:08x} ..., the pack has {:08x} {:08x} ...)",
|
||||
short(&epoch_hex),
|
||||
want_w[0],
|
||||
want_w[1],
|
||||
seedw[0],
|
||||
seedw[1]
|
||||
)));
|
||||
}
|
||||
let want_k = seed_words_from_bytes(&day_bytes);
|
||||
if want_k != keyw {
|
||||
return Err(PackFault::Disagree(format!("IGNEUM_KEY_INIT is not the key of the day seed {} ", day_label(&day_hex))));
|
||||
}
|
||||
let want_epoch_hex = hex(want_epoch);
|
||||
let want_day_hex = hex(want_day);
|
||||
if epoch_hex != want_epoch_hex || day_hex != want_day_hex {
|
||||
return Err(PackFault::OutOfDate { pack_epoch: epoch_hex, pack_day: day_hex, want_epoch: want_epoch_hex, want_day: want_day_hex });
|
||||
}
|
||||
Ok(PackIdentity { epoch_hex, day_hex, attempt, seedw, keyw })
|
||||
}
|
||||
|
||||
/// [`verify_pack_texts`] over a pack directory.
|
||||
pub fn verify_pack_dir(dir: &Path, want_epoch: &[u8], want_day: &[u8]) -> Result<PackIdentity, PackFault> {
|
||||
let program_h = std::fs::read_to_string(dir.join("program.h")).map_err(|e| PackFault::Unreadable(format!("cannot read {}/program.h: {e}", dir.display())))?;
|
||||
let seeds = std::fs::read_to_string(dir.join("seeds.txt")).ok();
|
||||
verify_pack_texts(&program_h, seeds.as_deref(), want_epoch, want_day)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::emit::program_header;
|
||||
use crate::generator::generate_from_seed_bytes;
|
||||
use crate::verify::Epoch;
|
||||
|
||||
// The two live devnet epochs of 5 October 2026 (epoch 33 mined, epoch 34 refused by the one-click workers)
|
||||
const EPOCH_33: &str = "bed7ab62cbece66cf791485336d81d90fa1452ffed28ecd8a7416960ef64164c";
|
||||
const EPOCH_34: &str = "009858237e118f69abc8d096e9b1af21c24539eaecdfd1b896588825660a69ec";
|
||||
// "igneum-day/" || le64(20731)
|
||||
const DAY_20731: &str = "69676e65756d2d6461792ffb50000000000000";
|
||||
|
||||
fn bytes(h: &str) -> Vec<u8> {
|
||||
unhex(h).unwrap()
|
||||
}
|
||||
|
||||
/// The attempt vectors the C side pins too (proto-cuda/nvrtc/emu/packfile-test.c): a change to either
|
||||
/// derivation fails on one side first.
|
||||
#[test]
|
||||
fn attempt_words_vectors_shared_with_the_workers() {
|
||||
let e = bytes(EPOCH_34);
|
||||
assert_eq!(attempt_words(&e, 0), [0x06af2a61, 0x4d67274e, 0x4ebda738, 0xad1dea73, 0x6233cd8c, 0x50371601, 0x39d0b873, 0x6af024a2]);
|
||||
assert_eq!(attempt_words(&e, 1), [0x0dcff56b, 0x6b1beb0d, 0x234dc70c, 0xe4016fa9, 0x72397152, 0xb558aa79, 0x3ffb3299, 0x72b9962e]);
|
||||
// epoch 33's bare words, as the CUDA worker printed them on 5 October ("seed words be5983a6 f750dab7 ...")
|
||||
assert_eq!(attempt_words(&bytes(EPOCH_33), 0)[..2], [0xbe5983a6, 0xf750dab7]);
|
||||
}
|
||||
|
||||
/// The incident: epoch 34's program is a later attempt, epoch 33's is the bare seed. A worker that derives the
|
||||
/// words from the bare seed accepts 33 and refuses 34.
|
||||
#[test]
|
||||
fn epoch_34_program_is_a_later_attempt() {
|
||||
let p34 = generate_from_seed_bytes("epoch 34", &bytes(EPOCH_34));
|
||||
assert!(p34.attempt >= 1, "epoch 34 must be a retried program for the incident to reproduce; attempt {}", p34.attempt);
|
||||
assert_eq!(p34.seed, attempt_words(&bytes(EPOCH_34), p34.attempt));
|
||||
assert_ne!(p34.seed, attempt_words(&bytes(EPOCH_34), 0));
|
||||
let p33 = generate_from_seed_bytes("epoch 33", &bytes(EPOCH_33));
|
||||
assert_eq!(p33.attempt, 0);
|
||||
}
|
||||
|
||||
fn pack_texts(epoch_hex: &str, day_hex: &str) -> (String, String, u32) {
|
||||
let (e, d) = (bytes(epoch_hex), bytes(day_hex));
|
||||
let epoch = Epoch::from_seed_bytes(&e, &d, "test");
|
||||
let h = program_header(&epoch.program, "test day", &epoch.dataset);
|
||||
let s = format!("epoch_seed_hex {epoch_hex}\nday_seed_hex {day_hex}\nday_index 20731\n");
|
||||
(h, s, epoch.program.attempt)
|
||||
}
|
||||
|
||||
/// Known-good: the pack of a retried program verifies against its own seeds, with its attempt.
|
||||
#[test]
|
||||
fn known_good_pack_of_a_later_attempt_verifies() {
|
||||
let (h, s, attempt) = pack_texts(EPOCH_34, DAY_20731);
|
||||
assert!(attempt >= 1);
|
||||
let id = verify_pack_texts(&h, Some(&s), &bytes(EPOCH_34), &bytes(DAY_20731)).expect("the pack verifies");
|
||||
assert_eq!(id.attempt, attempt);
|
||||
assert_eq!(id.epoch_hex, EPOCH_34);
|
||||
assert_eq!(id.seedw, attempt_words(&bytes(EPOCH_34), attempt));
|
||||
// without seeds.txt program.h's own bytes carry the pack
|
||||
assert!(verify_pack_texts(&h, None, &bytes(EPOCH_34), &bytes(DAY_20731)).is_ok());
|
||||
}
|
||||
|
||||
/// Known-mismatched: a well-formed pack for the previous epoch is "out of date" against the new one, in plain
|
||||
/// words with both epochs named.
|
||||
#[test]
|
||||
fn known_mismatched_pack_is_out_of_date() {
|
||||
let (h, s, _) = pack_texts(EPOCH_33, DAY_20731);
|
||||
let err = verify_pack_texts(&h, Some(&s), &bytes(EPOCH_34), &bytes(DAY_20731)).unwrap_err();
|
||||
assert!(matches!(err, PackFault::OutOfDate { .. }), "{err}");
|
||||
assert_eq!(err.to_string(), "program pack out of date: the pack is for epoch bed7ab62cbece66c day 20731, the node is on epoch 009858237e118f69 day 20731");
|
||||
}
|
||||
|
||||
/// A pack that contradicts itself is "disagree", never "out of date": seeds.txt of one epoch with program.h of
|
||||
/// another (a half rewritten directory), or init words that are not the attempt's words (a worker on the old
|
||||
/// rule would have produced this verdict for every retried program).
|
||||
#[test]
|
||||
fn inconsistent_pack_disagrees() {
|
||||
let (h33, _, _) = pack_texts(EPOCH_33, DAY_20731);
|
||||
let s34 = format!("epoch_seed_hex {EPOCH_34}\nday_seed_hex {DAY_20731}\n");
|
||||
let err = verify_pack_texts(&h33, Some(&s34), &bytes(EPOCH_34), &bytes(DAY_20731)).unwrap_err();
|
||||
assert!(matches!(err, PackFault::Disagree(_)), "{err}");
|
||||
assert!(err.to_string().starts_with("program pack and its seeds disagree: seeds.txt names epoch 009858237e118f69"), "{err}");
|
||||
|
||||
let (h34, s, _) = pack_texts(EPOCH_34, DAY_20731);
|
||||
let bare = attempt_words(&bytes(EPOCH_34), 0);
|
||||
let edited = h34.lines().map(|l| if l.starts_with("#define IGNEUM_SEEDW_INIT") { format!("#define IGNEUM_SEEDW_INIT {{ {} }}", bare.iter().map(|w| format!("0x{w:08x}")).collect::<Vec<_>>().join(", ")) } else { l.to_string() }).collect::<Vec<_>>().join("\n");
|
||||
let err = verify_pack_texts(&edited, Some(&s), &bytes(EPOCH_34), &bytes(DAY_20731)).unwrap_err();
|
||||
assert!(err.to_string().contains("IGNEUM_SEEDW_INIT is not attempt"), "{err}");
|
||||
// and a pack with no attempt line at all is read as attempt 0 (the packs before generator version 2)
|
||||
let no_attempt = h34.lines().filter(|l| !l.starts_with("#define IGNEUM_PROGRAM_ATTEMPT")).collect::<Vec<_>>().join("\n");
|
||||
assert!(verify_pack_texts(&no_attempt, Some(&s), &bytes(EPOCH_34), &bytes(DAY_20731)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn day_label_reads_the_index() {
|
||||
assert_eq!(day_label(DAY_20731), "20731");
|
||||
assert_eq!(day_label("abcd"), "abcd");
|
||||
}
|
||||
}
|
||||
115
proto-cuda/nvrtc/emu/packfile-test.c
Normal file
115
proto-cuda/nvrtc/emu/packfile-test.c
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
// packfile-test.c: the pack loader's seed rule (packfile.h pf_load) on a known-good and a known-mismatched pack.
|
||||
// C99, no GPU, no NVRTC: `emu/packfile-test.sh` compiles and runs it on the Mac in a second, and CI runs it too.
|
||||
//
|
||||
// 5 October 2026, epoch 34 on both PCs: pf_load derived the expected IGNEUM_SEEDW_INIT from the bare epoch seed, but
|
||||
// a pack carries the words of its program's ATTEMPT (igneum-pow attempt_words: seed || k_le32 for k > 0), so every
|
||||
// pack of a retried program was refused and the workers restarted for an hour. These cases pin the rule:
|
||||
// 1. the attempt vectors of epoch 34's seed, the same constants igneum-pow/src/packcheck.rs pins (one vector, two
|
||||
// implementations: a change on either side fails here or there first);
|
||||
// 2. known-good: a pack of attempt 1 with seeds.txt loads, and so does the checked-in attempt-0 pack;
|
||||
// 3. known-mismatched: the same pack with the bare seed's words under IGNEUM_PROGRAM_ATTEMPT 1 (what the old
|
||||
// rule expected) is refused in plain words; a seeds.txt of another epoch is refused as a disagreement.
|
||||
#include "../packfile.h"
|
||||
#include <sys/stat.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define EPOCH_34 "009858237e118f69abc8d096e9b1af21c24539eaecdfd1b896588825660a69ec"
|
||||
#define EPOCH_33 "bed7ab62cbece66cf791485336d81d90fa1452ffed28ecd8a7416960ef64164c"
|
||||
#define DAY_20731 "69676e65756d2d6461792ffb50000000000000"
|
||||
|
||||
static int failures = 0;
|
||||
#define CHECK(cond, what) do { if (cond) printf("ok %s\n", what); else { printf("FAIL %s (%s:%d)\n", what, __FILE__, __LINE__); failures++; } } while (0)
|
||||
|
||||
static void write_file(const char* dir, const char* name, const char* text) {
|
||||
char path[1024];
|
||||
FILE* f;
|
||||
snprintf(path, sizeof(path), "%s/%s", dir, name);
|
||||
f = fopen(path, "wb");
|
||||
if (!f) { perror(path); exit(2); }
|
||||
fputs(text, f);
|
||||
fclose(f);
|
||||
}
|
||||
|
||||
static void words_hex(const uint32_t w[8], char* out) {
|
||||
int i;
|
||||
out[0] = 0;
|
||||
for (i = 0; i < 8; ++i) sprintf(out + strlen(out), "%s0x%08x", i ? ", " : "", w[i]);
|
||||
}
|
||||
|
||||
// A program.h with only what pf_load reads, for the given seeds, attempt and init words.
|
||||
static void write_program_h(const char* dir, const char* epochHex, const char* dayHex, uint32_t attempt, const uint32_t seedw[8], const uint32_t keyw[8]) {
|
||||
char sw[200], kw[200], text[2000];
|
||||
words_hex(seedw, sw); words_hex(keyw, kw);
|
||||
snprintf(text, sizeof(text),
|
||||
"// test pack\n#pragma once\n#define IGNEUM_SEED_STRING \"test\"\n#define IGNEUM_SEED_BYTES_HEX \"%s\"\n#define IGNEUM_GENERATOR 2\n"
|
||||
"#define IGNEUM_PROGRAM_ATTEMPT %u\n#define IGNEUM_DAY_BYTES_HEX \"%s\"\n#define IGNEUM_DATASET_LOG2 28\n#define IGNEUM_DATASET_MODE 1\n"
|
||||
"#define IGNEUM_SEEDW_INIT { %s }\n#define IGNEUM_KEY_INIT { %s }\n#define IGNEUM_CACHE_LOG2_WORDS 26\n#define IGNEUM_CACHE_SEGMENTS 4096u\n",
|
||||
epochHex, (unsigned)attempt, dayHex, sw, kw);
|
||||
write_file(dir, "program.h", text);
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
const char* checked_in = argc > 1 ? argv[1] : NULL; // proto-cuda/packs/igneum-devnet-v4-epoch0 (attempt 0, no seeds.txt)
|
||||
char dir[512];
|
||||
uint8_t e34[32], day[64];
|
||||
size_t n = 0, dn = 0;
|
||||
uint32_t bare[8], att1[8], keyw[8];
|
||||
static const uint32_t want_bare[8] = { 0x06af2a61, 0x4d67274e, 0x4ebda738, 0xad1dea73, 0x6233cd8c, 0x50371601, 0x39d0b873, 0x6af024a2 };
|
||||
static const uint32_t want_att1[8] = { 0x0dcff56b, 0x6b1beb0d, 0x234dc70c, 0xe4016fa9, 0x72397152, 0xb558aa79, 0x3ffb3299, 0x72b9962e };
|
||||
PfPack pk;
|
||||
char err[512];
|
||||
|
||||
pf_unhex(EPOCH_34, e34, 32, &n);
|
||||
pf_unhex(DAY_20731, day, sizeof(day), &dn);
|
||||
CHECK(n == 32 && dn == 19, "the fixture seeds unhex (32 epoch bytes, 19 day bytes)");
|
||||
|
||||
// 1. the attempt vectors shared with igneum-pow
|
||||
pf_program_words(e34, 32, 0, bare);
|
||||
pf_program_words(e34, 32, 1, att1);
|
||||
pf_seed_words_from_bytes(day, dn, keyw);
|
||||
CHECK(memcmp(bare, want_bare, 32) == 0, "attempt 0 of epoch 34 = the bare seed words (06af2a61 4d67274e ...)");
|
||||
CHECK(memcmp(att1, want_att1, 32) == 0, "attempt 1 of epoch 34 = words of seed || 01000000 (0dcff56b 6b1beb0d ...)");
|
||||
CHECK(memcmp(bare, att1, 32) != 0, "the two attempts differ");
|
||||
|
||||
// 2. known-good: a pack of attempt 1 with seeds.txt, as igneum-miner writes it
|
||||
snprintf(dir, sizeof(dir), "%s/igneum-packfile-test-%d", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp", (int)getpid());
|
||||
mkdir(dir, 0755);
|
||||
write_program_h(dir, EPOCH_34, DAY_20731, 1, att1, keyw);
|
||||
write_file(dir, "seeds.txt", "epoch_seed_hex " EPOCH_34 "\nday_seed_hex " DAY_20731 "\nday_index 20731\n");
|
||||
err[0] = 0;
|
||||
CHECK(pf_load(dir, &pk, err, sizeof(err)) == 1, "known-good: the attempt-1 pack loads");
|
||||
if (err[0]) printf(" (%s)\n", err);
|
||||
CHECK(pk.attempt == 1 && memcmp(pk.seedw, att1, 32) == 0 && strcmp(pk.epochHex, EPOCH_34) == 0, "known-good: attempt, words and epoch hex read back");
|
||||
if (checked_in) {
|
||||
err[0] = 0;
|
||||
CHECK(pf_load(checked_in, &pk, err, sizeof(err)) == 1, "known-good: the checked-in attempt-0 pack loads from program.h's own bytes");
|
||||
if (err[0]) printf(" (%s)\n", err);
|
||||
CHECK(pk.attempt == 0, "the checked-in pack is attempt 0");
|
||||
}
|
||||
|
||||
// 3. known-mismatched: the bare words under attempt 1 (the old rule's expectation) are refused, in plain words
|
||||
write_program_h(dir, EPOCH_34, DAY_20731, 1, bare, keyw);
|
||||
err[0] = 0;
|
||||
CHECK(pf_load(dir, &pk, err, sizeof(err)) == 0, "known-mismatched: bare words under attempt 1 are refused");
|
||||
CHECK(strstr(err, "program pack and its seeds disagree: IGNEUM_SEEDW_INIT is not attempt 1 of the epoch seed 009858237e118f69") == err, "the refusal names the attempt and the epoch in plain words");
|
||||
printf(" (%s)\n", err);
|
||||
// and a seeds.txt of another epoch against this program.h is a disagreement, not a load
|
||||
write_program_h(dir, EPOCH_34, DAY_20731, 1, att1, keyw);
|
||||
write_file(dir, "seeds.txt", "epoch_seed_hex " EPOCH_33 "\nday_seed_hex " DAY_20731 "\n");
|
||||
err[0] = 0;
|
||||
CHECK(pf_load(dir, &pk, err, sizeof(err)) == 0, "known-mismatched: seeds.txt of epoch 33 with program.h of epoch 34 is refused");
|
||||
CHECK(strstr(err, "seeds.txt epoch_seed_hex differs from program.h") != NULL, "the refusal says the files disagree");
|
||||
// a pack with no IGNEUM_PROGRAM_ATTEMPT line is attempt 0 (packs before generator version 2)
|
||||
{
|
||||
char text[2000], sw[200], kw[200];
|
||||
words_hex(bare, sw); words_hex(keyw, kw);
|
||||
snprintf(text, sizeof(text), "#define IGNEUM_SEED_BYTES_HEX \"%s\"\n#define IGNEUM_DAY_BYTES_HEX \"%s\"\n#define IGNEUM_DATASET_LOG2 28\n#define IGNEUM_DATASET_MODE 1\n#define IGNEUM_SEEDW_INIT { %s }\n#define IGNEUM_KEY_INIT { %s }\n#define IGNEUM_CACHE_LOG2_WORDS 26\n#define IGNEUM_CACHE_SEGMENTS 4096u\n", EPOCH_34, DAY_20731, sw, kw);
|
||||
write_file(dir, "program.h", text);
|
||||
write_file(dir, "seeds.txt", "epoch_seed_hex " EPOCH_34 "\nday_seed_hex " DAY_20731 "\n");
|
||||
err[0] = 0;
|
||||
CHECK(pf_load(dir, &pk, err, sizeof(err)) == 1 && pk.attempt == 0, "no attempt line reads as attempt 0 and the bare words load");
|
||||
}
|
||||
|
||||
printf("%s: %d failure(s)\n", argv[0], failures);
|
||||
return failures ? 1 : 0;
|
||||
}
|
||||
11
proto-cuda/nvrtc/emu/packfile-test.sh
Executable file
11
proto-cuda/nvrtc/emu/packfile-test.sh
Executable file
|
|
@ -0,0 +1,11 @@
|
|||
#!/usr/bin/env bash
|
||||
# The pack loader's seed rule (packfile.h) on a known-good and a known-mismatched pack: emu/packfile-test.c, C99,
|
||||
# no GPU. Runs on the Mac in a second and in CI. Usage: emu/packfile-test.sh
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$HERE/../../.." && pwd)"
|
||||
OUT="${TMPDIR:-/tmp}/igneum-packfile-test"
|
||||
mkdir -p "$OUT"
|
||||
CC="${CC:-cc}"
|
||||
"$CC" -std=c99 -Wall -Wextra -Wno-unused-function -O1 -o "$OUT/packfile-test" "$HERE/packfile-test.c"
|
||||
"$OUT/packfile-test" "$ROOT/proto-cuda/packs/igneum-devnet-v4-epoch0"
|
||||
|
|
@ -23,6 +23,7 @@
|
|||
typedef struct {
|
||||
// program.h
|
||||
uint32_t datasetLog2, cacheLog2Words, cacheSegments, datasetMode, generator;
|
||||
uint32_t attempt; // IGNEUM_PROGRAM_ATTEMPT: seedw are the words of this attempt of the epoch seed (0 = bare seed)
|
||||
uint32_t seedw[8], keyw[8];
|
||||
char seedString[600];
|
||||
// read-width experiment (5 October 2026): the load class (0 when absent), bytes per hash, variant 5's scratch
|
||||
|
|
@ -211,6 +212,20 @@ static void pf_seed_words_from_bytes(const uint8_t* b, size_t n, uint32_t out[8]
|
|||
}
|
||||
}
|
||||
|
||||
// attempt_words of igneum-pow/src/generator.rs: the words of attempt k of a program seed are
|
||||
// seed_words_from_bytes(seed || k_le32) for k > 0 and the bare seed's words for k = 0. The generator retries a
|
||||
// rejected candidate with the next attempt, so a pack's IGNEUM_SEEDW_INIT is the words of IGNEUM_PROGRAM_ATTEMPT,
|
||||
// not of the bare seed. 5 October 2026: pf_load derived the expected words from the bare seed and refused every
|
||||
// pack of a retried program (epoch 34, both PCs, the miner restarting the worker for an hour); this is the rule.
|
||||
static void pf_program_words(const uint8_t* b, size_t n, uint32_t attempt, uint32_t out[8]) {
|
||||
uint8_t buf[256 + 4];
|
||||
if (attempt == 0) { pf_seed_words_from_bytes(b, n, out); return; }
|
||||
if (n > 256) n = 256;
|
||||
memcpy(buf, b, n);
|
||||
buf[n] = (uint8_t)attempt; buf[n + 1] = (uint8_t)(attempt >> 8); buf[n + 2] = (uint8_t)(attempt >> 16); buf[n + 3] = (uint8_t)(attempt >> 24);
|
||||
pf_seed_words_from_bytes(buf, n + 4, out);
|
||||
}
|
||||
|
||||
static uint64_t pf_fnv1a64(const void* p, size_t n) {
|
||||
const uint8_t* b = (const uint8_t*)p;
|
||||
uint64_t h = 0xcbf29ce484222325ull;
|
||||
|
|
@ -256,6 +271,7 @@ static int pf_load(const char* dir, PfPack* pk, char* err, size_t cap) {
|
|||
if (!pf_define_u32(prog, "IGNEUM_CACHE_LOG2_WORDS", &pk->cacheLog2Words)) { free(prog); return pf_fail(err, cap, "program.h has no IGNEUM_CACHE_LOG2_WORDS"); }
|
||||
if (!pf_define_u32(prog, "IGNEUM_CACHE_SEGMENTS", &pk->cacheSegments)) { free(prog); return pf_fail(err, cap, "program.h has no IGNEUM_CACHE_SEGMENTS"); }
|
||||
if (!pf_define_u32(prog, "IGNEUM_GENERATOR", &pk->generator)) pk->generator = 1;
|
||||
if (!pf_define_u32(prog, "IGNEUM_PROGRAM_ATTEMPT", &pk->attempt)) pk->attempt = 0;
|
||||
if (pf_define_words(prog, "IGNEUM_SEEDW_INIT", pk->seedw, 8) != 8) { free(prog); return pf_fail(err, cap, "program.h has no IGNEUM_SEEDW_INIT with 8 words"); }
|
||||
pk->loadsPerHash = 128; pf_define_u32(prog, "IGNEUM_LOADS_PER_HASH", &pk->loadsPerHash);
|
||||
pk->bytesPerHash = pk->loadsPerHash * 4u; pf_define_u32(prog, "IGNEUM_BYTES_PER_HASH", &pk->bytesPerHash);
|
||||
|
|
@ -289,12 +305,18 @@ static int pf_load(const char* dir, PfPack* pk, char* err, size_t cap) {
|
|||
// protocol still carries 64-hex seeds; a string-seed pack can only be benched (--bench, --bench-pack, --check).
|
||||
if (strlen(ehex) < 2 || strlen(ehex) % 2 != 0) return pf_fail(err, cap, "epoch seed is not an even-length hex string");
|
||||
strcpy(pk->epochHex, ehex); strcpy(pk->dayHex, dhex);
|
||||
// The seed words derived from the bytes must be the pack's own words: otherwise the pack and the seeds disagree
|
||||
// The seed words derived from the bytes AND the attempt must be the pack's own words: otherwise the pack and
|
||||
// its seeds disagree (a half rewritten directory, or an exporter on another rule)
|
||||
{
|
||||
uint32_t w[8];
|
||||
char m[400];
|
||||
if (!pf_unhex(ehex, bytes, sizeof(bytes), &blen) || blen == 0) return pf_fail(err, cap, "epoch seed hex is malformed");
|
||||
pf_seed_words_from_bytes(bytes, blen, w);
|
||||
if (memcmp(w, pk->seedw, 32) != 0) return pf_fail(err, cap, "the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT (wrong seeds.txt for this pack?)");
|
||||
pf_program_words(bytes, blen, pk->attempt, w);
|
||||
if (memcmp(w, pk->seedw, 32) != 0) {
|
||||
snprintf(m, sizeof(m), "program pack and its seeds disagree: IGNEUM_SEEDW_INIT is not attempt %u of the epoch seed %.16s (attempt %u gives %08x %08x ..., the pack has %08x %08x ...); run igneum-miner export-pack again",
|
||||
(unsigned)pk->attempt, ehex, (unsigned)pk->attempt, w[0], w[1], pk->seedw[0], pk->seedw[1]);
|
||||
return pf_fail(err, cap, m);
|
||||
}
|
||||
if (!pf_unhex(dhex, bytes, sizeof(bytes), &blen)) return pf_fail(err, cap, "day seed hex is malformed");
|
||||
pf_seed_words_from_bytes(bytes, blen, w);
|
||||
if (memcmp(w, pk->keyw, 32) != 0) return pf_fail(err, cap, "the day seed bytes do not give the pack's IGNEUM_KEY_INIT (wrong seeds.txt for this pack?)");
|
||||
|
|
|
|||
|
|
@ -398,6 +398,18 @@ static bool rtcCompile(Ctx& c, const std::string& src, const char* name, const s
|
|||
// ---------------------------------------------------------------------------------------------
|
||||
// A resident pair: one pack compiled, its cache and dataset on the device, self-tested
|
||||
|
||||
static bool hexEq(const std::string& a, const std::string& b) {
|
||||
if (a.size() != b.size()) return false;
|
||||
for (size_t i = 0; i < a.size(); ++i) if (std::tolower((unsigned char)a[i]) != std::tolower((unsigned char)b[i])) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
struct Pair;
|
||||
// A pair is the pair of a job when the job's seeds (the hex the node sent) are the pair's seeds. The derived seed
|
||||
// words are no identity: a retried program's words are its attempt's words, not the bare seed's (packfile.h,
|
||||
// 5 October 2026), so comparing words refused every job of a retried program.
|
||||
static bool pairIs(const Pair* p, const std::string& epochHex, const std::string& dayHex);
|
||||
|
||||
struct Pair {
|
||||
std::string dir, epochHex, dayHex, seedString;
|
||||
uint32_t sw[8] = {0}, kw[8] = {0};
|
||||
|
|
@ -423,6 +435,10 @@ struct Pair {
|
|||
size_t scratchBytes = 0;
|
||||
};
|
||||
|
||||
static bool pairIs(const Pair* p, const std::string& epochHex, const std::string& dayHex) {
|
||||
return p && hexEq(p->epochHex, epochHex) && hexEq(p->dayHex, dayHex);
|
||||
}
|
||||
|
||||
// The job loop and a race take turns on the card: a variant is timed with no job running (exclusive numbers), and
|
||||
// mining resumes between variants. Held per chunk by the job loop, per variant by the race.
|
||||
static std::mutex gpuMutex;
|
||||
|
|
@ -1099,8 +1115,7 @@ static int runServe(Ctx& c, const Options& o, Pair* cur) {
|
|||
pf_seed_words_from_bytes(daySeed, dl, kw);
|
||||
double t0 = wallMs();
|
||||
bool switched = false;
|
||||
if ((std::memcmp(sw, cur->sw, 32) != 0 || std::memcmp(kw, cur->kw, 32) != 0) && !task &&
|
||||
!(prepared && std::memcmp(sw, prepared->sw, 32) == 0 && std::memcmp(kw, prepared->kw, 32) == 0)) {
|
||||
if (!pairIs(cur, f[6], f[7]) && !task && !pairIs(prepared, f[6], f[7])) {
|
||||
// Self-heal: a job on seeds this worker has no pair for and no prepare in flight (a prepare failed, or
|
||||
// the miner never sent one). The miner writes a pack per pair under its --prepare-packs root; find it by
|
||||
// seeds.txt and build it now, in the foreground. The miner only re-sends prepare for the pair after this one.
|
||||
|
|
@ -1116,14 +1131,14 @@ static int runServe(Ctx& c, const Options& o, Pair* cur) {
|
|||
else emit("error " + jobId + " could not build " + dir + ": " + berr);
|
||||
}
|
||||
}
|
||||
if (std::memcmp(sw, cur->sw, 32) != 0 || std::memcmp(kw, cur->kw, 32) != 0) {
|
||||
if (prepared && std::memcmp(sw, prepared->sw, 32) == 0 && std::memcmp(kw, prepared->kw, 32) == 0) {
|
||||
if (!pairIs(cur, f[6], f[7])) {
|
||||
if (pairIs(prepared, f[6], f[7])) {
|
||||
if (old) releasePair(c, old);
|
||||
old = cur; cur = prepared; prepared = nullptr; switched = true;
|
||||
info(fmt("switched to the prepared pair epoch %.16s day %s in %.2f ms", cur->epochHex.c_str(), cur->dayHex.c_str(), wallMs() - t0));
|
||||
} else if (std::memcmp(sw, cur->sw, 32) != 0) {
|
||||
} else if (!hexEq(cur->epochHex, f[6])) {
|
||||
emit(fmt("need %s %s", f[6].c_str(), f[7].c_str())); // the miner prepares this pair (4 October 2026)
|
||||
emit(fmt("error %s epoch seed mismatch: this worker holds epoch %.16s (seed words %08x %08x ...)%s, the job's epoch seed %.16s gives %08x %08x ...; send prepare with a pack directory",
|
||||
emit(fmt("error %s epoch seed mismatch: this worker holds epoch %.16s (program words %08x %08x ...)%s, the job is for epoch %.16s (bare seed words %08x %08x ...); send prepare with a pack directory",
|
||||
jobId.c_str(), cur->epochHex.c_str(), cur->sw[0], cur->sw[1], prepared ? " plus one prepared pair" : "", f[6].c_str(), sw[0], sw[1]));
|
||||
continue;
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -1054,6 +1054,23 @@ typedef struct {
|
|||
int checked;
|
||||
} ServePair;
|
||||
|
||||
static int hexEq(const char* a, const char* b) {
|
||||
size_t i;
|
||||
if (strlen(a) != strlen(b)) return 0;
|
||||
for (i = 0; a[i]; ++i) if (tolower((unsigned char)a[i]) != tolower((unsigned char)b[i])) return 0;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* A pair is the pair of a job when the job's seeds (the hex the node sent) are the pair's seeds. The derived seed
|
||||
* words are no identity: a retried program's words are its attempt's words, not the bare seed's (packfile.h,
|
||||
* 5 October 2026), so comparing words refused every job of a retried program. The compiled-in placeholder pack
|
||||
* (no --pack, no prepared pair) has no seed hex; it keeps the word comparison. */
|
||||
static int pairIs(const ServePair* p, const char* epochHex, const char* dayHex, const uint32_t sw[8], const uint32_t kw[8]) {
|
||||
if (!p) return 0;
|
||||
if (p->epochHex[0]) return hexEq(p->epochHex, epochHex) && hexEq(p->dayHex, dayHex);
|
||||
return memcmp(sw, p->sw, 32) == 0 && memcmp(kw, p->kw, 32) == 0;
|
||||
}
|
||||
|
||||
static void releasePair(ServePair* p) {
|
||||
if (!p) return;
|
||||
if (p->ds) { clReleaseMemObject(p->ds); ++gMemReleased; }
|
||||
|
|
@ -1505,15 +1522,15 @@ static int runServe(Device* dv, const DeviceInfo* di, const Options* o) {
|
|||
seedWordsFromBytes(epochSeed, 32, sw);
|
||||
seedWordsFromBytes(daySeed, dayLen, kw);
|
||||
t0 = wallMs();
|
||||
if (memcmp(sw, cur->sw, 32) != 0 || memcmp(kw, cur->kw, 32) != 0) {
|
||||
if (prepared && memcmp(sw, prepared->sw, 32) == 0 && memcmp(kw, prepared->kw, 32) == 0) {
|
||||
if (!pairIs(cur, f[6], f[7], sw, kw)) {
|
||||
if (pairIs(prepared, f[6], f[7], sw, kw)) {
|
||||
/* The prepared pair: switch now, release the old one after this job */
|
||||
if (old) releasePair(old);
|
||||
old = cur; cur = prepared; prepared = NULL; switched = 1;
|
||||
printf("info switched to the prepared pair epoch %.16s day %s in %.2f ms\n", cur->epochHex, cur->dayHex, wallMs() - t0); fflush(stdout);
|
||||
} else if (memcmp(sw, cur->sw, 32) != 0) {
|
||||
} else if (cur->epochHex[0] ? !hexEq(cur->epochHex, f[6]) : memcmp(sw, cur->sw, 32) != 0) {
|
||||
printf("need %s %s\n", f[6], f[7]); /* the miner prepares this pair (4 October 2026) */
|
||||
printf("error %s epoch seed mismatch: this worker holds %s%s (seed words %08x %08x ...)%s, the job's epoch seed %.16s gives %08x %08x ...; send prepare with a pack directory, or run igneum-miner export-pack and rebuild\n",
|
||||
printf("error %s epoch seed mismatch: this worker holds %s%s (program words %08x %08x ...)%s, the job is for epoch %.16s (bare seed words %08x %08x ...); send prepare with a pack directory, or run igneum-miner export-pack and rebuild\n",
|
||||
jobId, cur->epochHex[0] ? "prepared epoch " : "pack \"" IGNEUM_SEED_STRING "\"", cur->epochHex[0] ? cur->epochHex : "", cur->sw[0], cur->sw[1], prepared ? " plus one prepared pair" : "", f[6], sw[0], sw[1]);
|
||||
fflush(stdout); continue;
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ export const kv = (text, k) => { const m = new RegExp(`(?:^|[\\s(])${k}=([^\\s,)
|
|||
export const kvNum = (text, k) => { const v = kv(text, k); return v === null ? null : Number(v); };
|
||||
export const lastMatch = (lines, re) => { for (let i = lines.length - 1; i >= 0; i--) { const m = re.exec(lines[i]); if (m) return m; } return null; };
|
||||
export const FAULT = /WORKER MISMATCH|worker error|worker exited|panicked|CUDA error|submit error/;
|
||||
// A program pack the worker refused at start or the miner found stale (5 October 2026, epoch 34 on both PCs: the
|
||||
// bare restart count hid an hour-long loop). The card says "pack mismatch, rebuilding" instead of a bare count.
|
||||
export const PACK_MISMATCH = /error 0 pack |PACK OUT OF DATE|program pack out of date|program pack and its seeds disagree/;
|
||||
|
||||
// A card's hash is "now" only while its worker's STATUS line (every 30 s, uploaded every 60 s) is this fresh; older
|
||||
// than this the card is marked stale and its hash leaves the machine total (4 October 2026: PC 2 showed 117 MH/s "now"
|
||||
|
|
@ -59,6 +62,9 @@ export function parseMinerTail(tail) {
|
|||
const faults = lines.filter(l => FAULT.test(l));
|
||||
c.faults = faults.length;
|
||||
if (faults.length) c.fault = faults[faults.length - 1].replace(/^\d+(\.\d+)? /, '').slice(0, 200);
|
||||
const packs = lines.filter(l => PACK_MISMATCH.test(l));
|
||||
c.pack_mismatch = packs.length > 0;
|
||||
if (packs.length) c.fault = `pack mismatch, rebuilding (${packs.length} refusal${packs.length === 1 ? '' : 's'} in the tail${c.restarts ? ', ' + c.restarts + ' restarts' : ''})`;
|
||||
const acc = lastMatch(lines, /^(\d+(?:\.\d+)?) ACCEPTED block/);
|
||||
if (acc) c.last_accepted_at = new Date(Number(acc[1]) * 1000).toISOString();
|
||||
return c;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
// node --test relay/test/parse.test.mjs relay/test/auth.test.mjs (no dependencies; CI runs both in the site job)
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { parseLabel, parseMinerTail, markStale, STALE_S } from '../lib/parse.mjs';
|
||||
import { parseLabel, parseMinerTail, markStale, STALE_S, PACK_MISMATCH } from '../lib/parse.mjs';
|
||||
|
||||
test('labels: every vendor the app names, the app log, the node log, the legacy launchers', () => {
|
||||
assert.deepEqual(parseLabel('miner-nvidia-ae432dc7-1'), { id: 'ae432dc7', platform: 'win', stream: 'miner', vendor: 'nvidia', card: 1 });
|
||||
|
|
@ -84,3 +84,32 @@ test('app tail: the newest update line decides the OTA state: staged beats an ol
|
|||
const inst = parseAppTail(mac + '\n1791146700 [info] installing Igneum Miner 0.3.4: the miners stop, then the node, then the app opens again');
|
||||
assert.deepEqual([inst.ota.state, inst.ota.version], ['installing', '0.3.4']);
|
||||
});
|
||||
|
||||
test('pack mismatch: the refused-pack tail of 5 October 2026 reads "pack mismatch, rebuilding", a healthy tail does not', () => {
|
||||
// PC 2, nvidia-1ccfe586-1, 18:27 to 18:28 UTC: the worker refused packs\devnet at every start and the miner restarted it
|
||||
const refused = [
|
||||
"1791224818.418 epoch seed 009858237e118f69abc8d096e9b1af21c24539eaecdfd1b896588825660a69ec day 20731 (daa 122513): CPU program and cache ready in 179 ms",
|
||||
"! 1791224818.419 worker error: error 0 pack packs\\devnet: the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT (wrong seeds.txt for this pack?)",
|
||||
'! 1791224818.933 worker exited (code Some(1)); restarting it in 21 s (restart 1)',
|
||||
"1791224840.037 STATUS 'nvidia-1ccfe586-1' [worker]: 22s jobs=0 accepted=0 rejected=0 fee=0 mismatched=0 extra=0 rate=0.00 blocks/s hash=0.00 MH/s wall (0.00 MH/s inside jobs) now=0.00 MH/s wall (0.00 MH/s inside jobs, 0 jobs, seed walk 0 calls) template_age=0.45s synced=true idle=100.0% (last 22s: 100.0%) queued=2 restarts=1 faults=0 identities=8 accepted_by_identity=0/0/0/0/0/0/0/0",
|
||||
"! 1791224840.037 worker error: error 0 pack packs\\devnet: the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT (wrong seeds.txt for this pack?)",
|
||||
'! 1791224840.548 worker exited (code Some(1)); restarting it in 41 s (restart 2)',
|
||||
].join('\n');
|
||||
const c = parseMinerTail(refused);
|
||||
assert.equal(c.pack_mismatch, true);
|
||||
assert.equal(c.fault, 'pack mismatch, rebuilding (2 refusals in the tail, 1 restarts)');
|
||||
assert.equal(c.restarts, 1);
|
||||
// the miner's own line (0.3.11 miners) counts too
|
||||
const own = parseMinerTail('1791224840.100 PACK OUT OF DATE packs\\devnet: the worker refused its program pack; rebuilding the program pack before the restart');
|
||||
assert.equal(own.pack_mismatch, true);
|
||||
assert.ok(own.fault.startsWith('pack mismatch, rebuilding'));
|
||||
// known-good: a mining tail with a job error that is not a refusal keeps the plain fault text
|
||||
const healthy = [
|
||||
"1791140579.479 STATUS 'other-37ba0461-1' [worker]: 61s jobs=37 accepted=2 rejected=0 mismatched=0 extra=0 rate=0.02 blocks/s hash=1.28 MH/s wall (1.55 MH/s inside jobs) now=1.53 MH/s wall",
|
||||
'! 1791140580.000 worker error: error 17 epoch seed mismatch: this worker holds epoch bed7ab62cbece66c',
|
||||
].join('\n');
|
||||
const h = parseMinerTail(healthy);
|
||||
assert.equal(h.pack_mismatch, false);
|
||||
assert.ok(h.fault.includes('worker error: error 17'));
|
||||
assert.equal(PACK_MISMATCH.test('1 ACCEPTED block x'), false);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue