diff --git a/app/igneum-app/src/engine.rs b/app/igneum-app/src/engine.rs index 63318b4b..3ad329d0 100644 --- a/app/igneum-app/src/engine.rs +++ b/app/igneum-app/src/engine.rs @@ -430,6 +430,8 @@ struct MinerSlot { error_at: Option, /// the per-card watchdog (src/watchdog.rs): one restart, then faulted watch: crate::watchdog::CardWatch, + /// pack refusals (src/watchdog.rs): the pack is exported again before the restart, capped per epoch + pack_rebuilds: crate::watchdog::PackRebuilds, } pub struct Engine { @@ -1337,6 +1339,7 @@ impl Engine { last_status: None, error_at: None, watch: crate::watchdog::CardWatch::new(), + pack_rebuilds: crate::watchdog::PackRebuilds::new(), }); } if self.miners.is_empty() { @@ -2460,6 +2463,7 @@ impl Engine { if !p.alive() { let code = p.exit_code.unwrap_or(-1); let tail = tail_of(&p.log_path, 3); + let long_tail = if code == crate::watchdog::PACK_OUT_OF_DATE_CODE { tail_of(&p.log_path, 40) } else { Vec::new() }; self.miners[i].proc = None; let t = self.secs(now); let verdict = self.miners[i].watch.event(t, crate::watchdog::Event::Exited(code)); @@ -2469,6 +2473,37 @@ impl Engine { self.miners[i].needs_rebuild = true; } self.miners[i].restart_at = Some(now); + } else if code == crate::watchdog::PACK_OUT_OF_DATE_CODE { + // The worker refused its program pack and the miner could not rebuild it (or hit its own + // cap): export the pack from the node again before the next start, not a blind restart; + // at most PACK_REBUILD_CAP times per epoch, then the card shows the reason + let why = long_tail.iter().rev().find_map(|l| crate::watchdog::pack_refusal(l)).unwrap_or_else(|| "the worker refused its program pack".into()); + let epoch = std::fs::read_to_string(self.shared.runtime.app_dir.join("packs").join("devnet").join("seeds.txt")).ok().and_then(|s| crate::watchdog::pack_epoch_of(&s)).unwrap_or_default(); + let name = self.st().mining.cards.get(card_idx).map(|c| c.name.clone()).unwrap_or_else(|| self.miners[i].label.clone()); + match self.miners[i].pack_rebuilds.decide(&epoch, &why) { + crate::watchdog::PackAction::Rebuild { n, cap } => { + self.shared.event("build", "program pack out of date, rebuilding"); + self.shared.log(&format!("{name}: program pack out of date, rebuilding (export {n} of {cap} for epoch {epoch}): {why}")); + self.miners[i].prepared = false; // prepare_worker exports the pack again before the start + self.miners[i].restart_at = Some(now); + if let Some(c) = self.st().mining.cards.get_mut(card_idx) { + c.state = "restarting".into(); + c.hash_now = 0.0; + c.message = "program pack out of date, rebuilding".into(); + } + } + crate::watchdog::PackAction::GiveUp { n: _, reason } => { + self.shared.event("error", &format!("{name}: {reason}")); + self.shared.log(&format!("{name}: {reason}; next try in 10 minutes or at the next hour")); + self.miners[i].prepared = false; + self.miners[i].restart_at = Some(now + Duration::from_secs(600)); + if let Some(c) = self.st().mining.cards.get_mut(card_idx) { + c.state = "failed".into(); + c.hash_now = 0.0; + c.message = reason; + } + } + } } else if verdict != crate::watchdog::Action::None { // exit 43: the miner gave up on its worker; once more, then the card is faulted self.watchdog_verdict(i, verdict, &tail); @@ -2546,6 +2581,25 @@ impl Engine { } } + /// The strip, the log and the card for a program pack the worker refused or the miner found stale: the miner + /// rebuilds the pack and restarts the worker itself (or exits 44 for the app to export it). One strip line per + /// 30 s: the miner prints the refusal on stderr and its own line on stdout. + fn pack_notice(&mut self, i: usize, card: usize, card_name: &str, why: &str) { + let now = Instant::now(); + self.shared.log(&format!("{card_name}: program pack out of date, rebuilding: {why}")); + if now.duration_since(self.last_error_event) >= Duration::from_secs(30) { + self.last_error_event = now; + self.shared.event("build", "program pack out of date, rebuilding"); + } + self.miners[i].error_at = Some(now); + let t = self.secs(now); + self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart("program pack out of date, rebuilding")); + if let Some(c) = self.st().mining.cards.get_mut(card) { + c.hash_now = 0.0; + c.message = "program pack out of date, rebuilding".into(); + } + } + /// Applies a watchdog verdict to slot `i` (its process already stopped or gone): a restart now with the reason on /// the card, or the card marked faulted with the reason in the UI and the log while the other cards keep mining. fn watchdog_verdict(&mut self, i: usize, verdict: crate::watchdog::Action, tail: &[String]) { @@ -2867,6 +2921,10 @@ impl Engine { if let Some(c) = self.st().mining.cards.get_mut(card) { c.message = "the node is not answering; the miner retries".into(); } + } else if let Some(why) = crate::watchdog::pack_refusal(text) { + // the worker refused its program pack; the miner rebuilds the pack and restarts the worker itself + // (or exits 44 for us to export it): the strip and the card name the condition in plain words + self.pack_notice(i, card, &card_name, &why); } else if text.contains("WORKER MISMATCH") || text.contains("worker error") || text.contains("worker exited") || text.contains("worker killed by a guard") || text.contains("panicked") || text.contains("CUDA error") || text.contains("submit error") { let now = Instant::now(); if now.duration_since(self.last_error_event) >= Duration::from_secs(30) { @@ -2889,6 +2947,12 @@ impl Engine { } return; } + if text.contains("PACK OUT OF DATE") { + // the miner found its pack stale or refused (stdout): it rebuilds the pack before the restart + let why = crate::watchdog::pack_refusal(text).unwrap_or_else(|| short(text, 160)); + self.pack_notice(i, card, &card_name, &why); + return; + } if text.contains(" WORKER FAULT ") { // the miner's guards (interval, job time, cpu re-check, stall) killed the worker; it restarts it itself let reason = crate::watchdog::fault_reason(text).unwrap_or_else(|| short(text, 160)); @@ -3398,9 +3462,17 @@ fn civil_from_days(z: i64) -> (i64, u32, u32) { (if m <= 2 { y + 1 } else { y }, m, d) } +/// One pack export at a time (5 October 2026). prepare_worker runs on a thread per card, so two cards starting +/// together ran two `igneum-miner export-pack` processes into the same folder; across an epoch change they +/// interleaved and PC 1's packs\devnet was left with one epoch's program.h and the other's seeds.txt, which every +/// OpenCL worker start then refused ("the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT") until the next +/// export. The second export of a pair rewrites the same pack, which is harmless. +static EXPORT_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + /// Exports this hour's program pack from the node to \packs\devnet (the prebuilt workers read it with --pack). #[allow(unused_variables)] fn export_pack(shared: &Arc, bins: &Bins) -> Result<(), String> { + let _one_at_a_time = EXPORT_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let pack = shared.runtime.app_dir.join("packs").join("devnet"); let _ = std::fs::create_dir_all(&pack); let out = crate::detect::run_timeout(std::process::Command::new(&bins.miner).args(["export-pack", &shared.runtime.rpc_url(), &pack.display().to_string()]), None, Duration::from_secs(120)).unwrap_or_default(); @@ -3419,6 +3491,7 @@ fn build_worker_from_source(shared: &Arc, bins: &Bins, vendor: &str) -> #[cfg(windows)] { use std::process::Command; + let _one_at_a_time = EXPORT_LOCK.lock().unwrap_or_else(|e| e.into_inner()); let pack = shared.runtime.app_dir.join("packs").join("devnet"); let _ = std::fs::create_dir_all(&pack); let out = crate::detect::run_timeout(Command::new(&bins.miner).args(["export-pack", &shared.runtime.rpc_url(), &pack.display().to_string()]), None, Duration::from_secs(120)).unwrap_or_default();