igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

engine: one pack export at a time (EXPORT_LOCK around both export-pack call sites; the hunk of opencl-rdna4 23810df, whose other changes conflict with gpu-hotplug and wait for the next cut)

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-05 19:23:59 +00:00
parent d5da02a851
commit a0864f0842
1 changed files with 73 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
app/igneum-app/src/engine.rs
View file

@ -430,6 +430,8 @@ struct MinerSlot {
error_at: Option<Instant>,
/// the per-card watchdog (src/watchdog.rs): one restart, then faulted
watch: crate::watchdog::CardWatch,
/// pack refusals (src/watchdog.rs): the pack is exported again before the restart, capped per epoch
pack_rebuilds: crate::watchdog::PackRebuilds,
}
pub struct Engine {
@ -1337,6 +1339,7 @@ impl Engine {
last_status: None,
error_at: None,
watch: crate::watchdog::CardWatch::new(),
pack_rebuilds: crate::watchdog::PackRebuilds::new(),
});
}
if self.miners.is_empty() {
@ -2460,6 +2463,7 @@ impl Engine {
if !p.alive() {
let code = p.exit_code.unwrap_or(-1);
let tail = tail_of(&p.log_path, 3);
let long_tail = if code == crate::watchdog::PACK_OUT_OF_DATE_CODE { tail_of(&p.log_path, 40) } else { Vec::new() };
self.miners[i].proc = None;
let t = self.secs(now);
let verdict = self.miners[i].watch.event(t, crate::watchdog::Event::Exited(code));
@ -2469,6 +2473,37 @@ impl Engine {
self.miners[i].needs_rebuild = true;
}
self.miners[i].restart_at = Some(now);
} else if code == crate::watchdog::PACK_OUT_OF_DATE_CODE {
// The worker refused its program pack and the miner could not rebuild it (or hit its own
// cap): export the pack from the node again before the next start, not a blind restart;
// at most PACK_REBUILD_CAP times per epoch, then the card shows the reason
let why = long_tail.iter().rev().find_map(|l| crate::watchdog::pack_refusal(l)).unwrap_or_else(|| "the worker refused its program pack".into());
let epoch = std::fs::read_to_string(self.shared.runtime.app_dir.join("packs").join("devnet").join("seeds.txt")).ok().and_then(|s| crate::watchdog::pack_epoch_of(&s)).unwrap_or_default();
let name = self.st().mining.cards.get(card_idx).map(|c| c.name.clone()).unwrap_or_else(|| self.miners[i].label.clone());
match self.miners[i].pack_rebuilds.decide(&epoch, &why) {
crate::watchdog::PackAction::Rebuild { n, cap } => {
self.shared.event("build", "program pack out of date, rebuilding");
self.shared.log(&format!("{name}: program pack out of date, rebuilding (export {n} of {cap} for epoch {epoch}): {why}"));
self.miners[i].prepared = false; // prepare_worker exports the pack again before the start
self.miners[i].restart_at = Some(now);
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
c.state = "restarting".into();
c.hash_now = 0.0;
c.message = "program pack out of date, rebuilding".into();
}
}
crate::watchdog::PackAction::GiveUp { n: _, reason } => {
self.shared.event("error", &format!("{name}: {reason}"));
self.shared.log(&format!("{name}: {reason}; next try in 10 minutes or at the next hour"));
self.miners[i].prepared = false;
self.miners[i].restart_at = Some(now + Duration::from_secs(600));
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
c.state = "failed".into();
c.hash_now = 0.0;
c.message = reason;
}
}
}
} else if verdict != crate::watchdog::Action::None {
// exit 43: the miner gave up on its worker; once more, then the card is faulted
self.watchdog_verdict(i, verdict, &tail);
@ -2546,6 +2581,25 @@ impl Engine {
}
}
/// The strip, the log and the card for a program pack the worker refused or the miner found stale: the miner
/// rebuilds the pack and restarts the worker itself (or exits 44 for the app to export it). One strip line per
/// 30 s: the miner prints the refusal on stderr and its own line on stdout.
fn pack_notice(&mut self, i: usize, card: usize, card_name: &str, why: &str) {
let now = Instant::now();
self.shared.log(&format!("{card_name}: program pack out of date, rebuilding: {why}"));
if now.duration_since(self.last_error_event) >= Duration::from_secs(30) {
self.last_error_event = now;
self.shared.event("build", "program pack out of date, rebuilding");
}
self.miners[i].error_at = Some(now);
let t = self.secs(now);
self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart("program pack out of date, rebuilding"));
if let Some(c) = self.st().mining.cards.get_mut(card) {
c.hash_now = 0.0;
c.message = "program pack out of date, rebuilding".into();
}
}
/// Applies a watchdog verdict to slot `i` (its process already stopped or gone): a restart now with the reason on
/// the card, or the card marked faulted with the reason in the UI and the log while the other cards keep mining.
fn watchdog_verdict(&mut self, i: usize, verdict: crate::watchdog::Action, tail: &[String]) {
@ -2867,6 +2921,10 @@ impl Engine {
if let Some(c) = self.st().mining.cards.get_mut(card) {
c.message = "the node is not answering; the miner retries".into();
}
} else if let Some(why) = crate::watchdog::pack_refusal(text) {
// the worker refused its program pack; the miner rebuilds the pack and restarts the worker itself
// (or exits 44 for us to export it): the strip and the card name the condition in plain words
self.pack_notice(i, card, &card_name, &why);
} else if text.contains("WORKER MISMATCH") || text.contains("worker error") || text.contains("worker exited") || text.contains("worker killed by a guard") || text.contains("panicked") || text.contains("CUDA error") || text.contains("submit error") {
let now = Instant::now();
if now.duration_since(self.last_error_event) >= Duration::from_secs(30) {
@ -2889,6 +2947,12 @@ impl Engine {
}
return;
}
if text.contains("PACK OUT OF DATE") {
// the miner found its pack stale or refused (stdout): it rebuilds the pack before the restart
let why = crate::watchdog::pack_refusal(text).unwrap_or_else(|| short(text, 160));
self.pack_notice(i, card, &card_name, &why);
return;
}
if text.contains(" WORKER FAULT ") {
// the miner's guards (interval, job time, cpu re-check, stall) killed the worker; it restarts it itself
let reason = crate::watchdog::fault_reason(text).unwrap_or_else(|| short(text, 160));
@ -3398,9 +3462,17 @@ fn civil_from_days(z: i64) -> (i64, u32, u32) {
(if m <= 2 { y + 1 } else { y }, m, d)
}
/// One pack export at a time (5 October 2026). prepare_worker runs on a thread per card, so two cards starting
/// together ran two `igneum-miner export-pack` processes into the same folder; across an epoch change they
/// interleaved and PC 1's packs\devnet was left with one epoch's program.h and the other's seeds.txt, which every
/// OpenCL worker start then refused ("the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT") until the next
/// export. The second export of a pair rewrites the same pack, which is harmless.
static EXPORT_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
/// Exports this hour's program pack from the node to <app data>\packs\devnet (the prebuilt workers read it with --pack).
#[allow(unused_variables)]
fn export_pack(shared: &Arc<Shared>, bins: &Bins) -> Result<(), String> {
let _one_at_a_time = EXPORT_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let pack = shared.runtime.app_dir.join("packs").join("devnet");
let _ = std::fs::create_dir_all(&pack);
let out = crate::detect::run_timeout(std::process::Command::new(&bins.miner).args(["export-pack", &shared.runtime.rpc_url(), &pack.display().to_string()]), None, Duration::from_secs(120)).unwrap_or_default();
@ -3419,6 +3491,7 @@ fn build_worker_from_source(shared: &Arc<Shared>, bins: &Bins, vendor: &str) ->
#[cfg(windows)]
{
use std::process::Command;
let _one_at_a_time = EXPORT_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let pack = shared.runtime.app_dir.join("packs").join("devnet");
let _ = std::fs::create_dir_all(&pack);
let out = crate::detect::run_timeout(Command::new(&bins.miner).args(["export-pack", &shared.runtime.rpc_url(), &pack.display().to_string()]), None, Duration::from_secs(120)).unwrap_or_default();