engine: one pack export at a time (EXPORT_LOCK around both export-pack call sites; the hunk of opencl-rdna4 23810df, whose other changes conflict with gpu-hotplug and wait for the next cut)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/igneum-app/src/engine.rs

@@ -430,6 +430,8 @@ struct MinerSlot {
     error_at: Option<Instant>,
     /// the per-card watchdog (src/watchdog.rs): one restart, then faulted
     watch: crate::watchdog::CardWatch,
+    /// pack refusals (src/watchdog.rs): the pack is exported again before the restart, capped per epoch
+    pack_rebuilds: crate::watchdog::PackRebuilds,
 }
 
 pub struct Engine {
@@ -1337,6 +1339,7 @@ impl Engine {
                 last_status: None,
                 error_at: None,
                 watch: crate::watchdog::CardWatch::new(),
+                pack_rebuilds: crate::watchdog::PackRebuilds::new(),
             });
         }
         if self.miners.is_empty() {
@@ -2460,6 +2463,7 @@ impl Engine {
                 if !p.alive() {
                     let code = p.exit_code.unwrap_or(-1);
                     let tail = tail_of(&p.log_path, 3);
+                    let long_tail = if code == crate::watchdog::PACK_OUT_OF_DATE_CODE { tail_of(&p.log_path, 40) } else { Vec::new() };
                     self.miners[i].proc = None;
                     let t = self.secs(now);
                     let verdict = self.miners[i].watch.event(t, crate::watchdog::Event::Exited(code));
@@ -2469,6 +2473,37 @@ impl Engine {
                             self.miners[i].needs_rebuild = true;
                         }
                         self.miners[i].restart_at = Some(now);
+                    } else if code == crate::watchdog::PACK_OUT_OF_DATE_CODE {
+                        // The worker refused its program pack and the miner could not rebuild it (or hit its own
+                        // cap): export the pack from the node again before the next start, not a blind restart;
+                        // at most PACK_REBUILD_CAP times per epoch, then the card shows the reason
+                        let why = long_tail.iter().rev().find_map(|l| crate::watchdog::pack_refusal(l)).unwrap_or_else(|| "the worker refused its program pack".into());
+                        let epoch = std::fs::read_to_string(self.shared.runtime.app_dir.join("packs").join("devnet").join("seeds.txt")).ok().and_then(|s| crate::watchdog::pack_epoch_of(&s)).unwrap_or_default();
+                        let name = self.st().mining.cards.get(card_idx).map(|c| c.name.clone()).unwrap_or_else(|| self.miners[i].label.clone());
+                        match self.miners[i].pack_rebuilds.decide(&epoch, &why) {
+                            crate::watchdog::PackAction::Rebuild { n, cap } => {
+                                self.shared.event("build", "program pack out of date, rebuilding");
+                                self.shared.log(&format!("{name}: program pack out of date, rebuilding (export {n} of {cap} for epoch {epoch}): {why}"));
+                                self.miners[i].prepared = false; // prepare_worker exports the pack again before the start
+                                self.miners[i].restart_at = Some(now);
+                                if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
+                                    c.state = "restarting".into();
+                                    c.hash_now = 0.0;
+                                    c.message = "program pack out of date, rebuilding".into();
+                                }
+                            }
+                            crate::watchdog::PackAction::GiveUp { n: _, reason } => {
+                                self.shared.event("error", &format!("{name}: {reason}"));
+                                self.shared.log(&format!("{name}: {reason}; next try in 10 minutes or at the next hour"));
+                                self.miners[i].prepared = false;
+                                self.miners[i].restart_at = Some(now + Duration::from_secs(600));
+                                if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
+                                    c.state = "failed".into();
+                                    c.hash_now = 0.0;
+                                    c.message = reason;
+                                }
+                            }
+                        }
                     } else if verdict != crate::watchdog::Action::None {
                         // exit 43: the miner gave up on its worker; once more, then the card is faulted
                         self.watchdog_verdict(i, verdict, &tail);
@@ -2546,6 +2581,25 @@ impl Engine {
         }
     }
 
+    /// The strip, the log and the card for a program pack the worker refused or the miner found stale: the miner
+    /// rebuilds the pack and restarts the worker itself (or exits 44 for the app to export it). One strip line per
+    /// 30 s: the miner prints the refusal on stderr and its own line on stdout.
+    fn pack_notice(&mut self, i: usize, card: usize, card_name: &str, why: &str) {
+        let now = Instant::now();
+        self.shared.log(&format!("{card_name}: program pack out of date, rebuilding: {why}"));
+        if now.duration_since(self.last_error_event) >= Duration::from_secs(30) {
+            self.last_error_event = now;
+            self.shared.event("build", "program pack out of date, rebuilding");
+        }
+        self.miners[i].error_at = Some(now);
+        let t = self.secs(now);
+        self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart("program pack out of date, rebuilding"));
+        if let Some(c) = self.st().mining.cards.get_mut(card) {
+            c.hash_now = 0.0;
+            c.message = "program pack out of date, rebuilding".into();
+        }
+    }
+
     /// Applies a watchdog verdict to slot `i` (its process already stopped or gone): a restart now with the reason on
     /// the card, or the card marked faulted with the reason in the UI and the log while the other cards keep mining.
     fn watchdog_verdict(&mut self, i: usize, verdict: crate::watchdog::Action, tail: &[String]) {
@@ -2867,6 +2921,10 @@ impl Engine {
                 if let Some(c) = self.st().mining.cards.get_mut(card) {
                     c.message = "the node is not answering; the miner retries".into();
                 }
+            } else if let Some(why) = crate::watchdog::pack_refusal(text) {
+                // the worker refused its program pack; the miner rebuilds the pack and restarts the worker itself
+                // (or exits 44 for us to export it): the strip and the card name the condition in plain words
+                self.pack_notice(i, card, &card_name, &why);
             } else if text.contains("WORKER MISMATCH") || text.contains("worker error") || text.contains("worker exited") || text.contains("worker killed by a guard") || text.contains("panicked") || text.contains("CUDA error") || text.contains("submit error") {
                 let now = Instant::now();
                 if now.duration_since(self.last_error_event) >= Duration::from_secs(30) {
@@ -2889,6 +2947,12 @@ impl Engine {
             }
             return;
         }
+        if text.contains("PACK OUT OF DATE") {
+            // the miner found its pack stale or refused (stdout): it rebuilds the pack before the restart
+            let why = crate::watchdog::pack_refusal(text).unwrap_or_else(|| short(text, 160));
+            self.pack_notice(i, card, &card_name, &why);
+            return;
+        }
         if text.contains(" WORKER FAULT ") {
             // the miner's guards (interval, job time, cpu re-check, stall) killed the worker; it restarts it itself
             let reason = crate::watchdog::fault_reason(text).unwrap_or_else(|| short(text, 160));
@@ -3398,9 +3462,17 @@ fn civil_from_days(z: i64) -> (i64, u32, u32) {
     (if m <= 2 { y + 1 } else { y }, m, d)
 }
 
+/// One pack export at a time (5 October 2026). prepare_worker runs on a thread per card, so two cards starting
+/// together ran two `igneum-miner export-pack` processes into the same folder; across an epoch change they
+/// interleaved and PC 1's packs\devnet was left with one epoch's program.h and the other's seeds.txt, which every
+/// OpenCL worker start then refused ("the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT") until the next
+/// export. The second export of a pair rewrites the same pack, which is harmless.
+static EXPORT_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
+
 /// Exports this hour's program pack from the node to <app data>\packs\devnet (the prebuilt workers read it with --pack).
 #[allow(unused_variables)]
 fn export_pack(shared: &Arc<Shared>, bins: &Bins) -> Result<(), String> {
+    let _one_at_a_time = EXPORT_LOCK.lock().unwrap_or_else(|e| e.into_inner());
     let pack = shared.runtime.app_dir.join("packs").join("devnet");
     let _ = std::fs::create_dir_all(&pack);
     let out = crate::detect::run_timeout(std::process::Command::new(&bins.miner).args(["export-pack", &shared.runtime.rpc_url(), &pack.display().to_string()]), None, Duration::from_secs(120)).unwrap_or_default();
@@ -3419,6 +3491,7 @@ fn build_worker_from_source(shared: &Arc<Shared>, bins: &Bins, vendor: &str) ->
     #[cfg(windows)]
     {
         use std::process::Command;
+        let _one_at_a_time = EXPORT_LOCK.lock().unwrap_or_else(|e| e.into_inner());
         let pack = shared.runtime.app_dir.join("packs").join("devnet");
         let _ = std::fs::create_dir_all(&pack);
         let out = crate::detect::run_timeout(Command::new(&bins.miner).args(["export-pack", &shared.runtime.rpc_url(), &pack.display().to_string()]), None, Duration::from_secs(120)).unwrap_or_default();