Merge origin/miner-reliability into release-0.3.5
Conflicts resolved keeping both: engine.rs carries the sweep state (miner-eff) and the node watchdog (reliability); the STATUS line goes through watchdog::parse_status and still feeds the sweep's rate sample; main.rs declares both modules; bench-log.md keeps both entries. site/build.mjs keeps master's partial-injected pages and adds the /miners bench table through the same page() with active: 'miners'; the Miners link is in site/partials/nav.html; sitemap gains /miners; every generated page rebuilt with node site/build.mjs.
This commit is contained in:
igneum-labs
commit
cfb937b42d
22 changed files with 1666 additions and 74 deletions
|
|
@ -379,6 +379,8 @@ struct MinerSlot {
|
|||
prepared: bool, // the pack is exported (and the worker built) for the next start
|
||||
last_status: Option<Instant>,
|
||||
error_at: Option<Instant>,
|
||||
/// the per-card watchdog (src/watchdog.rs): one restart, then faulted
|
||||
watch: crate::watchdog::CardWatch,
|
||||
}
|
||||
|
||||
pub struct Engine {
|
||||
|
|
@ -454,6 +456,10 @@ pub struct Engine {
|
|||
/// after an abort, a card waits before the scheduler tries it again
|
||||
sweep_retry: std::collections::HashMap<usize, Instant>,
|
||||
sweep_attempts: std::collections::HashMap<usize, u32>,
|
||||
/// the node watchdog (src/watchdog.rs): a silent node of ours is restarted
|
||||
node_watch: crate::watchdog::NodeWatch,
|
||||
/// the watchdogs' clock origin (they take seconds)
|
||||
t0: Instant,
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
|
|
@ -534,6 +540,8 @@ impl Engine {
|
|||
sweep_next_check: now + Duration::from_secs(20),
|
||||
sweep_retry: std::collections::HashMap::new(),
|
||||
sweep_attempts: std::collections::HashMap::new(),
|
||||
node_watch: crate::watchdog::NodeWatch::new(),
|
||||
t0: now,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -541,6 +549,11 @@ impl Engine {
|
|||
self.shared.state.lock().unwrap()
|
||||
}
|
||||
|
||||
/// Seconds on the watchdogs' clock.
|
||||
fn secs(&self, now: Instant) -> f64 {
|
||||
now.duration_since(self.t0).as_secs_f64()
|
||||
}
|
||||
|
||||
pub fn run(mut self) {
|
||||
let setup_done = self.shared.settings.lock().unwrap().setup_done;
|
||||
self.shared.log(&format!(
|
||||
|
|
@ -677,6 +690,13 @@ impl Engine {
|
|||
self.shared.save_settings();
|
||||
self.st().mining.paused = false;
|
||||
self.shared.event("ok", "mining resumed");
|
||||
// a user action: faulted cards try again
|
||||
for m in self.miners.iter_mut() {
|
||||
if m.watch.faulted().is_some() {
|
||||
m.watch.event(0.0, crate::watchdog::Event::Reset);
|
||||
m.restart_at = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
if !self.running {
|
||||
self.start();
|
||||
}
|
||||
|
|
@ -685,6 +705,8 @@ impl Engine {
|
|||
self.shared.event("info", &format!("settings changed ({why}); the miners restart"));
|
||||
self.stop_miners("settings changed");
|
||||
for m in self.miners.iter_mut() {
|
||||
// a user action: a faulted card tries again
|
||||
m.watch.event(0.0, crate::watchdog::Event::Reset);
|
||||
m.restart_at = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
|
@ -1096,6 +1118,7 @@ impl Engine {
|
|||
prepared: false,
|
||||
last_status: None,
|
||||
error_at: None,
|
||||
watch: crate::watchdog::CardWatch::new(),
|
||||
});
|
||||
}
|
||||
if self.miners.is_empty() {
|
||||
|
|
@ -1204,6 +1227,8 @@ impl Engine {
|
|||
self.miners[i].proc = Some(p);
|
||||
self.miners[i].restart_at = None;
|
||||
self.miners[i].last_status = None;
|
||||
let t = self.secs(Instant::now());
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::Started);
|
||||
}
|
||||
Err(e) => {
|
||||
self.shared.event("error", &format!("the miner could not start: {e}"));
|
||||
|
|
@ -1217,17 +1242,19 @@ impl Engine {
|
|||
if any {
|
||||
self.shared.log(&format!("stopping the miners ({why})"));
|
||||
}
|
||||
let t = self.secs(Instant::now());
|
||||
for m in self.miners.iter_mut() {
|
||||
if let Some(mut p) = m.proc.take() {
|
||||
p.write_stdin("quit\n");
|
||||
p.stop(8);
|
||||
}
|
||||
m.restart_at = None;
|
||||
m.watch.event(t, crate::watchdog::Event::Stopped);
|
||||
}
|
||||
let mut st = self.st();
|
||||
let paused = st.mining.paused;
|
||||
for c in st.mining.cards.iter_mut() {
|
||||
if c.enabled {
|
||||
if c.enabled && c.state != "faulted" {
|
||||
c.state = if paused { "off".into() } else { "waiting".into() };
|
||||
c.hash_now = 0.0;
|
||||
c.pid = 0;
|
||||
|
|
@ -1306,6 +1333,7 @@ impl Engine {
|
|||
if enabled {
|
||||
self.miners[pos].restart_at = Some(Instant::now());
|
||||
self.miners[pos].prepared = false;
|
||||
self.miners[pos].watch.event(0.0, crate::watchdog::Event::Reset);
|
||||
self.shared.event("info", &format!("{name}: {identities} identit{}; its worker restarts", if identities == 1 { "y" } else { "ies" }));
|
||||
} else {
|
||||
self.miners.remove(pos);
|
||||
|
|
@ -2028,13 +2056,7 @@ impl Engine {
|
|||
if self.node_external {
|
||||
self.shared.event("info", "remote job asked for a node restart, but the node is external; nothing done");
|
||||
} else {
|
||||
self.stop_miners("remote job: restart node");
|
||||
self.stop_node();
|
||||
self.node_restart_at = Some(Instant::now() + Duration::from_secs(3));
|
||||
for m in self.miners.iter_mut() {
|
||||
m.restart_at = Some(Instant::now());
|
||||
}
|
||||
self.st().node.message = "restart asked by a remote job".into();
|
||||
self.restart_node("restart asked by a remote job", Duration::from_secs(3));
|
||||
}
|
||||
}
|
||||
Action::RestartApp => {
|
||||
|
|
@ -2186,6 +2208,15 @@ impl Engine {
|
|||
st.node.message = "no answer from the RPC yet (still opening its database?)".into();
|
||||
}
|
||||
}
|
||||
// the node watchdog: our node silent for 120 s (no reading, nothing accepted) is restarted in-process
|
||||
if self.node.is_some() && !self.node_external && self.node_restart_at.is_none() {
|
||||
let silent_s = self.node_last_reading.map(|t| now.duration_since(t)).unwrap_or_else(|| self.node_started_at.elapsed()).as_secs_f64();
|
||||
if let Some(delay) = self.node_watch.tick(self.secs(now), true, silent_s, accepted_recent) {
|
||||
let n = self.node_watch.restarts_in_window();
|
||||
self.shared.event("error", &format!("no reading from the node for {} s; the watchdog restarts it in {delay} s (restart {n} in the last 10 minutes)", silent_s as u64));
|
||||
self.restart_node("no reading from the node for 120 s", Duration::from_secs(delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn tick_miners(&mut self, now: Instant) {
|
||||
|
|
@ -2199,12 +2230,17 @@ impl Engine {
|
|||
let code = p.exit_code.unwrap_or(-1);
|
||||
let tail = tail_of(&p.log_path, 3);
|
||||
self.miners[i].proc = None;
|
||||
let t = self.secs(now);
|
||||
let verdict = self.miners[i].watch.event(t, crate::watchdog::Event::Exited(code));
|
||||
if code == 42 {
|
||||
self.shared.event("build", "the hourly program changed; the worker restarts on the new one");
|
||||
if self.bins_worker_missing(card_idx) {
|
||||
self.miners[i].needs_rebuild = true;
|
||||
}
|
||||
self.miners[i].restart_at = Some(now);
|
||||
} else if verdict != crate::watchdog::Action::None {
|
||||
// exit 43: the miner gave up on its worker; once more, then the card is faulted
|
||||
self.watchdog_verdict(i, verdict, &tail);
|
||||
} else {
|
||||
self.miners[i].restarts += 1;
|
||||
let delay = jitter_secs(crate::platform::unix_now() + i as u64);
|
||||
|
|
@ -2220,17 +2256,34 @@ impl Engine {
|
|||
c.message = tail.last().cloned().unwrap_or_default();
|
||||
}
|
||||
}
|
||||
} else if let Some(t) = self.miners[i].last_status {
|
||||
if now.duration_since(t) > Duration::from_secs(self.shared.runtime.status_secs as u64 * 4) {
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
||||
if c.state == "mining" {
|
||||
c.message = "no status from the miner for a while".into();
|
||||
} else {
|
||||
// the watchdog: no status for 90 s, or a zero rate for 60 s while synced, is one restart, then faulted
|
||||
let t = self.secs(now);
|
||||
let verdict = self.miners[i].watch.tick(t, synced);
|
||||
if verdict != crate::watchdog::Action::None {
|
||||
if let Some(mut p) = self.miners[i].proc.take() {
|
||||
p.write_stdin("quit\n");
|
||||
p.stop(3);
|
||||
}
|
||||
self.watchdog_verdict(i, verdict, &[]);
|
||||
continue;
|
||||
}
|
||||
if let Some(t) = self.miners[i].last_status {
|
||||
if now.duration_since(t) > Duration::from_secs(self.shared.runtime.status_secs as u64 * 4) {
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
||||
if c.state == "mining" {
|
||||
c.message = "no status from the miner for a while".into();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if self.miners[i].watch.faulted().is_some() {
|
||||
// faulted by the watchdog: not restarted until the user changes the card's settings or resumes
|
||||
continue;
|
||||
}
|
||||
if self.job_hold {
|
||||
// a remote job has the GPU; the miners wait until it lets go (src/jobrun.rs)
|
||||
continue;
|
||||
|
|
@ -2262,6 +2315,58 @@ impl Engine {
|
|||
}
|
||||
}
|
||||
|
||||
/// Applies a watchdog verdict to slot `i` (its process already stopped or gone): a restart now with the reason on
|
||||
/// the card, or the card marked faulted with the reason in the UI and the log while the other cards keep mining.
|
||||
fn watchdog_verdict(&mut self, i: usize, verdict: crate::watchdog::Action, tail: &[String]) {
|
||||
use crate::watchdog::Action;
|
||||
let card_idx = self.miners[i].card;
|
||||
let name = self.st().mining.cards.get(card_idx).map(|c| c.name.clone()).unwrap_or_else(|| self.miners[i].label.clone());
|
||||
for l in tail {
|
||||
self.shared.log(&format!(" miner: {l}"));
|
||||
}
|
||||
match verdict {
|
||||
Action::Restart(reason) => {
|
||||
self.miners[i].restarts += 1;
|
||||
self.miners[i].restart_at = Some(Instant::now() + Duration::from_secs(1));
|
||||
self.shared.event("error", &format!("{name}: {reason}; the watchdog restarts the miner (restart {} of 1 before the card is marked faulted)", self.miners[i].watch.watchdog_restarts()));
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
||||
c.state = "restarting".into();
|
||||
c.restarts = self.miners[i].restarts;
|
||||
c.hash_now = 0.0;
|
||||
c.pid = 0;
|
||||
c.message = format!("watchdog: {reason}");
|
||||
}
|
||||
}
|
||||
Action::Fault(reason) => {
|
||||
self.miners[i].restart_at = None;
|
||||
self.shared.event("error", &format!("{name}: {reason}; the card is marked faulted and its miner is not restarted again (the other cards keep mining; change the card's settings or resume mining to try again)"));
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
||||
c.state = "faulted".into();
|
||||
c.hash_now = 0.0;
|
||||
c.pid = 0;
|
||||
c.restart_in_s = 0;
|
||||
c.message = format!("faulted: {reason}");
|
||||
}
|
||||
}
|
||||
Action::None => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops the miners and the node and schedules the node's start after `delay`; the miners follow once it is
|
||||
/// synced. The remote-job restart kind and the node watchdog share it.
|
||||
fn restart_node(&mut self, why: &str, delay: Duration) {
|
||||
self.stop_miners(why);
|
||||
self.stop_node();
|
||||
self.node_restart_at = Some(Instant::now() + delay);
|
||||
for m in self.miners.iter_mut() {
|
||||
m.restart_at = Some(Instant::now());
|
||||
}
|
||||
let mut st = self.st();
|
||||
st.node.state = "restarting".into();
|
||||
st.node.synced = false;
|
||||
st.node.message = format!("{why}; restart in {} s", delay.as_secs());
|
||||
}
|
||||
|
||||
fn bins_worker_missing(&self, card_idx: usize) -> bool {
|
||||
let st = self.st();
|
||||
match st.mining.cards.get(card_idx).map(|c| c.worker.as_str()) {
|
||||
|
|
@ -2497,7 +2602,7 @@ impl Engine {
|
|||
if synced && !was_synced {
|
||||
self.shared.event("ok", &format!("node synced: {blocks} blocks, {peers} peer(s)"));
|
||||
for m in self.miners.iter_mut() {
|
||||
if m.proc.is_none() && m.restart_at.is_none() {
|
||||
if m.proc.is_none() && m.restart_at.is_none() && m.watch.faulted().is_none() {
|
||||
m.restart_at = Some(now);
|
||||
}
|
||||
}
|
||||
|
|
@ -2523,16 +2628,38 @@ impl Engine {
|
|||
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
||||
c.message = "the node is not answering; the miner retries".into();
|
||||
}
|
||||
} else if text.contains("WORKER MISMATCH") || text.contains("worker error") || text.contains("worker exited") || text.contains("panicked") || text.contains("CUDA error") || text.contains("submit error") {
|
||||
} else if text.contains("WORKER MISMATCH") || text.contains("worker error") || text.contains("worker exited") || text.contains("worker killed by a guard") || text.contains("panicked") || text.contains("CUDA error") || text.contains("submit error") {
|
||||
let now = Instant::now();
|
||||
if now.duration_since(self.last_error_event) >= Duration::from_secs(30) {
|
||||
self.last_error_event = now;
|
||||
self.shared.event("error", &format!("{card_name}: {}", short(text, 160)));
|
||||
}
|
||||
self.miners[i].error_at = Some(now);
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
||||
c.message = short(text, 160);
|
||||
if text.contains("worker exited") || text.contains("worker killed by a guard") {
|
||||
// the miner restarts its worker itself: the watchdog waits for `ready` instead of restarting the miner too
|
||||
let t = self.secs(now);
|
||||
let reason = short(text.split_once(' ').map(|(_, r)| r).unwrap_or(text), 160);
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart(&reason));
|
||||
}
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
||||
// the restart note never hides the fault reason the card already shows
|
||||
if !(c.message.starts_with("worker fault: ") && (text.contains("worker killed by a guard") || text.contains("worker exited"))) {
|
||||
c.message = short(text, 160);
|
||||
}
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if text.contains(" WORKER FAULT ") {
|
||||
// the miner's guards (interval, job time, cpu re-check, stall) killed the worker; it restarts it itself
|
||||
let reason = crate::watchdog::fault_reason(text).unwrap_or_else(|| short(text, 160));
|
||||
let t = self.secs(Instant::now());
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart(&reason));
|
||||
self.shared.event("error", &format!("{card_name}: worker fault: {}; the miner restarts the worker", short(&reason, 200)));
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
||||
c.faults += 1;
|
||||
c.hash_now = 0.0;
|
||||
c.message = format!("worker fault: {}", short(&reason, 160));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
|
@ -2562,31 +2689,14 @@ impl Engine {
|
|||
}
|
||||
drop(st);
|
||||
self.shared.event("block", &format!("block accepted by the node ({card_name})"));
|
||||
} else if text.contains(" dev-fee block ") {
|
||||
// the software dev fee: one template in 100 paid the dev address (igneum-miner, "Software dev fee")
|
||||
let total = {
|
||||
let mut s = self.shared.settings.lock().unwrap();
|
||||
s.fee_total += 1;
|
||||
s.fee_total
|
||||
};
|
||||
let mut st = self.st();
|
||||
st.mining.fee_session += 1;
|
||||
st.mining.fee_total = total;
|
||||
drop(st);
|
||||
self.shared.event("block", &format!("that block was the dev fee's (1 in 100, {card_name})"));
|
||||
} else if let Some(rest) = text.split(" dev fee ").nth(1) {
|
||||
// the miner's start line: "dev fee 1% (1 block in 100) to 0x...; --dev-fee 0 turns it off", or
|
||||
// "dev fee off (--dev-fee 0); the default is 1% (1 block in 100) to 0x...", or "dev fee off: <why>"
|
||||
let on = !rest.starts_with("off");
|
||||
let percent = if on { rest.split('%').next().and_then(|p| p.trim().parse::<u32>().ok()).unwrap_or(1) } else { 0 };
|
||||
let address = rest.find("0x").map(|k| rest[k..].chars().take(42).collect::<String>()).filter(|a| a.len() == 42 && keys::valid_address(a)).unwrap_or_default();
|
||||
let mut st = self.st();
|
||||
st.dev_fee = crate::state::DevFeeState { on, percent, address, line: rest.to_string() };
|
||||
} else if text.contains(" STATUS '") {
|
||||
self.miners[i].last_status = Some(Instant::now());
|
||||
} else if let Some(s) = crate::watchdog::parse_status(text) {
|
||||
let now = Instant::now();
|
||||
self.miners[i].last_status = Some(now);
|
||||
let t = self.secs(now);
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::Status(&s));
|
||||
let avg = kv_f64(text, "hash").unwrap_or(0.0);
|
||||
// the v4 miner's interval rate: "now=12.34 MH/s wall (...)"; older miners: the average
|
||||
let now_rate = kv_f64(text, "now").unwrap_or(avg);
|
||||
let now_rate = if s.hash_now > 0.0 { s.hash_now } else { avg };
|
||||
if let Some(run) = self.sweep.as_mut() {
|
||||
if run.card == card {
|
||||
run.sample_rate(now_rate);
|
||||
|
|
@ -2595,21 +2705,26 @@ impl Engine {
|
|||
let mut st = self.st();
|
||||
if let Some(c) = st.mining.cards.get_mut(card) {
|
||||
c.hash_avg = avg;
|
||||
c.hash_now = now_rate;
|
||||
// the v4 miner's interval rate: "now=12.34 MH/s wall (...)"; older miners: the average
|
||||
c.hash_now = s.hash_now;
|
||||
c.template_age_s = kv_f64(text, "template_age").unwrap_or(0.0);
|
||||
c.synced = kv(text, "synced") == Some("true");
|
||||
c.synced = s.synced;
|
||||
c.mismatched = s.mismatched;
|
||||
c.faults = c.faults.max(s.faults);
|
||||
c.last_status_age_s = 0.0;
|
||||
if c.state == "starting" || c.state == "ready" {
|
||||
c.state = "mining".into();
|
||||
}
|
||||
if c.state == "mining" {
|
||||
c.message = String::new();
|
||||
if c.state == "mining" && s.hash_now > 0.0 {
|
||||
c.message = if s.mismatched > 0 { format!("{} share(s) failed the CPU re-check this run", s.mismatched) } else { String::new() };
|
||||
}
|
||||
}
|
||||
} else if text.contains(" worker: race ") {
|
||||
self.race_line(card, &card_name, text);
|
||||
} else if text.contains(" worker: ready ") {
|
||||
let prepare = text.contains(" prepare 1");
|
||||
let t = self.secs(Instant::now());
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::Ready);
|
||||
let mut st = self.st();
|
||||
if let Some(c) = st.mining.cards.get_mut(card) {
|
||||
c.ready = true;
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ mod jobrun;
|
|||
mod jobbuild;
|
||||
mod prover;
|
||||
mod sweep;
|
||||
mod watchdog;
|
||||
|
||||
use std::io::{BufRead, Write};
|
||||
use std::sync::mpsc::channel;
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ pub struct CardState {
|
|||
pub detail: String, // memory, cores
|
||||
pub device: String, // the worker's --device value (Windows)
|
||||
pub enabled: bool,
|
||||
pub state: String, // off | waiting | starting | ready | mining | restarting | failed
|
||||
pub state: String, // off | waiting | starting | ready | mining | restarting | failed | faulted (the watchdog gave up on it)
|
||||
pub hash_now: f64, // MH/s, the last interval
|
||||
pub hash_avg: f64, // MH/s since the start
|
||||
pub accepted: u64,
|
||||
|
|
@ -57,6 +57,10 @@ pub struct CardState {
|
|||
pub pid: u32,
|
||||
pub last_status_age_s: f64,
|
||||
pub message: String,
|
||||
/// `WORKER FAULT` lines seen (the miner killed and restarted its worker) and the miner's own `faults=` count
|
||||
pub faults: u64,
|
||||
/// shares that failed the miner's CPU re-check this run (`mismatched=` on the STATUS line)
|
||||
pub mismatched: u64,
|
||||
// power and heat (NVIDIA through nvidia-smi; 0 = unknown)
|
||||
pub power_w: f64, // draw now
|
||||
pub power_limit_w: f64, // the limit in force
|
||||
|
|
|
|||
482
app/igneum-app/src/watchdog.rs
Normal file
482
app/igneum-app/src/watchdog.rs
Normal file
|
|
@ -0,0 +1,482 @@
|
|||
//! The engine's watchdog, as state machines with no clock and no process (seconds come from the caller), so the
|
||||
//! rules can be unit-tested with recorded miner lines.
|
||||
//!
|
||||
//! Per card (`CardWatch`): a miner that prints no status line for 90 s, or reports a hash rate of 0 for 60 s while
|
||||
//! the node is synced, is restarted once; when that recurs before five minutes of healthy status, the card is marked
|
||||
//! faulted with the reason, its miner is not restarted again, and the other cards keep mining. The miner's own
|
||||
//! worker restarts (`WORKER FAULT`, `worker exited`) suspend both rules until the worker is ready again, so the app
|
||||
//! never restarts a miner that is already restarting its worker (no double restarts); if the worker is not back
|
||||
//! within 180 s the app steps in. Exit code 43 (the miner gave up on its worker after three guard trips) counts
|
||||
//! like a watchdog restart: once, then faulted.
|
||||
//!
|
||||
//! Per node (`NodeWatch`): a node of ours that answers no `watch` reading for 120 s is restarted by the app, with a
|
||||
//! growing delay when it repeats inside ten minutes.
|
||||
//!
|
||||
//! Review round 4, X21 (4 October 2026): the app now reads `mismatched=` and `faults=` from STATUS lines and the
|
||||
//! `WORKER FAULT` lines, and shows them on the card.
|
||||
|
||||
/// No status line from a running miner for this long: restart it.
|
||||
pub const NO_STATUS_S: f64 = 90.0;
|
||||
/// Hash rate 0 while the node is synced and the worker is ready for this long: restart the miner.
|
||||
pub const ZERO_RATE_S: f64 = 60.0;
|
||||
/// The miner is restarting its own worker: the app waits this long for `ready` before it steps in.
|
||||
pub const WORKER_RESTART_GRACE_S: f64 = 180.0;
|
||||
/// Continuous healthy status (rate above 0) that renews the one-restart budget.
|
||||
pub const HEALTHY_RESET_S: f64 = 300.0;
|
||||
/// `igneum-miner` exit code when its guards gave up on the worker (three trips in ten minutes).
|
||||
pub const MINER_GAVE_UP_CODE: i32 = 43;
|
||||
/// No `watch` reading from our node for this long: restart it.
|
||||
pub const NODE_SILENT_S: f64 = 120.0;
|
||||
/// Node restarts inside this window grow the delay before the next one.
|
||||
pub const NODE_WINDOW_S: f64 = 600.0;
|
||||
|
||||
/// What the watchdog reads from one `STATUS` line of `igneum-miner`.
|
||||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
pub struct Status {
|
||||
/// MH/s of the last interval (`now=`; older miners: the average `hash=`)
|
||||
pub hash_now: f64,
|
||||
pub mismatched: u64,
|
||||
pub faults: u64,
|
||||
pub restarts: u64,
|
||||
pub synced: bool,
|
||||
}
|
||||
|
||||
/// Parses a miner STATUS line; None for any other line.
|
||||
pub fn parse_status(text: &str) -> Option<Status> {
|
||||
if !text.contains(" STATUS '") {
|
||||
return None;
|
||||
}
|
||||
let avg = kv_f64(text, "hash").unwrap_or(0.0);
|
||||
Some(Status {
|
||||
hash_now: kv_f64(text, "now").unwrap_or(avg),
|
||||
mismatched: kv_u64(text, "mismatched").unwrap_or(0),
|
||||
faults: kv_u64(text, "faults").unwrap_or(0),
|
||||
restarts: kv_u64(text, "restarts").unwrap_or(0),
|
||||
synced: kv(text, "synced") == Some("true"),
|
||||
})
|
||||
}
|
||||
|
||||
/// The reason text of a `WORKER FAULT` line, without the timestamp and the trailing restart note.
|
||||
pub fn fault_reason(text: &str) -> Option<String> {
|
||||
let i = text.find("WORKER FAULT ")?;
|
||||
let rest = &text["WORKER FAULT ".len() + i..];
|
||||
let end = rest.find("; restarting the worker").or_else(|| rest.find("; killing the worker")).unwrap_or(rest.len());
|
||||
Some(rest[..end].trim().to_string())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Event<'a> {
|
||||
/// The app started the miner process.
|
||||
Started,
|
||||
/// The worker said ready.
|
||||
Ready,
|
||||
/// A STATUS line.
|
||||
Status(&'a Status),
|
||||
/// The miner killed its worker (a guard) or saw it exit; it restarts the worker itself.
|
||||
WorkerRestart(&'a str),
|
||||
/// The miner process ended with this code.
|
||||
Exited(i32),
|
||||
/// The app stopped the miner on purpose (pause, settings, node restart).
|
||||
Stopped,
|
||||
/// The user changed the card's settings: a faulted card may try again.
|
||||
Reset,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Action {
|
||||
None,
|
||||
/// Stop the miner and start it again now, for this reason.
|
||||
Restart(String),
|
||||
/// Mark the card faulted with this reason; do not restart its miner.
|
||||
Fault(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct CardWatch {
|
||||
started_s: Option<f64>,
|
||||
last_status_s: Option<f64>,
|
||||
ready: bool,
|
||||
zero_since: Option<f64>,
|
||||
healthy_since: Option<f64>,
|
||||
worker_restart_since: Option<f64>,
|
||||
/// app-level restarts without five healthy minutes since
|
||||
restarts: u32,
|
||||
faulted: Option<String>,
|
||||
last_fault: String,
|
||||
}
|
||||
|
||||
impl CardWatch {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn faulted(&self) -> Option<&str> {
|
||||
self.faulted.as_deref()
|
||||
}
|
||||
|
||||
pub fn watchdog_restarts(&self) -> u32 {
|
||||
self.restarts
|
||||
}
|
||||
|
||||
fn arm(&mut self, now_s: f64) {
|
||||
self.started_s = Some(now_s);
|
||||
self.last_status_s = None;
|
||||
self.ready = false;
|
||||
self.zero_since = None;
|
||||
self.healthy_since = None;
|
||||
self.worker_restart_since = None;
|
||||
}
|
||||
|
||||
fn escalate(&mut self, reason: String) -> Action {
|
||||
self.started_s = None;
|
||||
self.last_status_s = None;
|
||||
self.ready = false;
|
||||
self.zero_since = None;
|
||||
self.healthy_since = None;
|
||||
self.worker_restart_since = None;
|
||||
if self.restarts >= 1 {
|
||||
let r = format!("{reason} (restarted once already)");
|
||||
self.faulted = Some(r.clone());
|
||||
Action::Fault(r)
|
||||
} else {
|
||||
self.restarts += 1;
|
||||
Action::Restart(reason)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn event(&mut self, now_s: f64, ev: Event<'_>) -> Action {
|
||||
match ev {
|
||||
Event::Started => {
|
||||
self.arm(now_s);
|
||||
Action::None
|
||||
}
|
||||
Event::Ready => {
|
||||
self.ready = true;
|
||||
self.worker_restart_since = None;
|
||||
Action::None
|
||||
}
|
||||
Event::Status(s) => {
|
||||
self.last_status_s = Some(now_s);
|
||||
if s.hash_now > 0.0 {
|
||||
self.zero_since = None;
|
||||
let since = *self.healthy_since.get_or_insert(now_s);
|
||||
if now_s - since >= HEALTHY_RESET_S {
|
||||
self.restarts = 0;
|
||||
}
|
||||
} else {
|
||||
self.healthy_since = None;
|
||||
if self.ready && self.worker_restart_since.is_none() {
|
||||
self.zero_since.get_or_insert(now_s);
|
||||
}
|
||||
}
|
||||
Action::None
|
||||
}
|
||||
Event::WorkerRestart(reason) => {
|
||||
self.last_fault = reason.to_string();
|
||||
self.worker_restart_since = Some(now_s);
|
||||
self.ready = false;
|
||||
self.zero_since = None;
|
||||
self.healthy_since = None;
|
||||
Action::None
|
||||
}
|
||||
Event::Exited(code) => {
|
||||
let running = self.started_s.is_some();
|
||||
self.started_s = None;
|
||||
if code == MINER_GAVE_UP_CODE && running {
|
||||
let why = if self.last_fault.is_empty() { "its guards tripped three times in ten minutes".to_string() } else { self.last_fault.clone() };
|
||||
self.escalate(format!("the miner gave up on its worker: {why}"))
|
||||
} else {
|
||||
Action::None
|
||||
}
|
||||
}
|
||||
Event::Stopped => {
|
||||
self.started_s = None;
|
||||
self.last_status_s = None;
|
||||
self.ready = false;
|
||||
self.zero_since = None;
|
||||
self.worker_restart_since = None;
|
||||
Action::None
|
||||
}
|
||||
Event::Reset => {
|
||||
*self = Self::default();
|
||||
Action::None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Called every engine tick while the miner process is alive.
|
||||
pub fn tick(&mut self, now_s: f64, node_synced: bool) -> Action {
|
||||
if self.faulted.is_some() {
|
||||
return Action::None;
|
||||
}
|
||||
let Some(started) = self.started_s else { return Action::None };
|
||||
if let Some(t) = self.worker_restart_since {
|
||||
// the miner is restarting its worker: its own guards own the card until the worker is ready
|
||||
if now_s - t > WORKER_RESTART_GRACE_S {
|
||||
return self.escalate(format!("the worker did not come back within {} s of the miner restarting it ({})", WORKER_RESTART_GRACE_S as u64, self.last_fault));
|
||||
}
|
||||
return Action::None;
|
||||
}
|
||||
let last = self.last_status_s.unwrap_or(started);
|
||||
if now_s - last > NO_STATUS_S {
|
||||
return self.escalate(format!("no status line from the miner for {} s", NO_STATUS_S as u64));
|
||||
}
|
||||
if !node_synced {
|
||||
// a zero rate while the node syncs is expected; the timer starts again once it is synced
|
||||
self.zero_since = None;
|
||||
}
|
||||
if node_synced && self.ready {
|
||||
if let Some(z) = self.zero_since {
|
||||
if now_s - z >= ZERO_RATE_S {
|
||||
return self.escalate(format!("hash rate 0 for {} s while the node is synced", ZERO_RATE_S as u64));
|
||||
}
|
||||
}
|
||||
}
|
||||
Action::None
|
||||
}
|
||||
}
|
||||
|
||||
/// The node watchdog: no reading for `NODE_SILENT_S` restarts our node, with a growing delay when it repeats.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct NodeWatch {
|
||||
restarts_s: Vec<f64>,
|
||||
}
|
||||
|
||||
impl NodeWatch {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// `silent_s`: seconds since the last reading (or since the node started, when it never answered). Returns the
|
||||
/// delay in seconds before the restart when one is due.
|
||||
pub fn tick(&mut self, now_s: f64, ours: bool, silent_s: f64, accepted_recent: bool) -> Option<u64> {
|
||||
if !ours || accepted_recent || silent_s < NODE_SILENT_S {
|
||||
return None;
|
||||
}
|
||||
self.restarts_s.retain(|t| now_s - *t <= NODE_WINDOW_S);
|
||||
self.restarts_s.push(now_s);
|
||||
let n = self.restarts_s.len();
|
||||
Some((3u64.saturating_mul(1u64 << (n.saturating_sub(1) * 2).min(8))).min(300))
|
||||
}
|
||||
|
||||
pub fn restarts_in_window(&self) -> usize {
|
||||
self.restarts_s.len()
|
||||
}
|
||||
}
|
||||
|
||||
fn kv<'a>(line: &'a str, key: &str) -> Option<&'a str> {
|
||||
let pat = format!(" {key}=");
|
||||
let i = line.find(&pat)? + pat.len();
|
||||
let rest = &line[i..];
|
||||
let end = rest.find(|c: char| c == ' ' || c == ',' || c == ')' || c == ';').unwrap_or(rest.len());
|
||||
Some(&rest[..end])
|
||||
}
|
||||
|
||||
fn kv_u64(line: &str, key: &str) -> Option<u64> {
|
||||
kv(line, key)?.parse().ok()
|
||||
}
|
||||
|
||||
fn kv_f64(line: &str, key: &str) -> Option<f64> {
|
||||
kv(line, key)?.trim_end_matches('s').parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// Lines as igneum-miner 0.3.x prints them (devnet v4, 4 October 2026; identities and labels shortened).
|
||||
const STATUS_OK: &str = "1791138616.597 STATUS 'win-1' [worker]: 70s jobs=486 accepted=3 rejected=0 mismatched=0 extra=0 rate=0.04 blocks/s hash=123.90 MH/s wall (124.20 MH/s inside jobs) now=124.10 MH/s wall (124.30 MH/s inside jobs, 70 jobs, seed walk 0 calls) template_age=0.31s synced=true idle=0.2% (last 10s: 0.1%) queued=2 restarts=0 faults=0 identities=8 accepted_by_identity=1/0/1/0/0/1/0/0";
|
||||
const STATUS_ZERO: &str = "1791138626.597 STATUS 'win-1' [worker]: 80s jobs=486 accepted=3 rejected=0 mismatched=0 extra=0 rate=0.04 blocks/s hash=108.41 MH/s wall (124.20 MH/s inside jobs) now=0.00 MH/s wall (0.00 MH/s inside jobs, 0 jobs, seed walk 0 calls) template_age=0.31s synced=true idle=12.5% (last 10s: 100.0%) queued=2 restarts=0 faults=0 identities=8 accepted_by_identity=1/0/1/0/0/1/0/0";
|
||||
const STATUS_MISMATCH: &str = "1791138636.597 STATUS 'win-1' [worker]: 90s jobs=500 accepted=3 rejected=0 mismatched=3 extra=0 rate=0.03 blocks/s hash=110.00 MH/s wall (124.20 MH/s inside jobs) now=124.00 MH/s wall (124.30 MH/s inside jobs, 70 jobs, seed walk 0 calls) template_age=0.31s synced=true idle=0.2% (last 10s: 0.1%) queued=2 restarts=1 faults=1 identities=8 accepted_by_identity=1/0/1/0/0/1/0/0";
|
||||
const FAULT_LINE: &str = "1791138640.100 WORKER FAULT the last 10 s ran at 4300000.00 MH/s inside jobs against 3.30 MH/s over this worker's healthy intervals, over 10x: the worker is not running its kernel; restarting the worker (fault 1)";
|
||||
|
||||
#[test]
|
||||
fn parses_status_and_fault_lines() {
|
||||
let s = parse_status(STATUS_OK).unwrap();
|
||||
assert_eq!(s, Status { hash_now: 124.10, mismatched: 0, faults: 0, restarts: 0, synced: true });
|
||||
let m = parse_status(STATUS_MISMATCH).unwrap();
|
||||
assert_eq!((m.mismatched, m.faults, m.restarts), (3, 1, 1));
|
||||
assert_eq!(parse_status(STATUS_ZERO).unwrap().hash_now, 0.0);
|
||||
assert_eq!(parse_status("1791138640.100 worker: ready metal Apple_M5_Max prepare 1"), None);
|
||||
assert_eq!(fault_reason(FAULT_LINE).unwrap(), "the last 10 s ran at 4300000.00 MH/s inside jobs against 3.30 MH/s over this worker's healthy intervals, over 10x: the worker is not running its kernel");
|
||||
assert_eq!(fault_reason(STATUS_OK), None);
|
||||
}
|
||||
|
||||
fn healthy(w: &mut CardWatch, from: f64, to: f64) {
|
||||
let s = parse_status(STATUS_OK).unwrap();
|
||||
let mut t = from;
|
||||
while t <= to {
|
||||
assert_eq!(w.event(t, Event::Status(&s)), Action::None);
|
||||
assert_eq!(w.tick(t, true), Action::None);
|
||||
t += 10.0;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn healthy_miner_is_left_alone() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
healthy(&mut w, 10.0, 3600.0);
|
||||
assert_eq!(w.watchdog_restarts(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_rate_restarts_once_then_faults() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
healthy(&mut w, 10.0, 100.0);
|
||||
let z = parse_status(STATUS_ZERO).unwrap();
|
||||
for t in [110.0, 120.0, 130.0, 140.0, 150.0, 160.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
assert_eq!(w.tick(t, true), Action::None, "under 60 s at {t}");
|
||||
}
|
||||
w.event(170.0, Event::Status(&z));
|
||||
let a = w.tick(170.0, true);
|
||||
assert!(matches!(a, Action::Restart(ref r) if r.contains("hash rate 0 for 60 s")), "{a:?}");
|
||||
// the app restarted it; still zero: faulted, not restarted again
|
||||
w.event(172.0, Event::Started);
|
||||
w.event(174.0, Event::Ready);
|
||||
for t in [180.0, 190.0, 200.0, 210.0, 220.0, 230.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
assert_eq!(w.tick(t, true), Action::None);
|
||||
}
|
||||
w.event(240.0, Event::Status(&z));
|
||||
let a = w.tick(240.0, true);
|
||||
assert!(matches!(a, Action::Fault(ref r) if r.contains("restarted once already")), "{a:?}");
|
||||
assert!(w.faulted().is_some());
|
||||
// faulted stays: no more actions
|
||||
w.event(250.0, Event::Status(&z));
|
||||
assert_eq!(w.tick(260.0, true), Action::None);
|
||||
// the user changed the card's settings: a fresh start
|
||||
w.event(300.0, Event::Reset);
|
||||
assert!(w.faulted().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_rate_only_counts_with_a_synced_node_and_a_ready_worker() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
let z = parse_status(STATUS_ZERO).unwrap();
|
||||
// not ready yet (program loading): no zero timer
|
||||
for t in [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
assert_eq!(w.tick(t, true), Action::None);
|
||||
}
|
||||
w.event(82.0, Event::Ready);
|
||||
// node not synced: no zero timer either
|
||||
for t in [90.0, 100.0, 110.0, 120.0, 130.0, 140.0, 150.0, 160.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
assert_eq!(w.tick(t, false), Action::None);
|
||||
}
|
||||
assert_eq!(w.tick(170.0, true), Action::None, "the timer starts at the first zero status after ready");
|
||||
for t in [180.0, 190.0, 200.0, 210.0, 220.0, 230.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
w.tick(t, true);
|
||||
}
|
||||
assert!(matches!(w.tick(240.0, true), Action::Restart(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_status_for_90s_restarts() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
healthy(&mut w, 10.0, 60.0);
|
||||
// the miner goes silent (a stopped process, a hung RPC)
|
||||
assert_eq!(w.tick(149.0, true), Action::None);
|
||||
let a = w.tick(151.0, true);
|
||||
assert!(matches!(a, Action::Restart(ref r) if r.contains("no status line")), "{a:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_status_from_the_start() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
assert_eq!(w.tick(89.0, true), Action::None);
|
||||
assert!(matches!(w.tick(91.0, true), Action::Restart(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_miners_own_worker_restart_is_not_doubled() {
|
||||
// The gfx1036 fault: the miner prints WORKER FAULT, kills the worker, restarts it 2 s later; STATUS lines in
|
||||
// between say now=0. The app must not restart the miner on top of that.
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
healthy(&mut w, 10.0, 570.0);
|
||||
w.event(580.0, Event::WorkerRestart(&fault_reason(FAULT_LINE).unwrap()));
|
||||
let z = parse_status(STATUS_ZERO).unwrap();
|
||||
for t in [580.0, 590.0, 600.0, 610.0, 620.0, 630.0, 640.0, 650.0, 660.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
assert_eq!(w.tick(t, true), Action::None, "the miner owns the restart at {t}");
|
||||
}
|
||||
w.event(665.0, Event::Ready);
|
||||
healthy(&mut w, 670.0, 900.0);
|
||||
assert_eq!(w.watchdog_restarts(), 0, "no app restart happened");
|
||||
// a worker restart that never comes back: the app steps in after the grace
|
||||
w.event(910.0, Event::WorkerRestart("no job completed for 60 s with 2 queued"));
|
||||
for t in (920..=1080).step_by(10) {
|
||||
w.event(t as f64, Event::Status(&z));
|
||||
assert_eq!(w.tick(t as f64, true), Action::None);
|
||||
}
|
||||
let a = w.tick(1091.0, true);
|
||||
assert!(matches!(a, Action::Restart(ref r) if r.contains("did not come back within 180 s")), "{a:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exit_43_once_then_faulted() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
healthy(&mut w, 10.0, 60.0);
|
||||
w.event(70.0, Event::WorkerRestart("cpu re-check: 3 consecutive mismatches (mismatched=3 in this run): the worker computes a wrong program"));
|
||||
let a = w.event(75.0, Event::Exited(43));
|
||||
assert!(matches!(a, Action::Restart(ref r) if r.contains("gave up") && r.contains("wrong program")), "{a:?}");
|
||||
w.event(80.0, Event::Started);
|
||||
let a = w.event(300.0, Event::Exited(43));
|
||||
assert!(matches!(a, Action::Fault(_)), "{a:?}");
|
||||
// an ordinary crash is the engine's own jittered restart, not the watchdog's
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
assert_eq!(w.event(5.0, Event::Exited(1)), Action::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn five_healthy_minutes_renew_the_budget() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
assert!(matches!(w.tick(100.0, true), Action::Restart(_)));
|
||||
w.event(101.0, Event::Started);
|
||||
w.event(103.0, Event::Ready);
|
||||
healthy(&mut w, 110.0, 420.0);
|
||||
assert_eq!(w.watchdog_restarts(), 0);
|
||||
// a second incident later is again a restart, not a fault
|
||||
assert!(matches!(w.tick(520.0, true), Action::Restart(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_deliberate_stop_is_not_a_fault() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
w.event(30.0, Event::Stopped);
|
||||
assert_eq!(w.tick(500.0, true), Action::None);
|
||||
assert_eq!(w.event(500.0, Event::Exited(0)), Action::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_watch_restarts_a_silent_node_with_growing_delay() {
|
||||
let mut n = NodeWatch::new();
|
||||
assert_eq!(n.tick(100.0, true, 119.0, false), None);
|
||||
assert_eq!(n.tick(100.0, false, 500.0, false), None, "an external node is never restarted");
|
||||
assert_eq!(n.tick(100.0, true, 500.0, true), None, "our block was accepted in the last minute: the node is alive");
|
||||
assert_eq!(n.tick(100.0, true, 120.0, false), Some(3));
|
||||
assert_eq!(n.tick(300.0, true, 120.0, false), Some(12));
|
||||
assert_eq!(n.tick(500.0, true, 120.0, false), Some(48));
|
||||
assert_eq!(n.restarts_in_window(), 3);
|
||||
assert_eq!(n.tick(2000.0, true, 120.0, false), Some(3), "the window passed");
|
||||
}
|
||||
}
|
||||
|
|
@ -613,12 +613,12 @@
|
|||
var tb = $('d-cards');
|
||||
if (!on.length) tb.innerHTML = '<div class="empty">' + (m.cards.length ? 'No card switched on. Open settings to pick one.' : 'No GPU worker. The node runs on its own.') + '</div>';
|
||||
else tb.innerHTML = on.map(function (cd) {
|
||||
var cls = cd.state === 'mining' ? 'mining' : (cd.state === 'failed' || cd.state === 'restarting') ? 'bad' : '';
|
||||
var cls = cd.state === 'mining' ? 'mining' : (cd.state === 'failed' || cd.state === 'restarting' || cd.state === 'faulted') ? 'bad' : '';
|
||||
var word = cd.state === 'restarting' ? ('restart in ' + cd.restart_in_s + ' s') : cd.state;
|
||||
var extra = cd.ids.length ? '<span>id <b>' + esc(cd.ids.join(' ')) + '</b></span>' : '';
|
||||
return '<div class="gpu-tile' + (cd.state === 'mining' ? '' : ' idle') + '"><div class="n">' + esc(cd.name) + ' <span class="kind ' + esc(cd.kind) + '">' + kindWord(cd.kind) + '</span></div>' +
|
||||
'<div class="h">' + cd.hash_now.toFixed(1) + '<span class="unit">MH/s</span></div>' +
|
||||
'<div class="m"><span>blocks <b>' + cd.accepted + '</b>' + (cd.rejected ? ' <span class="dim">/ ' + cd.rejected + ' rejected</span>' : '') + '</span><span>avg <b>' + cd.hash_avg.toFixed(1) + '</b></span><span>identities <b>' + cd.identities + '</b></span>' + extra + '</div>' +
|
||||
'<div class="m"><span>blocks <b>' + cd.accepted + '</b>' + (cd.rejected ? ' <span class="dim">/ ' + cd.rejected + ' rejected</span>' : '') + '</span><span>avg <b>' + cd.hash_avg.toFixed(1) + '</b></span><span>identities <b>' + cd.identities + '</b></span>' + extra + (cd.faults ? '<span class="dim">worker faults <b>' + cd.faults + '</b></span>' : '') + (cd.mismatched ? '<span class="dim">re-check misses <b>' + cd.mismatched + '</b></span>' : '') + '</div>' +
|
||||
telemetryHtml(cd) +
|
||||
'<span class="st ' + cls + '"><i></i>' + esc(word) + (cd.prepared ? ' · next program ready' : '') + '</span>' + (cd.message ? '<div class="msg">' + esc(cd.message) + '</div>' : '') + '</div>';
|
||||
}).join('');
|
||||
|
|
|
|||
|
|
@ -1176,3 +1176,49 @@ not run on a GPU.
|
|||
|
||||
Next: the PC 1 job (ready in `docs/plans/miner-perf.md`); the Mac card paused for a clean absolute table; the
|
||||
Mac app's own worker on this build (its hourly prepare then races by itself and logs the TUNING record).
|
||||
|
||||
## 4 October 2026, miner fault guards and the app watchdog measured against a fake worker (miner-community-lead, app owner)
|
||||
|
||||
Machine: Apple M5 Max, load average 17 to 92 (other agents' builds and runs throughout), everything at nice 19 under
|
||||
`tools/lock/with-lock.sh run`. These are recoveries and seconds, not hash rates: nothing here is a performance number.
|
||||
Private test networks only: one `igneumd` (devnet suffix 9950, ports 29950 to 29953) for the miner scenarios and the
|
||||
app's own private node (suffix 9960, ports 29960 and 29961) for the app scenarios; the live devnet was not touched.
|
||||
Worker: `tools/reliability/fake-worker.mjs`, a stand-in that speaks the serve protocol and misbehaves on command
|
||||
(jobs done in 0.3 ms, 0 hashes, silence, wrong hashes, refused prepares); it never hashes, so no block was found
|
||||
and the fake rate of about 157 MH/s inside jobs is an invented number. Miner: fork branch `miner-reliability`
|
||||
(`aea5ac6d`, `501363e0`, `945153ab`), `igneum-miner` release with `--status-secs 10`. App: `app/igneum-app` at
|
||||
`f39e240` plus the card-message fix, `IGNEUM_APP_STATUS_SECS=10`. Harness: `tools/reliability/run.mjs` and
|
||||
`app-run.mjs`; every scenario states what must and must not appear, and the detectors were shown to fire on the
|
||||
faulting scenarios and stay quiet on the healthy stretches before any number below was kept.
|
||||
|
||||
Miner guards (review round 4 M26, M27, X21), one fresh miner process per scenario:
|
||||
|
||||
| Scenario | What the worker did | Result | Seconds |
|
||||
|---|---|---|---|
|
||||
| slow-first (M26) | 5 s jobs for the first 25 s, then the true rate, 47x faster | one trip, one restart, STATUS lines every 10 s throughout (8 in 85 s, max gap 10.1 s), 4 healthy intervals after it with no further trip; the old guard tripped on every interval and printed nothing | slow start to trip 40.1, trip to worker restarted 2.0 |
|
||||
| fake-fast (the gfx1036 fault) | jobs "done" in 0.3 ms with the full count | the job-time guard fired on the first such job; STATUS with `faults=1` printed after the trip; one restart | trip 0.0 after injection, worker restarted 2.0, ready 2.5, first healthy STATUS 12.5 |
|
||||
| fake-fast-stays | the same fault persists | trips at 0.2, 3.2, 8.7 s with restart delays 2 then 4 s; the third trip in the window ends the miner with exit 43 (`WORKER FAULT 3 guard trips in 10 minutes`) | injection to exit 8.8 |
|
||||
| badfound (X21) | a wrong hash on every found | 3 `WORKER MISMATCH` lines, then `WORKER FAULT cpu re-check: 3 consecutive mismatches`; 3 wrong shares seen, 0 submitted; STATUS shows `mismatched=3`; one restart | first mismatch 0.3, trip 0.5, first healthy STATUS after the trip 2.9 |
|
||||
| silent | stops answering with jobs queued | STATUS lines kept printing during the silence (5; the old loop printed none and never restarted); the stall guard fired at 60 s; one restart | silence to trip 60.1, trip to first healthy STATUS 12.9 |
|
||||
| prepare-flap (M27) | refuses every prepare across epochs of 60 DAA (a 3-thread CPU block producer on genesis bits 0x1f100000) | 5 epochs turned (333 blocks in 5 min); one `PREPARE sent` per epoch, each answered `prepare-failed`, each retry held (`PREPARE held ... within 30 s of the last prepare`) and the epoch turned before a retry was due: 5 sends, 5 held, 0 repeats; before the limiter a refused prepare was re-sent at the next job fill, a pack write and a GPU build each time | smallest gap between sends 50.6 |
|
||||
|
||||
Two defects the harness found on the way, both fixed on the branch: after a guard kill with the job queue not full,
|
||||
the fill loop's `continue` re-ran the failed stdin write forever (140,000 lines in 45 s, no restart, no STATUS;
|
||||
`501363e0`); and the 60 s line-read timeout meant no STATUS line at all while a worker was silent (`945153ab`: the
|
||||
read now waits one status interval). The "first healthy STATUS" figures above are bounded by the 10 s status interval:
|
||||
the worker is back about 2.5 s after a trip, the next STATUS line reports it.
|
||||
|
||||
App watchdog (`src/watchdog.rs`), one engine run, the fake worker as the Metal worker, steps in order:
|
||||
|
||||
| Step | What happened | Result | Seconds |
|
||||
|---|---|---|---|
|
||||
| own-restart | the worker reports jobs done in 0.3 ms | the miner's guard restarted the worker; the card showed the fault and `worker faults 1`; the app did NOT restart the miner (same pid, app restarts 0) | fault on the card 1.0 after injection, mining again 9.1 after the fault |
|
||||
| zero-once | jobs complete with 0 hashes | `watchdog: hash rate 0 for 60 s while the node is synced`; one app restart; mining on the new miner process | zero to restart 79.6 (60 s rule plus the status interval and the quit grace), restart to mining 11.3, zero to mining 91.0 |
|
||||
| zero-faulted | the same again inside five minutes | card `faulted: hash rate 0 for 60 s while the node is synced (restarted once already)`; 45 s later still faulted, no miner process, no further restart; the node kept running and the app stayed up; `resume` cleared it and mining resumed | zero to faulted 75.4 |
|
||||
| no-status | the miner process stopped with SIGSTOP | `watchdog: no status line from the miner for 90 s`; the stopped process killed; mining on a new process | quiet to restart 90.4, restart to mining 11.0 |
|
||||
| node-silent | the node process stopped with SIGSTOP | the app restarted the node in-process (the remote-job restart kind), the new node synced, mining resumed | quiet to restart 150.7 (120 s rule plus the 30 s terminate grace on a process that cannot answer SIGTERM), restart to synced 7.2, quiet to mining 160.0 |
|
||||
|
||||
Not measured: any real GPU. The job-time and interval guards, exit 43 and the app's faulted card have not run on an
|
||||
RTX 5090, the gfx1036 or a Metal card; the first real run is owed from the fleet logs. The "no status" rule has not
|
||||
been tried against a hung RPC (only a stopped process). Unit tests: `cargo test -p igneum-miner -- guard` (7) and
|
||||
`cargo test -- watchdog` in `app/igneum-app` (11), both replaying recorded STATUS and WORKER FAULT lines.
|
||||
|
|
|
|||
|
|
@ -1730,7 +1730,9 @@ Evidence: the files above. Experiment: `igneum-miner` with `IGNEUM_POW_DAY_MS=14
|
|||
### M26. The interval fault guard freezes its baseline and loops
|
||||
"On a trip you skip the STATUS print, so the baseline it would have updated stays frozen, and you roll the counters back to it. Any healthy rate over ten times a slow first interval trips again every interval, forever, with no STATUS line and no `faults=` for the app to read."
|
||||
|
||||
Status: Open (4 October 2026).
|
||||
Status: Fixed (4 October 2026, evening), fork commits `aea5ac6d`, `501363e0` and `945153ab` on `miner-reliability` (`igneum/miner/src/guard.rs`; the second fixes a fill loop that spun forever after a guard kill, found by the test network). Replaced: Open (4 October 2026).
|
||||
|
||||
Fix: `IntervalGuard` builds its baseline from the healthy intervals of the current worker process (a moving average, two intervals before it can trip) and forgets it when the worker restarts; the STATUS line is printed on a trip, with `faults=`; `RestartPolicy` restarts a guard-killed worker after 2 s, doubling per trip inside ten minutes, and the third trip ends the miner with exit 43 so the app shows a faulted card instead of a loop. The 60 s no-line timeout no longer skips the guards and the STATUS line. Measured against a fake worker on a private test network: `docs/bench-log.md`, "4 October 2026, miner fault guards and the app watchdog measured against a fake worker".
|
||||
|
||||
Answer: Correct. `igneum/miner/src/main.rs:1404-1418` with the update at `:1452-1454` skipped by `continue`; the restart has no cap and no growing back-off (`:1213-1216`). A slow first interval (a game on the GPU, a foreground self-heal build on a slow card) is enough. Fix: update the baseline on a trip, or compare to the previous interval; cap restarts with a growing back-off. Review id R4.2.1.
|
||||
|
||||
|
|
@ -1739,7 +1741,7 @@ Evidence: the file above. Experiment: `--status-secs 10` with a GPU stress tool
|
|||
### M27. A flapping node makes the worker rebuild once per template
|
||||
"A prepare goes out whenever the wanted pair differs from the prepared one. No count, no interval, no once-per-epoch. Each one writes a pack on the CPU with the job loop stalled and costs the worker a full build; `prepare-failed` resends on the next fill."
|
||||
|
||||
Status: Open (4 October 2026).
|
||||
Status: Fixed (4 October 2026, evening), fork commit `aea5ac6d` on `miner-reliability` (`guard::PrepareLimiter`): one `prepare` per pair per epoch, one retry after `prepare-failed`, none within 30 s of the last; a held prepare is printed once (`PREPARE held ...`). Measured with a worker that refused every prepare across three epochs: `docs/bench-log.md`, the same entry as M26. Replaced: Open (4 October 2026).
|
||||
|
||||
Answer: Correct. `igneum/miner/src/main.rs:1113-1165, 1337-1339`; `worker.cpp:644-658`; `host.c:1208-1225`. Estimated loss 30 to 60 percent against a node that alternates seeds per template; a stale home node on a fork is the realistic trigger. Fix: at most one prepare per pair per epoch and none within 30 s of the last. Review id R4.2.2.
|
||||
|
||||
|
|
@ -1757,7 +1759,7 @@ Evidence: the files above. Experiment: a tampered pack with matching vectors mus
|
|||
### X21. A wrong program burns power with a green rate
|
||||
"The CPU re-check counts mismatches and does nothing: no threshold, no stop. The app reads `hash`, `now`, `template_age` and `synced` from STATUS and nothing else, so `mismatched=` and `WORKER FAULT` never reach the card."
|
||||
|
||||
Status: Open (4 October 2026).
|
||||
Status: Fixed (4 October 2026, evening), fork commit `aea5ac6d` (`guard::MismatchGuard`: three consecutive CPU re-check mismatches kill the worker, `WORKER FAULT cpu re-check ...`) and app commit `f39e240` on `miner-reliability` (`app/igneum-app/src/watchdog.rs`: the app reads `mismatched=`, `faults=` and the `WORKER FAULT` lines, shows them on the card, and its own watchdog restarts a miner once for no status in 90 s or a zero rate for 60 s while synced, then marks the card faulted; a silent node is restarted in-process). Measured: `docs/bench-log.md`, the same entry as M26. Replaced: Open (4 October 2026).
|
||||
|
||||
Answer: Correct. `igneum/miner/src/main.rs:1250-1261`; `app/igneum-app/src/engine.rs:~1762-1780` (zero matches for either string). The PowerShell launcher matches them (`igneum-common.ps1:843`), which is what README.txt and TEST.md describe. Fix: stop the worker after 3 consecutive mismatches and show it on the card; the app reads both fields. Review id R4.2.3.
|
||||
|
||||
|
|
|
|||
|
|
@ -109,6 +109,7 @@ p{margin:0;color:var(--ink-2);max-width:52ch}
|
|||
<a href="/litepaper" data-nav="litepaper">Litepaper</a>
|
||||
<a href="/live" data-nav="live">Live devnet</a>
|
||||
<a href="/bench" data-nav="bench">Engineering log</a>
|
||||
<a href="/miners" data-nav="miners">Miners</a>
|
||||
<a href="/evidence" data-nav="evidence">Evidence</a>
|
||||
<a href="https://github.com/igneum-network/spec" rel="noopener">GitHub</a>
|
||||
<a href="/#mine" class="cta">Get the miner</a>
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -256,4 +256,32 @@ for (const [file, active] of PAGES) {
|
|||
writeFileSync(p, html);
|
||||
built.push(file);
|
||||
}
|
||||
|
||||
// the public bench table (/miners): one row per card, generator version and miner version, from site/miner-bench.json
|
||||
// (the bench-log numbers that exist today, and rows later jobs append); rendered through the same scrub as /bench
|
||||
{
|
||||
const bj = JSON.parse(readFileSync(join(here, 'miner-bench.json'), 'utf8'));
|
||||
const rows = bj.rows.slice().sort((a, b) => (a.card < b.card ? -1 : a.card > b.card ? 1 : a.generator < b.generator ? -1 : a.generator > b.generator ? 1 : b.mh_s - a.mh_s));
|
||||
const fmt = (n) => Number(n).toLocaleString('en-GB', { maximumFractionDigits: 1 });
|
||||
const cell = (r) => [
|
||||
r.card, r.generator, fmt(r.mh_s), r.mh_per_w == null ? 'not measured' : fmt(r.mh_per_w), r.miner, r.date, r.source, r.by + (r.note ? '. ' + r.note : ''),
|
||||
];
|
||||
const table = '<div class="tbl"><table><thead><tr>' + ['Card', 'Generator', 'Best MH/s', 'MH per watt', 'Miner', 'Date', 'Source', 'Who measured it'].map(h => `<th>${h}</th>`).join('') + '</tr></thead><tbody>' +
|
||||
rows.map(r => '<tr>' + cell(r).map(c => `<td>${esc(String(c))}</td>`).join('') + '</tr>').join('') + '</tbody></table></div>';
|
||||
const body = scrubBench([
|
||||
'<h2 id="table">The table</h2>',
|
||||
'<p>One row per card, generator version and miner version. The rate is the best one measured. Integrated GPUs are not listed. Prototype rows are bench numbers from before the devnet and say so in the miner column.</p>',
|
||||
table,
|
||||
'<h2 id="how">How a row gets here</h2>',
|
||||
'<p>Every row names the engineering log entry or the job it came from. "Measured by the team" means our own hardware and our own log. "Reported by the fleet" means a machine we do not own, read from the status lines its miner uploads.</p>',
|
||||
'<p>MH per watt needs the card\'s power draw during the run. The app reads it on NVIDIA cards through the driver. Rows get the figure when a run records it.</p>',
|
||||
'<p>There is no other Igneum miner to compare with yet, so this table compares cards, not miners.</p>',
|
||||
`<p>Rows: ${rows.length}. Source file: <code>site/miner-bench.json</code> in the repository.</p>`,
|
||||
].join('\n'));
|
||||
const toc = [{ lvl: 2, t: 'The table', id: 'table' }, { lvl: 2, t: 'How a row gets here', id: 'how' }];
|
||||
writeFileSync(join(here, 'miners.html'), page('Igneum GPU bench table', 'Measured Igneum hash rates per GPU: card, generator version, best MH/s, MH per watt where measured, miner version, date and the log entry each number came from.', body, toc,
|
||||
'Measured hash rates per card on the Igneum lottery hash, with the generator version, the miner version, the date and the log entry behind each number.',
|
||||
{ path: '/miners', heading: 'GPU bench table', active: 'miners' }));
|
||||
built.push('miners.html (' + rows.length + ' rows)');
|
||||
}
|
||||
console.log('built: ' + built.join(', '));
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ code{font-family:var(--f-mono);font-size:.92em;background:var(--obsidian);paddin
|
|||
<a href="/litepaper" data-nav="litepaper">Litepaper</a>
|
||||
<a href="/live" data-nav="live">Live devnet</a>
|
||||
<a href="/bench" data-nav="bench">Engineering log</a>
|
||||
<a href="/miners" data-nav="miners">Miners</a>
|
||||
<a href="/evidence" data-nav="evidence" aria-current="page">Evidence</a>
|
||||
<a href="https://github.com/igneum-network/spec" rel="noopener">GitHub</a>
|
||||
<a href="/#mine" class="cta">Get the miner</a>
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -50,6 +50,11 @@
|
|||
}
|
||||
],
|
||||
"log": [
|
||||
{
|
||||
"date": "2026-10-04",
|
||||
"text": "Miner fault guards and the app watchdog measured against a fake worker",
|
||||
"short": "Miner fault guards and the app watchdog measured against a fake worker"
|
||||
},
|
||||
{
|
||||
"date": "2026-10-04",
|
||||
"text": "Miner performance: variant racing",
|
||||
|
|
@ -190,6 +195,21 @@
|
|||
"text": "Devnet-v4 integration: nine branches merged, 3-node test network on the merged node, Windows cross-build",
|
||||
"short": "Devnet v4: nine branches merged into one node"
|
||||
},
|
||||
{
|
||||
"date": "2026-10-03",
|
||||
"text": "AMD gfx1036 , AMD OpenCL 2.1 driver 3652.0",
|
||||
"short": "AMD integrated GPU runs the hash through OpenCL"
|
||||
},
|
||||
{
|
||||
"date": "2026-10-03",
|
||||
"text": "RTX 5090 through NVIDIA OpenCL",
|
||||
"short": "RTX 5090 through NVIDIA OpenCL"
|
||||
},
|
||||
{
|
||||
"date": "2026-10-03",
|
||||
"text": "Igneum-node devnet v0: 3-node igneum-devnet at 1 BPS with the 80/20 coinbase and vote_key_hash",
|
||||
"short": "Devnet v0: three nodes at one block per second"
|
||||
},
|
||||
{
|
||||
"date": "2026-10-03",
|
||||
"text": "RTX 5090 first run",
|
||||
|
|
@ -229,26 +249,6 @@
|
|||
"date": "2026-10-03",
|
||||
"text": "Windows node package: igneumd cross-compiled for x86_64-pc-windows-gnu, two-peer sync test",
|
||||
"short": "Windows node package: cross-compiled, two-peer sync"
|
||||
},
|
||||
{
|
||||
"date": "2026-10-03",
|
||||
"text": "R3.26 / M15: PoW checked after the cheap checks, cache-build cap, attack before and after",
|
||||
"short": "Cache-build attack closed: 10.6 s of rebuilds to 14 ms"
|
||||
},
|
||||
{
|
||||
"date": "2026-10-03",
|
||||
"text": "Difficulty controller: devnet record, simulator, Igneum dual-lane rule, 3-node CPU test network",
|
||||
"short": "Igneum dual-lane difficulty rule built and simulated"
|
||||
},
|
||||
{
|
||||
"date": "2026-10-03",
|
||||
"text": "First devnet blocks on the real lottery hash: CPU, then Metal GPU, three worker implementations",
|
||||
"short": "First devnet blocks on the real lottery hash"
|
||||
},
|
||||
{
|
||||
"date": "2026-10-03",
|
||||
"text": "Proto-opencl: OpenCL path built and proven without AMD silicon",
|
||||
"short": "OpenCL worker built and proven without AMD silicon"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -226,6 +226,7 @@ body.all .pager{display:none}
|
|||
<a href="/litepaper" data-nav="litepaper" aria-current="page">Litepaper</a>
|
||||
<a href="/live" data-nav="live">Live devnet</a>
|
||||
<a href="/bench" data-nav="bench">Engineering log</a>
|
||||
<a href="/miners" data-nav="miners">Miners</a>
|
||||
<a href="/evidence" data-nav="evidence">Evidence</a>
|
||||
<a href="https://github.com/igneum-network/spec" rel="noopener">GitHub</a>
|
||||
<a href="/#mine" class="cta">Get the miner</a>
|
||||
|
|
|
|||
|
|
@ -191,6 +191,7 @@ main{padding-bottom:var(--sec)}
|
|||
<a href="/litepaper" data-nav="litepaper">Litepaper</a>
|
||||
<a href="/live" data-nav="live" aria-current="page">Live devnet</a>
|
||||
<a href="/bench" data-nav="bench">Engineering log</a>
|
||||
<a href="/miners" data-nav="miners">Miners</a>
|
||||
<a href="/evidence" data-nav="evidence">Evidence</a>
|
||||
<a href="https://github.com/igneum-network/spec" rel="noopener">GitHub</a>
|
||||
<a href="/#mine" class="cta">Get the miner</a>
|
||||
|
|
|
|||
71
site/miner-bench.json
Normal file
71
site/miner-bench.json
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
{
|
||||
"_about": "Rows of the public bench table at /miners (site/build.mjs). One row per card, generator version and miner version: the best measured rate. Later jobs append rows. Fields: card, generator (v1 | v2), mh_s (best measured MH/s), mh_per_w (MH per watt, null when the power was not measured), miner (the miner or package version), date (YYYY-MM-DD), source (the docs/bench-log.md heading, or the job id), by ('measured by the team' | 'reported by the fleet'), note (short, optional). No machine names, no addresses, no owner names: the build scrubs the page and fails on any that survive.",
|
||||
"rows": [
|
||||
{
|
||||
"card": "NVIDIA RTX 5090 (32 GB)",
|
||||
"generator": "v1",
|
||||
"mh_s": 228.95,
|
||||
"mh_per_w": null,
|
||||
"miner": "proto-cuda bench (prototype, not mining)",
|
||||
"date": "2026-10-03",
|
||||
"source": "bench log: 3 October 2026, RTX 5090, memory-hard dataset (pack igneum-genesis-mh)",
|
||||
"by": "measured by the team",
|
||||
"note": "genesis program, 104 loads per hash, 1 GiB dataset"
|
||||
},
|
||||
{
|
||||
"card": "NVIDIA RTX 5090 (32 GB)",
|
||||
"generator": "v1",
|
||||
"mh_s": 185.3,
|
||||
"mh_per_w": null,
|
||||
"miner": "proto-cuda bench (prototype, not mining)",
|
||||
"date": "2026-10-03",
|
||||
"source": "bench log: 3 October 2026, RTX 5090 first run, dataset sweep and second program",
|
||||
"by": "measured by the team",
|
||||
"note": "hourly program, 128 loads per hash, 1 GiB dataset"
|
||||
},
|
||||
{
|
||||
"card": "NVIDIA RTX 5090 (32 GB)",
|
||||
"generator": "v2",
|
||||
"mh_s": 124.2,
|
||||
"mh_per_w": null,
|
||||
"miner": "Igneum Miner 0.3.0 package, prebuilt NVRTC worker",
|
||||
"date": "2026-10-04",
|
||||
"source": "bench log: 4 October 2026, the gfx1036 worker fault and what the Mac could and could not reproduce",
|
||||
"by": "measured by the team",
|
||||
"note": "live devnet v4, 128 loads per hash, CPU re-check clean, 0 rejected"
|
||||
},
|
||||
{
|
||||
"card": "Apple M5 Max (40 GPU cores, Metal)",
|
||||
"generator": "v1",
|
||||
"mh_s": 45.2,
|
||||
"mh_per_w": null,
|
||||
"miner": "proto-metal bench (prototype, not mining)",
|
||||
"date": "2026-10-03",
|
||||
"source": "bench log: 3 October 2026, RTX 5090 first run (the Apple row of the same table)",
|
||||
"by": "measured by the team",
|
||||
"note": "genesis program, 1 GiB dataset"
|
||||
},
|
||||
{
|
||||
"card": "Apple M5 Max (40 GPU cores, Metal)",
|
||||
"generator": "v2",
|
||||
"mh_s": 26.7,
|
||||
"mh_per_w": null,
|
||||
"miner": "igneum-miner devnet v4, Metal worker with prepare",
|
||||
"date": "2026-10-04",
|
||||
"source": "bench log: 4 October 2026, first hourly program swap on the live devnet: compile-ahead, no pause, two cards",
|
||||
"by": "measured by the team",
|
||||
"note": "live devnet v4, unbroken through the hour boundary"
|
||||
},
|
||||
{
|
||||
"card": "Apple silicon laptop (model not reported)",
|
||||
"generator": "v2",
|
||||
"mh_s": 24.3,
|
||||
"mh_per_w": null,
|
||||
"miner": "Igneum Miner 0.3.1 (DMG)",
|
||||
"date": "2026-10-04",
|
||||
"source": "bench log: 4 October 2026, first outside machine on the devnet: an Apple silicon laptop through the Igneum Miner app",
|
||||
"by": "reported by the fleet",
|
||||
"note": "21.0 MH/s average over 7 minutes, 24.3 MH/s at the moment of the report, 33 accepted blocks"
|
||||
}
|
||||
]
|
||||
}
|
||||
223
site/miners.html
Normal file
223
site/miners.html
Normal file
|
|
@ -0,0 +1,223 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
||||
<title>Igneum GPU bench table</title>
|
||||
<meta name="description" content="Measured Igneum hash rates per GPU: card, generator version, best MH/s, MH per watt where measured, miner version, date and the log entry each number came from.">
|
||||
<link rel="canonical" href="https://igneum.network/miners">
|
||||
<meta name="theme-color" content="#0C0C0E">
|
||||
<meta property="og:type" content="website">
|
||||
<meta property="og:site_name" content="Igneum">
|
||||
<meta property="og:title" content="Igneum GPU bench table">
|
||||
<meta property="og:description" content="Measured Igneum hash rates per GPU: card, generator version, best MH/s, MH per watt where measured, miner version, date and the log entry each number came from.">
|
||||
<meta property="og:url" content="https://igneum.network/miners">
|
||||
<meta property="og:image" content="https://igneum.network/og.png?v=3">
|
||||
<meta property="og:image:width" content="1200">
|
||||
<meta property="og:image:height" content="630">
|
||||
<meta property="og:image:alt" content="Igneum. Mined by GPUs. Proven by fire.">
|
||||
<meta name="twitter:card" content="summary_large_image">
|
||||
<meta name="twitter:title" content="Igneum GPU bench table">
|
||||
<meta name="twitter:description" content="Measured Igneum hash rates per GPU: card, generator version, best MH/s, MH per watt where measured, miner version, date and the log entry each number came from.">
|
||||
<meta name="twitter:image" content="https://igneum.network/og.png?v=3">
|
||||
<meta name="twitter:image:alt" content="Igneum. Mined by GPUs. Proven by fire.">
|
||||
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1024 1024'%3E%3Crect width='1024' height='1024' fill='%230C0C0E'/%3E%3Cg transform='translate(166.95 166.95) scale(6.901)'%3E%3Cpolygon points='50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34' fill='%23F2541B'/%3E%3Cpolygon points='50,42 59,58 50,82 41,58' fill='%230C0C0E'/%3E%3C/g%3E%3C/svg%3E" type="image/svg+xml">
|
||||
<link rel="icon" href="/favicon.ico" sizes="48x48">
|
||||
<link rel="icon" href="/favicon-32.png" type="image/png" sizes="32x32">
|
||||
<link rel="icon" href="/icon-192.png" type="image/png" sizes="192x192">
|
||||
<link rel="icon" href="/icon-512.png" type="image/png" sizes="512x512">
|
||||
<link rel="apple-touch-icon" href="/apple-touch-icon.png" sizes="180x180">
|
||||
<link rel="manifest" href="/site.webmanifest">
|
||||
<!-- head:start -->
|
||||
<link rel="preload" href="/fonts/unbounded-900.woff2" as="font" type="font/woff2" crossorigin>
|
||||
<link rel="preload" href="/fonts/plex-sans-400.woff2" as="font" type="font/woff2" crossorigin>
|
||||
<style>
|
||||
/* Shared by every page. Source: site/partials/head.html, injected by site/build.mjs between the head markers. Edit the partial, not the page. */
|
||||
/* Fonts, self-hosted (latin subsets, OFL): Unbounded 500/700/900, IBM Plex Sans 400/500/600, IBM Plex Mono 400/500. Fallbacks carry size and ascent overrides so the swap does not move the layout. */
|
||||
@font-face{font-family:'Unbounded';font-style:normal;font-weight:500;font-display:swap;src:url(/fonts/unbounded-500.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||||
@font-face{font-family:'Unbounded';font-style:normal;font-weight:700;font-display:swap;src:url(/fonts/unbounded-700.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||||
@font-face{font-family:'Unbounded';font-style:normal;font-weight:900;font-display:swap;src:url(/fonts/unbounded-900.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||||
@font-face{font-family:'IBM Plex Sans';font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/plex-sans-400.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||||
@font-face{font-family:'IBM Plex Sans';font-style:normal;font-weight:500;font-display:swap;src:url(/fonts/plex-sans-500.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||||
@font-face{font-family:'IBM Plex Sans';font-style:normal;font-weight:600;font-display:swap;src:url(/fonts/plex-sans-600.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||||
@font-face{font-family:'IBM Plex Mono';font-style:normal;font-weight:400;font-display:swap;src:url(/fonts/plex-mono-400.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||||
@font-face{font-family:'IBM Plex Mono';font-style:normal;font-weight:500;font-display:swap;src:url(/fonts/plex-mono-500.woff2) format('woff2');unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
|
||||
@font-face{font-family:'Unbounded Fallback';src:local('Arial Black'),local('Arial-Black'),local('Impact');size-adjust:114%;ascent-override:87.5%;descent-override:21.5%;line-gap-override:0%}
|
||||
@font-face{font-family:'Plex Sans Fallback';src:local('Arial'),local('Helvetica Neue'),local('Helvetica');size-adjust:100.5%;ascent-override:102%;descent-override:27.4%;line-gap-override:0%}
|
||||
@font-face{font-family:'Plex Mono Fallback';src:local('Courier New'),local('Menlo');size-adjust:100%;ascent-override:102.5%;descent-override:27.5%;line-gap-override:0%}
|
||||
:root{--f-sans:'IBM Plex Sans','Plex Sans Fallback',system-ui,-apple-system,sans-serif;--f-display:'Unbounded','Unbounded Fallback',sans-serif;--f-mono:'IBM Plex Mono','Plex Mono Fallback',ui-monospace,Menlo,monospace;
|
||||
/* site chrome tokens (nav, footer, skip link). Dark by default; the litepaper sets light values and resets these in dark mode */
|
||||
--nav-h:68px;--ui-bg:rgba(12,12,14,.84);--ui-menu:#0C0C0E;--ui-ink:#F4F1EC;--ui-ink-2:#C9C7C2;--ui-ash:#9A9A9E;--ui-line:#2A2A30;--ui-line-2:#3A3A42;--ui-accent:#F2541B;--ui-accent-ink:#0C0C0E;--ui-hot:#FFB35C;--ui-hover:#FF6A2B;--ui-tint:rgba(242,84,27,.12)}
|
||||
html{-webkit-text-size-adjust:100%}
|
||||
h1,h2,h3{text-wrap:balance}
|
||||
p,li,dd,figcaption{text-wrap:pretty}
|
||||
a:focus-visible,button:focus-visible,summary:focus-visible,[tabindex]:focus-visible{outline:2px solid var(--ui-accent);outline-offset:3px;border-radius:6px}
|
||||
.skip{position:absolute;left:12px;top:-80px;z-index:50;background:var(--ui-accent);color:var(--ui-accent-ink);padding:10px 14px;border-radius:10px;font:600 15px/1.2 var(--f-sans);text-decoration:none}
|
||||
.skip:focus{top:12px}
|
||||
/* nav: one bar on every page, the mark in its black square, five links and the miner button, a menu under 901 px */
|
||||
.nav{position:sticky;top:0;z-index:20;background:var(--ui-bg);-webkit-backdrop-filter:blur(12px);backdrop-filter:blur(12px);border-bottom:1px solid var(--ui-line);color:var(--ui-ink)}
|
||||
.nav .wrap{display:flex;align-items:center;justify-content:space-between;gap:16px;min-height:var(--nav-h)}
|
||||
.nav a{color:inherit;text-decoration:none}
|
||||
.brand{display:inline-flex;align-items:center;gap:10px;color:var(--ui-ink);text-decoration:none;flex:0 0 auto}
|
||||
.brand .mark{display:block;flex:0 0 auto}
|
||||
.brand .word{font-family:var(--f-display);font-weight:900;font-size:20px;letter-spacing:.06em;line-height:1}
|
||||
.nav .links{display:flex;align-items:center;gap:clamp(14px,2vw,26px);font-size:15px;font-weight:500}
|
||||
.nav .links a{color:var(--ui-ink-2);padding:4px 0;border-bottom:2px solid transparent;transition:color .15s ease,border-color .15s ease}
|
||||
.nav .links a:hover,.nav .brand:hover{color:var(--ui-hot);text-decoration:none}
|
||||
.nav .brand:hover{color:var(--ui-ink)}
|
||||
.nav .links a[aria-current="page"]{color:var(--ui-ink);border-bottom-color:var(--ui-accent)}
|
||||
.nav .links a.cta{display:inline-flex;align-items:center;justify-content:center;min-height:40px;padding:8px 16px;border-radius:10px;font-weight:600;background:var(--ui-accent);color:var(--ui-accent-ink);border:1px solid var(--ui-accent);transition:background .15s ease,transform .15s ease}
|
||||
.nav .links a.cta:hover{background:var(--ui-hover);border-color:var(--ui-hover);color:var(--ui-accent-ink);transform:translateY(-1px)}
|
||||
.burger{display:none;background:none;border:1px solid var(--ui-line-2);color:var(--ui-ink);border-radius:10px;width:44px;height:44px;align-items:center;justify-content:center;cursor:pointer;padding:0;flex:0 0 auto}
|
||||
.burger .x{display:none}.burger[aria-expanded="true"] .x{display:block}.burger[aria-expanded="true"] .bars{display:none}
|
||||
@media (max-width:900px){
|
||||
.nav .links{display:none;position:absolute;left:0;right:0;top:100%;background:var(--ui-menu);border-bottom:1px solid var(--ui-line);box-shadow:0 24px 40px rgba(0,0,0,.35);padding:10px var(--gutter,16px) 18px;flex-direction:column;align-items:stretch;gap:2px}
|
||||
.nav .links a{padding:13px 10px;border-radius:10px;border-bottom:0;font-size:16px}
|
||||
.nav .links a[aria-current="page"]{background:var(--ui-tint)}
|
||||
.nav .links a.cta{margin-top:8px;min-height:48px}
|
||||
.nav .links.open{display:flex}
|
||||
.burger{display:inline-flex}
|
||||
}
|
||||
/* footer, the same on every page */
|
||||
.foot{border-top:1px solid var(--ui-line);color:var(--ui-ink);margin-top:var(--sec,64px)}
|
||||
.foot .wrap{padding-block:48px 32px}
|
||||
.foot-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(min(100%,200px),1fr));gap:32px}
|
||||
.foot-brand{display:flex;flex-direction:column;gap:12px;max-width:34ch}
|
||||
.foot-brand .brand{align-self:flex-start}.foot-brand .word{font-size:16px}
|
||||
.foot-brand p{color:var(--ui-ash);font-size:14px;margin:0}
|
||||
.foot-col{display:flex;flex-direction:column;gap:10px;font-size:15px}
|
||||
.foot-col .eyebrow{font-family:var(--f-mono);font-size:12px;letter-spacing:.18em;text-transform:uppercase;color:var(--ui-ash);margin-bottom:4px}
|
||||
.foot-col a{color:var(--ui-ink-2);text-decoration:none;display:inline-flex;align-items:center;gap:6px;align-self:flex-start}
|
||||
.foot-col a:hover{color:var(--ui-hot);text-decoration:none}
|
||||
.foot-base{display:flex;flex-wrap:wrap;justify-content:space-between;gap:8px 24px;margin-top:36px;padding-top:20px;border-top:1px solid var(--ui-line);font-size:13px;color:var(--ui-ash)}
|
||||
.foot-base span{text-wrap:balance}.foot-base .mono{font-family:var(--f-mono)}
|
||||
@media (prefers-reduced-motion:reduce){*,*::before,*::after{animation-duration:.01ms!important;animation-iteration-count:1!important;transition-duration:.01ms!important;scroll-behavior:auto!important}}
|
||||
</style>
|
||||
<!-- head:end -->
|
||||
<style>
|
||||
:root{--obsidian:#0C0C0E;--graphite:#16161A;--line:#2A2A30;--ember:#F2541B;--molten:#FFB35C;--bone:#F4F1EC;--ash:#9A9A9E;--ink-2:#C9C7C2;
|
||||
/* one system for every page: container, gutter, section rhythm, card and tile boxes, heading scale (same tokens as the homepage) */
|
||||
--max:1200px;--gutter:clamp(16px,4vw,32px);--sec:clamp(56px,8vw,96px);--head:clamp(40px,6vw,64px);
|
||||
--card-pad:clamp(18px,3vw,28px);--card-r:18px;--tile-pad:18px 20px;--tile-r:14px;--gap:24px;--gap-tile:12px;
|
||||
--fs-h1:clamp(32px,5.5vw,56px);--fs-h2:clamp(28px,4.2vw,44px);--fs-h3:clamp(18px,2vw,22px);--fs-tile:clamp(20px,2.2vw,26px)}
|
||||
*{box-sizing:border-box}body{margin:0;background:var(--obsidian);color:var(--bone);font-family:var(--f-sans);font-size:16px;line-height:1.6;padding:0;-webkit-font-smoothing:antialiased}
|
||||
a{color:var(--ember);text-decoration:none}a:hover{text-decoration:underline}
|
||||
[id]{scroll-margin-top:calc(var(--nav-h) + 16px)}
|
||||
.wrap{max-width:var(--max);margin:0 auto;padding-inline:var(--gutter)}
|
||||
.eyebrow{font-family:var(--f-mono);font-size:12px;letter-spacing:.18em;text-transform:uppercase;color:var(--ash)}
|
||||
h1{font-family:var(--f-display);font-weight:900;font-size:var(--fs-h1);line-height:1.05;margin:12px 0 12px}
|
||||
.head{padding-top:var(--head)}
|
||||
.note{color:var(--ash);font-size:15px;max-width:72ch;margin:0 0 32px}
|
||||
.layout{display:grid;grid-template-columns:minmax(0,1fr);gap:32px}
|
||||
@media(min-width:960px){.layout{grid-template-columns:260px minmax(0,1fr);gap:56px}}
|
||||
.toc{align-self:start;font-size:14px;border:1px solid var(--line);border-radius:var(--tile-r);background:var(--graphite);padding:8px}
|
||||
.toc .lbl{font-family:var(--f-mono);font-size:11px;letter-spacing:.12em;text-transform:uppercase;color:var(--ash);padding:6px 10px 8px}
|
||||
.toc ol{list-style:none;margin:0;padding:0;display:flex;flex-direction:column;gap:1px;counter-reset:s;max-height:40vh;overflow:auto;overscroll-behavior:contain;scrollbar-width:thin}
|
||||
@media(min-width:960px){.toc{position:sticky;top:calc(var(--nav-h) + 20px)}.toc ol{max-height:calc(100vh - var(--nav-h) - 96px)}}
|
||||
.toc a{display:flex;gap:10px;padding:7px 10px;border-radius:8px;color:var(--ink-2);counter-increment:s;line-height:1.35}
|
||||
.toc a::before{content:counter(s,decimal-leading-zero);font-family:var(--f-mono);font-size:11px;color:var(--ash);min-width:20px;padding-top:2px}
|
||||
.toc a:hover{background:var(--obsidian);text-decoration:none;color:var(--bone)}
|
||||
article{max-width:78ch;min-width:0}
|
||||
article h1{display:none}
|
||||
article h2{font-family:var(--f-display);font-weight:700;font-size:var(--fs-h3);line-height:1.3;margin:44px 0 12px;padding-top:24px;border-top:1px solid var(--line)}
|
||||
article h2:first-of-type{margin-top:0;padding-top:0;border-top:0}
|
||||
article h3{font-family:var(--f-display);font-weight:500;font-size:16px;margin:26px 0 8px}
|
||||
article h4{font-size:15px;margin:20px 0 6px;color:var(--molten)}
|
||||
article p{margin:0 0 14px}article ul,article ol{margin:0 0 14px;padding-left:22px}article li{margin-bottom:6px}
|
||||
article p,article li,article h2,article h3{overflow-wrap:anywhere}
|
||||
code{font-family:var(--f-mono);font-size:.92em;background:var(--graphite);padding:1px 5px;border-radius:4px;overflow-wrap:anywhere}
|
||||
pre{margin:0 0 16px;padding:16px 18px;background:var(--graphite);border-radius:var(--card-r);font-family:var(--f-mono);font-size:13px;line-height:1.55;overflow-x:auto;-webkit-overflow-scrolling:touch}
|
||||
pre code{background:none;padding:0;font-size:inherit}
|
||||
.tbl{overflow-x:auto;margin:18px 0 24px;border:1px solid var(--line);border-radius:var(--card-r);background:var(--graphite);-webkit-overflow-scrolling:touch}
|
||||
table{border-collapse:collapse;width:100%;min-width:560px;font-size:15px}th,td{padding:12px 14px;text-align:left;vertical-align:top;border-bottom:1px solid var(--line)}
|
||||
th{font-family:var(--f-mono);font-size:12px;letter-spacing:.12em;text-transform:uppercase;color:var(--ash);font-weight:500;background:var(--obsidian)}tr:last-child td{border-bottom:0}
|
||||
.gen{color:var(--ash);font-size:13px;margin:40px 0 0;max-width:72ch}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<!-- nav:start -->
|
||||
<a class="skip" href="#main">Skip to content</a>
|
||||
<nav class="nav" aria-label="Main">
|
||||
<div class="wrap">
|
||||
<a href="/" class="brand" aria-label="Igneum home">
|
||||
<svg class="mark" viewBox="0 0 1024 1024" width="36" height="36" aria-hidden="true"><rect width="1024" height="1024" fill="#0C0C0E"></rect><g transform="translate(166.95 166.95) scale(6.901)"><polygon points="50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34" fill="#F2541B"></polygon><polygon points="50,42 59,58 50,82 41,58" fill="#0C0C0E"></polygon></g></svg>
|
||||
<span class="word">IGNEUM</span>
|
||||
</a>
|
||||
<button class="burger" id="nav-burger" type="button" aria-expanded="false" aria-controls="nav-links" aria-label="Menu"><svg class="bars" viewBox="0 0 24 24" width="22" height="22" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" aria-hidden="true"><path d="M4 7h16M4 12h16M4 17h16"></path></svg><svg class="x" viewBox="0 0 24 24" width="22" height="22" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" aria-hidden="true"><path d="M6 6l12 12M18 6L6 18"></path></svg></button>
|
||||
<div class="links" id="nav-links">
|
||||
<a href="/litepaper" data-nav="litepaper">Litepaper</a>
|
||||
<a href="/live" data-nav="live">Live devnet</a>
|
||||
<a href="/bench" data-nav="bench">Engineering log</a>
|
||||
<a href="/miners" data-nav="miners" aria-current="page">Miners</a>
|
||||
<a href="/evidence" data-nav="evidence">Evidence</a>
|
||||
<a href="https://github.com/igneum-network/spec" rel="noopener">GitHub</a>
|
||||
<a href="/#mine" class="cta">Get the miner</a>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
<script>
|
||||
(function(){
|
||||
// the menu: open on the button, closed by any link, a tap outside, Escape (focus back on the button) or a resize past 900 px
|
||||
var b=document.getElementById('nav-burger'),l=document.getElementById('nav-links');if(!b||!l)return;
|
||||
function set(o){l.classList.toggle('open',o);b.setAttribute('aria-expanded',o?'true':'false');}
|
||||
b.addEventListener('click',function(){set(!l.classList.contains('open'));});
|
||||
l.addEventListener('click',function(e){if(e.target.closest('a'))set(false);});
|
||||
document.addEventListener('click',function(e){if(l.classList.contains('open')&&!l.contains(e.target)&&!b.contains(e.target))set(false);});
|
||||
document.addEventListener('keydown',function(e){if(e.key==='Escape'&&l.classList.contains('open')){set(false);b.focus();}});
|
||||
var mq=window.matchMedia('(min-width:901px)');if(mq.addEventListener)mq.addEventListener('change',function(){set(false);});
|
||||
})();
|
||||
</script>
|
||||
<!-- nav:end -->
|
||||
<main id="main" class="wrap">
|
||||
<div class="head">
|
||||
<div class="eyebrow">2 entries, newest at the bottom</div>
|
||||
<h1>GPU bench table</h1>
|
||||
<p class="note">Measured hash rates per card on the Igneum lottery hash, with the generator version, the miner version, the date and the log entry behind each number.</p>
|
||||
</div>
|
||||
<div class="layout">
|
||||
<nav class="toc" aria-label="Contents"><div class="lbl">Contents</div><ol><li><a href="#table">The table</a></li><li><a href="#how">How a row gets here</a></li></ol></nav>
|
||||
<article><h2 id="table">The table</h2>
|
||||
<p>One row per card, generator version and miner version. The rate is the best one measured. Integrated GPUs are not listed. Prototype rows are bench numbers from before the devnet and say so in the miner column.</p>
|
||||
<div class="tbl"><table><thead><tr><th>Card</th><th>Generator</th><th>Best MH/s</th><th>MH per watt</th><th>Miner</th><th>Date</th><th>Source</th><th>Who measured it</th></tr></thead><tbody><tr><td>Apple M5 Max (40 GPU cores, Metal)</td><td>v1</td><td>45.2</td><td>not measured</td><td>proto-metal bench (prototype, not mining)</td><td>2026-10-03</td><td>bench log: 3 October 2026, RTX 5090 first run (the Apple row of the same table)</td><td>measured by the team. genesis program, 1 GiB dataset</td></tr><tr><td>Apple M5 Max (40 GPU cores, Metal)</td><td>v2</td><td>26.7</td><td>not measured</td><td>igneum-miner devnet v4, Metal worker with prepare</td><td>2026-10-04</td><td>bench log: 4 October 2026, first hourly program swap on the live devnet: compile-ahead, no pause, two cards</td><td>measured by the team. live devnet v4, unbroken through the hour boundary</td></tr><tr><td>Apple silicon laptop (model not reported)</td><td>v2</td><td>24.3</td><td>not measured</td><td>Igneum Miner 0.3.1 (DMG)</td><td>2026-10-04</td><td>bench log: 4 October 2026, first outside machine on the devnet: an Apple silicon laptop through the Igneum Miner app</td><td>reported by the fleet. 21.0 MH/s average over 7 minutes, 24.3 MH/s at the moment of the report, 33 accepted blocks</td></tr><tr><td>NVIDIA RTX 5090 (32 GB)</td><td>v1</td><td>229</td><td>not measured</td><td>proto-cuda bench (prototype, not mining)</td><td>2026-10-03</td><td>bench log: 3 October 2026, RTX 5090, memory-hard dataset (pack igneum-genesis-mh)</td><td>measured by the team. genesis program, 104 loads per hash, 1 GiB dataset</td></tr><tr><td>NVIDIA RTX 5090 (32 GB)</td><td>v1</td><td>185.3</td><td>not measured</td><td>proto-cuda bench (prototype, not mining)</td><td>2026-10-03</td><td>bench log: 3 October 2026, RTX 5090 first run, dataset sweep and second program</td><td>measured by the team. hourly program, 128 loads per hash, 1 GiB dataset</td></tr><tr><td>NVIDIA RTX 5090 (32 GB)</td><td>v2</td><td>124.2</td><td>not measured</td><td>Igneum Miner 0.3.0 package, prebuilt NVRTC worker</td><td>2026-10-04</td><td>bench log: 4 October 2026, the gfx1036 worker fault and what the Apple M5 Max could and could not reproduce</td><td>measured by the team. live devnet v4, 128 loads per hash, CPU re-check clean, 0 rejected</td></tr></tbody></table></div>
|
||||
<h2 id="how">How a row gets here</h2>
|
||||
<p>Every row names the engineering log entry or the job it came from. "Measured by the team" means our own hardware and our own log. "Reported by the fleet" means a machine we do not own, read from the status lines its miner uploads.</p>
|
||||
<p>MH per watt needs the card's power draw during the run. The app reads it on NVIDIA cards through the driver. Rows get the figure when a run records it.</p>
|
||||
<p>There is no other Igneum miner to compare with yet, so this table compares cards, not miners.</p>
|
||||
<p>Rows: 6. Source file: <code>site/miner-bench.json</code> in the repository.</p></article>
|
||||
</div>
|
||||
<p class="gen">Generated from the repository at build time. Times are UTC. Machine names are model names.</p>
|
||||
</main>
|
||||
<!-- footer:start -->
|
||||
<footer class="foot">
|
||||
<div class="wrap">
|
||||
<div class="foot-grid">
|
||||
<div class="foot-brand">
|
||||
<a href="/" class="brand" aria-label="Igneum home"><svg class="mark" viewBox="0 0 1024 1024" width="30" height="30" aria-hidden="true"><rect width="1024" height="1024" fill="#0C0C0E"></rect><g transform="translate(166.95 166.95) scale(6.901)"><polygon points="50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34" fill="#F2541B"></polygon><polygon points="50,42 59,58 50,82 41,58" fill="#0C0C0E"></polygon></g></svg><span class="word">IGNEUM</span></a>
|
||||
<p>Mined by GPUs. Proven by fire.</p>
|
||||
</div>
|
||||
<nav class="foot-col" aria-label="Read">
|
||||
<div class="eyebrow">Read</div>
|
||||
<a href="/litepaper">Litepaper</a>
|
||||
<a href="/litepaper#limits">What Igneum does not claim</a>
|
||||
<a href="/litepaper#randomx">Igneum vs RandomX</a>
|
||||
<a href="/litepaper#shoulders">Built on the shoulders</a>
|
||||
<a href="/bench">Engineering log</a>
|
||||
<a href="/evidence">Evidence</a>
|
||||
</nav>
|
||||
<nav class="foot-col" aria-label="Follow">
|
||||
<div class="eyebrow">Follow</div>
|
||||
<a href="/live">Live devnet</a>
|
||||
<a href="/#journey">Journey</a>
|
||||
<a href="https://github.com/igneum-network/spec" rel="noopener"><svg viewBox="0 0 24 24" width="15" height="15" fill="currentColor" aria-hidden="true"><path d="M12 .5C5.7.5.5 5.7.5 12c0 5.1 3.3 9.4 7.9 10.9.6.1.8-.3.8-.6v-2.1c-3.2.7-3.9-1.4-3.9-1.4-.5-1.3-1.3-1.7-1.3-1.7-1-.7.1-.7.1-.7 1.2.1 1.8 1.2 1.8 1.2 1 1.8 2.7 1.3 3.4 1 .1-.8.4-1.3.7-1.6-2.6-.3-5.3-1.3-5.3-5.7 0-1.3.5-2.3 1.2-3.1-.1-.3-.5-1.5.1-3.1 0 0 1-.3 3.2 1.2.9-.3 1.9-.4 2.9-.4s2 .1 2.9.4c2.2-1.5 3.2-1.2 3.2-1.2.6 1.6.2 2.8.1 3.1.8.8 1.2 1.8 1.2 3.1 0 4.4-2.7 5.4-5.3 5.7.4.4.8 1.1.8 2.2v3.2c0 .3.2.7.8.6 4.6-1.5 7.9-5.8 7.9-10.9C23.5 5.7 18.3.5 12 .5z"></path></svg>GitHub, spec and vectors</a>
|
||||
<a href="/#mine">Get the miner</a>
|
||||
</nav>
|
||||
</div>
|
||||
<div class="foot-base">
|
||||
<span>© 2026 Igneum. Nothing on this page is an offer to sell anything.</span>
|
||||
<span class="mono">igneum.network</span>
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
<!-- footer:end -->
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -10,6 +10,7 @@
|
|||
<a href="/litepaper" data-nav="litepaper">Litepaper</a>
|
||||
<a href="/live" data-nav="live">Live devnet</a>
|
||||
<a href="/bench" data-nav="bench">Engineering log</a>
|
||||
<a href="/miners" data-nav="miners">Miners</a>
|
||||
<a href="/evidence" data-nav="evidence">Evidence</a>
|
||||
<a href="https://github.com/igneum-network/spec" rel="noopener">GitHub</a>
|
||||
<a href="/#mine" class="cta">Get the miner</a>
|
||||
|
|
|
|||
|
|
@ -5,4 +5,5 @@
|
|||
<url><loc>https://igneum.network/live</loc><lastmod>2026-10-04</lastmod><changefreq>daily</changefreq><priority>0.7</priority></url>
|
||||
<url><loc>https://igneum.network/bench</loc><lastmod>2026-10-04</lastmod><changefreq>weekly</changefreq><priority>0.6</priority></url>
|
||||
<url><loc>https://igneum.network/evidence</loc><lastmod>2026-10-04</lastmod><changefreq>weekly</changefreq><priority>0.6</priority></url>
|
||||
<url><loc>https://igneum.network/miners</loc><lastmod>2026-10-04</lastmod><changefreq>weekly</changefreq><priority>0.6</priority></url>
|
||||
</urlset>
|
||||
|
|
|
|||
217
tools/reliability/app-run.mjs
Executable file
217
tools/reliability/app-run.mjs
Executable file
|
|
@ -0,0 +1,217 @@
|
|||
#!/usr/bin/env node
|
||||
// The app engine's watchdog measured end to end on a Mac: a scratch Igneum Miner engine with its own private node
|
||||
// (ports 29960 and 29961, devnet suffix 9960, no peers, unsynced mining allowed, so the node counts as synced) and
|
||||
// fake-worker.mjs standing in for the Metal worker, driven through the control file and with signals.
|
||||
//
|
||||
// node tools/reliability/app-run.mjs --app <igneum-app> --miner <igneum-miner> [--node <igneumd>]
|
||||
//
|
||||
// Steps, in one engine run (each states what must happen and what must not):
|
||||
// own-restart the worker reports jobs done in 0.3 ms: the miner's guard restarts the worker; the app shows the
|
||||
// fault and does NOT restart the miner (same pid, no app restart)
|
||||
// zero-once the worker completes jobs with 0 hashes: hash rate 0 for 60 s while synced, the app restarts the
|
||||
// miner once; the worker is healthy again: mining resumes
|
||||
// zero-faulted the same again inside five minutes: the card is marked faulted with the reason, its miner is not
|
||||
// restarted, the node keeps running; "resume" clears it
|
||||
// no-status the miner process is stopped with SIGSTOP: no status line for 90 s, the app restarts it
|
||||
// node-silent the node is stopped with SIGSTOP: no reading for 120 s, the app restarts the node in-process and
|
||||
// the miner comes back once it is synced
|
||||
// Never touches the live devnet. The app's own log is in the scratch directory.
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdirSync, rmSync, writeFileSync, existsSync, symlinkSync, readFileSync, appendFileSync, chmodSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const args = process.argv.slice(2);
|
||||
const opt = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
|
||||
const APP = opt('--app');
|
||||
const MINER = opt('--miner');
|
||||
const NODE = opt('--node', join(here, '../../vendor/igneum-node/target-integration/release/igneumd'));
|
||||
const ONLY = opt('--only', '').split(',').filter(Boolean);
|
||||
const SCRATCH = process.env.SCRATCH || `/tmp/igneum-reliability-app-${process.pid}`;
|
||||
const RPC = 29960, P2P = 29961, SUFFIX = 9960;
|
||||
for (const [k, v] of Object.entries({ APP, MINER, NODE })) if (!v || !existsSync(v)) { console.error(`missing ${k} (${v})`); process.exit(2); }
|
||||
|
||||
const t0 = Date.now();
|
||||
const since = () => ((Date.now() - t0) / 1000).toFixed(1);
|
||||
const log = (...a) => console.log(new Date().toISOString().slice(11, 23), `t=${since()}s`, ...a);
|
||||
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
const s = (ms) => (ms / 1000).toFixed(1) + ' s';
|
||||
|
||||
rmSync(SCRATCH, { recursive: true, force: true });
|
||||
const bin = join(SCRATCH, 'stage', 'bin'); mkdirSync(bin, { recursive: true });
|
||||
symlinkSync(NODE, join(bin, 'igneumd'));
|
||||
symlinkSync(MINER, join(bin, 'igneum-miner'));
|
||||
chmodSync(join(here, 'fake-worker.mjs'), 0o755);
|
||||
symlinkSync(join(here, 'fake-worker.mjs'), join(bin, 'igneum-bench'));
|
||||
const data = join(SCRATCH, 'data'); mkdirSync(join(data, 'app'), { recursive: true });
|
||||
const logs = join(SCRATCH, 'logs'); mkdirSync(logs, { recursive: true });
|
||||
const CTL = join(SCRATCH, 'ctl');
|
||||
const setMode = (m) => { writeFileSync(CTL, m + '\n'); log(`fake worker mode -> ${m}`); };
|
||||
setMode('ok');
|
||||
writeFileSync(join(data, 'app', 'settings.json'), JSON.stringify({
|
||||
setup_done: true, address: '0x4242424242424242424242424242424242424242', address_source: 'pasted', key_saved: true, identities: 1,
|
||||
cards: { 'apple::Fake GPU': { enabled: true, identities: 1, power_pct: 0 } }, vote: false, paused: false, accepted_total: 0,
|
||||
auto_update: false, remote_jobs: false, prove: false,
|
||||
}, null, 2));
|
||||
|
||||
const env = {
|
||||
...process.env, IGNEUM_APP_DATA: data, IGNEUM_APP_LOGS: logs, IGNEUM_APP_BIN: bin, IGNEUM_APP_RPC_PORT: String(RPC), IGNEUM_APP_P2P_PORT: String(P2P),
|
||||
IGNEUM_APP_PEERS: '', IGNEUM_APP_UNSYNCED: '1', IGNEUM_APP_DEVNET_SUFFIX: String(SUFFIX), IGNEUM_APP_STATUS_SECS: '10', FAKE_WORKER_CTL: CTL,
|
||||
};
|
||||
const app = spawn(APP, ['--no-open'], { stdio: ['pipe', 'pipe', 'pipe'], env });
|
||||
const appOut = join(SCRATCH, 'app.out');
|
||||
app.stdout.on('data', d => appendFileSync(appOut, d)); app.stderr.on('data', d => appendFileSync(appOut, d));
|
||||
let appExit = null;
|
||||
app.on('exit', c => { appExit = c; log(`app exited ${c}`); });
|
||||
process.on('exit', () => { try { app.kill('SIGKILL'); } catch {} });
|
||||
process.on('SIGINT', () => process.exit(130));
|
||||
log(`app pid ${app.pid}, scratch ${SCRATCH}`);
|
||||
|
||||
let url = null;
|
||||
for (let i = 0; i < 100 && !url; i++) { try { url = readFileSync(join(data, 'app', 'app.url'), 'utf8').trim(); } catch { await sleep(200); } }
|
||||
if (!url) { log('no app.url'); process.exit(1); }
|
||||
async function state() {
|
||||
try { const r = await fetch(url + 'api/state'); return await r.json(); } catch { return null; }
|
||||
}
|
||||
const card = (st) => (st && st.mining && st.mining.cards && st.mining.cards[0]) || {};
|
||||
const events = []; // (t, card state, message, hash, pid, restarts, faults, node state) transitions, for the record
|
||||
let last = '';
|
||||
async function poll() {
|
||||
const st = await state();
|
||||
if (!st) return null;
|
||||
const c = card(st);
|
||||
const key = [c.state, c.message, c.pid, c.restarts, c.faults, st.node.state, c.hash_now > 0 ? '>0' : '0'].join('|');
|
||||
if (key !== last) { last = key; events.push({ t: Date.now(), key }); log(`card ${c.state} pid ${c.pid} restarts ${c.restarts} faults ${c.faults} hash ${Number(c.hash_now).toFixed(1)} node ${st.node.state} ${c.message ? '"' + c.message + '"' : ''}`); }
|
||||
return st;
|
||||
}
|
||||
async function until(pred, timeoutMs, what) {
|
||||
const end = Date.now() + timeoutMs;
|
||||
while (Date.now() < end) { const st = await poll(); if (st && pred(st, card(st))) return st; if (appExit !== null) return null; await sleep(1000); }
|
||||
log(`timeout waiting for: ${what}`);
|
||||
return null;
|
||||
}
|
||||
const mining = (st, c) => c.state === 'mining' && c.hash_now > 0 && st.node.synced;
|
||||
const results = [];
|
||||
function verdict(name, checks, metrics) {
|
||||
const pass = checks.every(c => c.ok);
|
||||
results.push({ name, pass, checks, metrics });
|
||||
log(`${pass ? 'PASS' : 'FAIL'} ${name}`);
|
||||
for (const c of checks) log(` ${c.ok ? 'ok ' : 'FAIL'} ${c.what}`);
|
||||
for (const [k, v] of Object.entries(metrics)) log(` ${k}: ${v}`);
|
||||
}
|
||||
async function steady(ms) { const end = Date.now() + ms; while (Date.now() < end) { await poll(); await sleep(1000); } }
|
||||
|
||||
const S = {};
|
||||
S['own-restart'] = async () => {
|
||||
const st0 = await until(mining, 180000, 'first mining');
|
||||
const pid0 = card(st0).pid, r0 = card(st0).restarts;
|
||||
const tInject = Date.now();
|
||||
setMode('fast');
|
||||
const faulted = await until((st, c) => c.faults >= 1, 90000, 'the worker fault to reach the card');
|
||||
const tFault = Date.now();
|
||||
setMode('ok');
|
||||
const back = await until((st, c) => mining(st, c) && c.faults >= 1, 90000, 'mining again after the worker restart');
|
||||
const tBack = Date.now();
|
||||
await steady(20000);
|
||||
const st1 = await poll();
|
||||
verdict('own-restart', [
|
||||
{ ok: !!st0, what: 'the card mined with a rate above 0 on the fake worker' },
|
||||
{ ok: !!faulted && /worker fault|killed by a guard/.test(card(faulted).message || ''), what: `the card showed the worker fault (${JSON.stringify(card(faulted || {}).message)})` },
|
||||
{ ok: !!back, what: 'mining resumed after the miner restarted its own worker' },
|
||||
{ ok: st1 && card(st1).pid === pid0 && card(st1).restarts === r0, what: `the app did not restart the miner (pid ${pid0} -> ${card(st1 || {}).pid}, app restarts ${r0} -> ${card(st1 || {}).restarts})` },
|
||||
], { inject_to_fault_on_card: s(tFault - tInject), fault_to_mining_again: s(tBack - tFault) });
|
||||
};
|
||||
S['zero-once'] = async () => {
|
||||
const st0 = await until(mining, 60000, 'mining');
|
||||
const pid0 = card(st0).pid, r0 = card(st0).restarts;
|
||||
const tInject = Date.now();
|
||||
setMode('zero');
|
||||
const rs = await until((st, c) => c.restarts > r0 || /watchdog/.test(c.message || ''), 150000, 'the watchdog restart');
|
||||
const tRestart = Date.now();
|
||||
setMode('ok');
|
||||
const back = await until((st, c) => mining(st, c) && c.pid !== pid0, 120000, 'mining on the restarted miner');
|
||||
const tBack = Date.now();
|
||||
await steady(15000);
|
||||
const st1 = await poll();
|
||||
verdict('zero-once', [
|
||||
{ ok: !!rs && /hash rate 0/.test(card(rs).message || ''), what: `the watchdog restarted the miner for a zero rate (${JSON.stringify(card(rs || {}).message)})` },
|
||||
{ ok: rs && tRestart - tInject >= 55000 && tRestart - tInject <= 100000, what: `between 55 and 100 s after the rate went to 0 (${s(tRestart - tInject)})` },
|
||||
{ ok: !!back, what: 'mining resumed on a new miner process' },
|
||||
{ ok: st1 && card(st1).restarts === r0 + 1, what: `exactly one app restart (${r0} -> ${card(st1 || {}).restarts})` },
|
||||
], { zero_to_restart: s(tRestart - tInject), restart_to_mining: s(tBack - tRestart), zero_to_mining: s(tBack - tInject) });
|
||||
};
|
||||
S['zero-faulted'] = async () => {
|
||||
const st0 = await until(mining, 60000, 'mining');
|
||||
const r0 = card(st0).restarts;
|
||||
const tInject = Date.now();
|
||||
setMode('zero');
|
||||
const f = await until((st, c) => c.state === 'faulted', 150000, 'the card to be marked faulted');
|
||||
const tFault = Date.now();
|
||||
await steady(45000);
|
||||
const st1 = await poll();
|
||||
setMode('ok');
|
||||
app.stdin.write('resume\n');
|
||||
const back = await until(mining, 90000, 'mining after resume');
|
||||
verdict('zero-faulted', [
|
||||
{ ok: !!f && /restarted once already/.test(card(f).message || ''), what: `the card was marked faulted with the reason (${JSON.stringify(card(f || {}).message)})` },
|
||||
{ ok: st1 && card(st1).state === 'faulted' && card(st1).pid === 0 && card(st1).restarts === r0, what: `45 s later: still faulted, no miner process, no further restart (restarts ${r0} -> ${card(st1 || {}).restarts})` },
|
||||
{ ok: st1 && st1.node.synced && appExit === null, what: 'the node kept running and the app stayed up' },
|
||||
{ ok: !!back, what: 'resume cleared the fault and mining resumed' },
|
||||
], { zero_to_faulted: s(tFault - tInject) });
|
||||
};
|
||||
S['no-status'] = async () => {
|
||||
const st0 = await until(mining, 60000, 'mining');
|
||||
const pid0 = card(st0).pid, r0 = card(st0).restarts;
|
||||
const tInject = Date.now();
|
||||
process.kill(pid0, 'SIGSTOP');
|
||||
log(`SIGSTOP miner pid ${pid0}`);
|
||||
const rs = await until((st, c) => c.restarts > r0 || /no status line/.test(c.message || ''), 150000, 'the watchdog restart');
|
||||
const tRestart = Date.now();
|
||||
const back = await until((st, c) => mining(st, c) && c.pid !== pid0 && c.pid > 0, 120000, 'mining on the restarted miner');
|
||||
const tBack = Date.now();
|
||||
let gone = false; try { process.kill(pid0, 0); } catch { gone = true; }
|
||||
if (!gone) { try { process.kill(pid0, 'SIGKILL'); } catch {} }
|
||||
verdict('no-status', [
|
||||
{ ok: !!rs && /no status line/.test(card(rs).message || ''), what: `the watchdog restarted the miner for missing status lines (${JSON.stringify(card(rs || {}).message)})` },
|
||||
{ ok: rs && tRestart - tInject >= 85000 && tRestart - tInject <= 130000, what: `between 85 and 130 s after the miner went quiet (${s(tRestart - tInject)})` },
|
||||
{ ok: !!back, what: 'mining resumed on a new miner process' },
|
||||
{ ok: gone, what: 'the stopped miner process was killed' },
|
||||
], { quiet_to_restart: s(tRestart - tInject), restart_to_mining: s(tBack - tRestart) });
|
||||
};
|
||||
S['node-silent'] = async () => {
|
||||
const st0 = await until(mining, 60000, 'mining');
|
||||
const npid = st0.node.pid, starts0 = st0.node.starts;
|
||||
const tInject = Date.now();
|
||||
process.kill(npid, 'SIGSTOP');
|
||||
log(`SIGSTOP node pid ${npid}`);
|
||||
const rs = await until((st) => st.node.state === 'restarting' || st.node.starts > starts0, 200000, 'the node restart');
|
||||
const tRestart = Date.now();
|
||||
const synced = await until((st) => st.node.starts > starts0 && st.node.synced, 120000, 'the restarted node to sync');
|
||||
const tSynced = Date.now();
|
||||
const back = await until(mining, 120000, 'mining again');
|
||||
const tBack = Date.now();
|
||||
let gone = false; try { process.kill(npid, 0); } catch { gone = true; }
|
||||
if (!gone) { try { process.kill(npid, 'SIGKILL'); } catch {} }
|
||||
verdict('node-silent', [
|
||||
{ ok: !!rs, what: 'the app restarted the node' },
|
||||
{ ok: rs && tRestart - tInject >= 115000 && tRestart - tInject <= 170000, what: `between 115 and 170 s after the node went quiet (${s(tRestart - tInject)})` },
|
||||
{ ok: !!synced, what: `the new node synced (starts ${starts0} -> ${synced ? synced.node.starts : '?'})` },
|
||||
{ ok: !!back, what: 'mining resumed' },
|
||||
{ ok: gone, what: 'the stopped node process was killed' },
|
||||
], { quiet_to_restart: s(tRestart - tInject), restart_to_synced: s(tSynced - tRestart), quiet_to_mining: s(tBack - tInject) });
|
||||
};
|
||||
|
||||
const names = ONLY.length ? ONLY : Object.keys(S);
|
||||
for (const n of names) {
|
||||
if (!S[n]) { log(`unknown step ${n}`); continue; }
|
||||
log(`=== ${n}`);
|
||||
try { await S[n](); } catch (e) { log(`step ${n} threw: ${e.stack || e}`); results.push({ name: n, pass: false, checks: [{ ok: false, what: String(e) }], metrics: {} }); }
|
||||
}
|
||||
app.stdin.write('quit\n');
|
||||
await sleep(8000);
|
||||
const report = { date: new Date().toISOString(), app: APP, miner: MINER, node: NODE, scratch: SCRATCH, results, transitions: events.map(e => ({ t: new Date(e.t).toISOString(), key: e.key })) };
|
||||
writeFileSync(join(SCRATCH, 'report.json'), JSON.stringify(report, null, 2));
|
||||
console.log(JSON.stringify({ ...report, transitions: undefined }, null, 2));
|
||||
process.exit(results.every(r => r.pass) ? 0 : 1);
|
||||
74
tools/reliability/fake-worker.mjs
Executable file
74
tools/reliability/fake-worker.mjs
Executable file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env node
|
||||
// A GPU worker stand-in that speaks the miner's serve protocol (ready / job / done / found / prepare / prepared /
|
||||
// quit) and misbehaves on command, so the miner's fault guards and the app's watchdog can be measured on a Mac with
|
||||
// no faulty GPU at hand. It never hashes: every `done` is invented, every `found` is wrong on purpose.
|
||||
//
|
||||
// FAKE_WORKER_CTL=<file> the file's first word is the mode, re-read before every job:
|
||||
// ok every job: `done` after 100 ms with the job's nonce count as hashes (about 167 MH/s at 2^24 nonces)
|
||||
// slow `done` after 5,000 ms (a self-heal build inside a job, a game on the GPU)
|
||||
// fast `done` after 0.3 ms with the full count: a runtime answering success while running nothing
|
||||
// (the 4 October 2026 gfx1036 fault)
|
||||
// zero `done` after 100 ms with 0 hashes: the worker completes jobs and produces nothing
|
||||
// silent never answers a job (the queue fills and nothing comes back)
|
||||
// badfound a `found` with a wrong hash before every `done`: a worker on a wrong program
|
||||
// preparefail every `prepare` is answered `prepare-failed`; jobs run as in ok
|
||||
// exit the process exits with code 7 at the next job
|
||||
// The mode is read from argv `--mode <m>` when no control file is set. `--serve` is accepted and ignored.
|
||||
// Lines on stderr are prefixed `fake-worker:` and never match a miner pattern.
|
||||
|
||||
import { readFileSync, existsSync, writeSync } from 'node:fs';
|
||||
import { createInterface } from 'node:readline';
|
||||
|
||||
const ctl = process.env.FAKE_WORKER_CTL;
|
||||
const argMode = (() => { const i = process.argv.indexOf('--mode'); return i >= 0 ? process.argv[i + 1] : 'ok'; })();
|
||||
function mode() {
|
||||
if (ctl && existsSync(ctl)) {
|
||||
const m = readFileSync(ctl, 'utf8').trim().split(/\s+/)[0];
|
||||
if (m) return m;
|
||||
}
|
||||
return argMode;
|
||||
}
|
||||
// synchronous writes: a piped stdout is asynchronous in Node and process.exit would drop pending lines
|
||||
const out = (s) => { writeSync(1, s + '\n'); };
|
||||
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
|
||||
out('ready fake Fake_GPU dataset-log2 28 batch 16777216 prepare 1');
|
||||
|
||||
const queue = [];
|
||||
let running = false;
|
||||
async function pump() {
|
||||
if (running) return;
|
||||
running = true;
|
||||
while (queue.length) {
|
||||
const f = queue.shift();
|
||||
const [id, , , nonceStart, nonces] = f;
|
||||
const m = mode();
|
||||
if (m === 'exit') process.exit(7);
|
||||
if (m === 'silent') { await sleep(1_000_000); continue; }
|
||||
const t0 = performance.now();
|
||||
let hashes = Number(nonces);
|
||||
let ms = 100;
|
||||
if (m === 'slow') ms = 5000;
|
||||
else if (m === 'fast') ms = 0.3;
|
||||
else if (m === 'zero') hashes = 0;
|
||||
await sleep(ms);
|
||||
if (m === 'badfound') out(`found ${id} ${BigInt(nonceStart) + 17n} 0000000000000001`);
|
||||
out(`done ${id} ${hashes} ${(performance.now() - t0).toFixed(2)}`);
|
||||
}
|
||||
running = false;
|
||||
}
|
||||
|
||||
const rl = createInterface({ input: process.stdin });
|
||||
rl.on('line', (line) => {
|
||||
const f = line.trim().split(/\s+/);
|
||||
switch (f[0]) {
|
||||
case 'job': queue.push(f.slice(1)); pump(); break;
|
||||
case 'prepare':
|
||||
if (mode() === 'preparefail') out(`prepare-failed ${f[1]} ${f[2]} fake worker refuses`);
|
||||
else setTimeout(() => out(`prepared ${f[1]} ${f[2]} 50 fake resident 2 programs 2 datasets`), 50);
|
||||
break;
|
||||
case 'quit': process.exit(0);
|
||||
default: break;
|
||||
}
|
||||
});
|
||||
rl.on('close', () => process.exit(0));
|
||||
313
tools/reliability/run.mjs
Executable file
313
tools/reliability/run.mjs
Executable file
|
|
@ -0,0 +1,313 @@
|
|||
#!/usr/bin/env node
|
||||
// Miner fault guards measured on a private test network (ports 29950 to 29953, devnet suffix 9950, data under the
|
||||
// scratch directory): one igneumd, one CPU block producer when a scenario needs epochs, and the miner under test
|
||||
// driving fake-worker.mjs, which misbehaves on command through a control file. Never touches the live devnet.
|
||||
//
|
||||
// node tools/reliability/run.mjs --miner <igneum-miner> [--node <igneumd>] [--only slow-first,fake-fast,...]
|
||||
//
|
||||
// Scenarios (each a fresh miner process; the fake worker's mode is switched by rewriting the control file):
|
||||
// slow-first the first 25 s of jobs take 5 s each, then the true rate (100x): the old guard tripped on every
|
||||
// interval from the second on (M26); expected now: no fault, STATUS every 10 s throughout
|
||||
// fake-fast 3 healthy intervals, then jobs "done" in 0.3 ms: one fault, a STATUS line with faults=1,
|
||||
// the worker back after the 2 s delay; the mode returns to ok at the fault
|
||||
// fake-fast-stays the same, but the fault persists: three trips at growing delay, then exit 43
|
||||
// badfound wrong hashes on every found: three consecutive CPU re-check mismatches stop the worker (X21)
|
||||
// silent the worker stops answering: STATUS lines keep coming, the stall guard fires at 60 s
|
||||
// prepare-flap with epochs every 60 DAA (a 3-thread CPU block producer on easy genesis bits), the worker refuses every prepare: at most two
|
||||
// sends per pair per epoch, 30 s apart (M27)
|
||||
//
|
||||
// A watcher is trusted only once it fires on a known-good and a known-bad case: every scenario states what must
|
||||
// appear AND what must not, and the run fails if the fault detector saw nothing in the scenarios that fault.
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdirSync, rmSync, writeFileSync, existsSync, appendFileSync, readFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const args = process.argv.slice(2);
|
||||
const opt = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
|
||||
const MINER = opt('--miner');
|
||||
const NODE = opt('--node', join(here, '../../vendor/igneum-node/target-integration/release/igneumd'));
|
||||
const ONLY = opt('--only', '').split(',').filter(Boolean);
|
||||
const SCRATCH = process.env.SCRATCH || `/tmp/igneum-reliability-${process.pid}`;
|
||||
const BASE = 29950, SUFFIX = 9950;
|
||||
const FAKE = join(here, 'fake-worker.mjs');
|
||||
if (!MINER || !existsSync(MINER)) { console.error('need --miner <igneum-miner with the guards>'); process.exit(2); }
|
||||
if (!existsSync(NODE)) { console.error(`missing node ${NODE}`); process.exit(2); }
|
||||
|
||||
const t0 = Date.now();
|
||||
const since = () => ((Date.now() - t0) / 1000).toFixed(1);
|
||||
const log = (...a) => console.log(new Date().toISOString().slice(11, 23), `t=${since()}s`, ...a);
|
||||
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
const started = [];
|
||||
process.on('exit', () => { for (const p of started) { try { p.kill('SIGKILL'); } catch {} } });
|
||||
process.on('SIGINT', () => process.exit(130));
|
||||
|
||||
rmSync(SCRATCH, { recursive: true, force: true }); mkdirSync(SCRATCH, { recursive: true });
|
||||
const CTL = join(SCRATCH, 'ctl');
|
||||
const setMode = (m) => { writeFileSync(CTL, m + '\n'); log(`fake worker mode -> ${m}`); };
|
||||
|
||||
function startNode(env = {}) {
|
||||
const dir = join(SCRATCH, 'node'); mkdirSync(dir, { recursive: true });
|
||||
const a = ['--devnet', `--devnet-suffix=${SUFFIX}`, '--nodnsseed', '--disable-upnp', '--nologfiles', '--enable-unsynced-mining', '--outpeers=0',
|
||||
`--appdir=${dir}`, `--rpclisten=127.0.0.1:${BASE}`, `--rpclisten-json=127.0.0.1:${BASE + 2}`, `--evm-rpclisten=127.0.0.1:${BASE + 3}`,
|
||||
`--listen=127.0.0.1:${BASE + 1}`, '--loglevel=warn', '--yes'];
|
||||
const p = spawn(NODE, a, { stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, ...env } });
|
||||
started.push(p);
|
||||
const f = join(SCRATCH, 'node.log');
|
||||
p.stdout.on('data', d => appendFileSync(f, d)); p.stderr.on('data', d => appendFileSync(f, d));
|
||||
log(`node pid ${p.pid} grpc ${BASE}`);
|
||||
return p;
|
||||
}
|
||||
|
||||
/// The miner under test: lines are kept with the receipt time (the miner's own unix stamp when the line carries one).
|
||||
function startMiner(secs, extra = []) {
|
||||
const a = ['mine', `grpc://127.0.0.1:${BASE}`, '1', String(secs), 'wt', '--worker', FAKE, '--status-secs', '10', '--no-vote', '--payout-label', 'wt', ...extra];
|
||||
const p = spawn(MINER, a, { stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, FAKE_WORKER_CTL: CTL } });
|
||||
started.push(p);
|
||||
const m = { proc: p, lines: [], exit: null };
|
||||
const f = join(SCRATCH, `miner-${Date.now()}.log`);
|
||||
const take = (err) => (chunk) => {
|
||||
for (const text of chunk.toString().split('\n')) {
|
||||
if (!text) continue;
|
||||
const stamp = /^(\d{10})\.(\d{3}) /.exec(text);
|
||||
const t = stamp ? Number(stamp[1]) * 1000 + Number(stamp[2]) : Date.now();
|
||||
m.lines.push({ t, err, text });
|
||||
appendFileSync(f, (err ? '! ' : '') + text + '\n');
|
||||
}
|
||||
};
|
||||
p.stdout.on('data', take(false)); p.stderr.on('data', take(true));
|
||||
p.on('exit', (code) => { m.exit = code; log(`miner exited ${code}`); });
|
||||
log(`miner pid ${p.pid} log ${f}`);
|
||||
return m;
|
||||
}
|
||||
// stdout only by default: the miner prints WORKER FAULT and the exit line on both streams
|
||||
const find = (m, re, after = 0, both = false) => m.lines.filter(l => l.t >= after && (both || !l.err) && re.test(l.text));
|
||||
async function waitForErr(m, re, timeoutMs, after = 0) {
|
||||
const end = Date.now() + timeoutMs;
|
||||
while (Date.now() < end) { const h = find(m, re, after, true); if (h.length) return h[0]; if (m.exit !== null) return null; await sleep(200); }
|
||||
return null;
|
||||
}
|
||||
async function waitFor(m, re, timeoutMs, after = 0) {
|
||||
const end = Date.now() + timeoutMs;
|
||||
while (Date.now() < end) { const h = find(m, re, after); if (h.length) return h[0]; if (m.exit !== null) return null; await sleep(200); }
|
||||
return null;
|
||||
}
|
||||
const stop = (m) => { try { m.proc.kill('SIGTERM'); } catch {} };
|
||||
const s = (ms) => (ms / 1000).toFixed(1) + ' s';
|
||||
const results = [];
|
||||
function verdict(name, checks, metrics) {
|
||||
const pass = checks.every(c => c.ok);
|
||||
results.push({ name, pass, checks, metrics });
|
||||
log(`${pass ? 'PASS' : 'FAIL'} ${name}`);
|
||||
for (const c of checks) log(` ${c.ok ? 'ok ' : 'FAIL'} ${c.what}`);
|
||||
for (const [k, v] of Object.entries(metrics)) log(` ${k}: ${v}`);
|
||||
}
|
||||
|
||||
const S = {};
|
||||
|
||||
S['slow-first'] = async () => {
|
||||
setMode('slow');
|
||||
const m = startMiner(120);
|
||||
const ready = await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(25000);
|
||||
setMode('ok');
|
||||
await sleep(60000);
|
||||
stop(m);
|
||||
const faults = find(m, /WORKER FAULT/);
|
||||
const status = find(m, / STATUS '/);
|
||||
const gaps = status.slice(1).map((l, i) => l.t - status[i].t);
|
||||
const restarted = find(m, /worker restarted/);
|
||||
const lastFault = faults.length ? faults[faults.length - 1].t : 0;
|
||||
const healthyAfter = find(m, / STATUS '.*now=(?!0\.00)/, lastFault);
|
||||
verdict('slow-first', [
|
||||
{ ok: !!ready, what: 'worker reported ready' },
|
||||
{ ok: faults.length <= 1, what: `at most one WORKER FAULT after a slow start (saw ${faults.length}; the old guard tripped on every interval)` },
|
||||
{ ok: faults.length === restarted.length, what: `every fault was followed by one restart (faults ${faults.length}, restarts ${restarted.length})` },
|
||||
{ ok: status.length >= 7, what: `STATUS lines kept coming (${status.length} in 85 s)` },
|
||||
{ ok: gaps.every(g => g < 25000), what: `no STATUS gap over 25 s (max ${s(Math.max(0, ...gaps))})` },
|
||||
{ ok: healthyAfter.length >= 3, what: `healthy STATUS lines after the last fault with no further trip (${healthyAfter.length})` },
|
||||
], { status_lines: status.length, faults: faults.length, restarts: restarted.length, slow_to_fault: faults.length && ready ? s(faults[0].t - ready.t) : 'none', fault_to_restart: faults.length && restarted.length ? s(restarted[0].t - faults[0].t) : 'n/a' });
|
||||
};
|
||||
|
||||
S['fake-fast'] = async () => {
|
||||
setMode('ok');
|
||||
const m = startMiner(180);
|
||||
await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(36000);
|
||||
const tInject = Date.now();
|
||||
setMode('fast');
|
||||
const fault = await waitFor(m, /WORKER FAULT/, 60000, tInject);
|
||||
if (fault) setMode('ok');
|
||||
const killed = await waitForErr(m, /worker killed by a guard/, 20000, tInject);
|
||||
const restarted = await waitFor(m, /worker restarted/, 30000, tInject);
|
||||
const ready = await waitFor(m, /worker: ready/, 30000, restarted ? restarted.t : tInject);
|
||||
// the first STATUS after the restart with a rate above 0
|
||||
let recovered = null;
|
||||
const end = Date.now() + 40000;
|
||||
while (!recovered && Date.now() < end) {
|
||||
recovered = find(m, / STATUS '.*now=(?!0\.00)/, ready ? ready.t : tInject)[0] || null;
|
||||
if (!recovered) await sleep(300);
|
||||
}
|
||||
await sleep(15000);
|
||||
stop(m);
|
||||
const statusAfterFault = fault ? find(m, / STATUS '.*faults=[1-9]/, fault.t) : [];
|
||||
const faultsTotal = find(m, /WORKER FAULT/).length;
|
||||
const restarts = find(m, /worker restarted/).length;
|
||||
if (!restarted) recovered = null; // a rate after the restart only counts once there was a restart
|
||||
verdict('fake-fast', [
|
||||
{ ok: !!fault, what: 'the guard fired on jobs done in 0.3 ms' },
|
||||
{ ok: statusAfterFault.length > 0, what: 'a STATUS line with faults=1 was printed after the trip (the old guard printed none)' },
|
||||
{ ok: !!killed && !!restarted, what: 'the worker was killed by the guard and restarted' },
|
||||
{ ok: !!recovered, what: 'a STATUS line with a rate above 0 followed the restart' },
|
||||
{ ok: faultsTotal === 1 && restarts === 1, what: `exactly one fault and one restart (faults ${faultsTotal}, restarts ${restarts})` },
|
||||
], {
|
||||
inject_to_fault: fault ? s(fault.t - tInject) : 'n/a',
|
||||
fault_to_restarted: fault && restarted ? s(restarted.t - fault.t) : 'n/a',
|
||||
fault_to_ready: fault && ready ? s(ready.t - fault.t) : 'n/a',
|
||||
fault_to_first_healthy_status: fault && recovered ? s(recovered.t - fault.t) : 'n/a',
|
||||
inject_to_first_healthy_status: recovered ? s(recovered.t - tInject) : 'n/a',
|
||||
});
|
||||
};
|
||||
|
||||
S['fake-fast-stays'] = async () => {
|
||||
setMode('ok');
|
||||
const m = startMiner(600);
|
||||
await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(36000);
|
||||
const tInject = Date.now();
|
||||
setMode('fast');
|
||||
const end = Date.now() + 300000;
|
||||
while (m.exit === null && Date.now() < end) await sleep(500);
|
||||
const faults = find(m, /WORKER FAULT/);
|
||||
const delays = find(m, /restarting it in (\d+) s/, 0, true).map(l => Number(/restarting it in (\d+) s/.exec(l.text)[1]));
|
||||
const gaveUp = find(m, /exiting with code 43/);
|
||||
verdict('fake-fast-stays', [
|
||||
{ ok: faults.length >= 3, what: `three or more faults (${faults.length})` },
|
||||
{ ok: delays.length >= 2 && delays[0] === 2 && delays[1] === 4, what: `restart delays grow 2, 4 s (saw ${delays.join(', ')})` },
|
||||
{ ok: m.exit === 43 && gaveUp.length === 1, what: `the miner gave up with exit 43 (exit ${m.exit})` },
|
||||
], {
|
||||
inject_to_exit: s((m.lines.at(-1)?.t || Date.now()) - tInject),
|
||||
fault_times: faults.map(l => s(l.t - tInject)).join(', '),
|
||||
});
|
||||
};
|
||||
|
||||
S['badfound'] = async () => {
|
||||
setMode('ok');
|
||||
const m = startMiner(180);
|
||||
await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(15000);
|
||||
const tInject = Date.now();
|
||||
setMode('badfound');
|
||||
const mism = await waitForErr(m, /WORKER MISMATCH/, 30000, tInject); // stderr
|
||||
const fault = await waitFor(m, /WORKER FAULT cpu re-check: 3 consecutive mismatches/, 60000, tInject);
|
||||
if (fault) setMode('ok');
|
||||
const restarted = await waitFor(m, /worker restarted/, 30000, tInject);
|
||||
const ready = await waitFor(m, /worker: ready/, 30000, restarted ? restarted.t : tInject);
|
||||
let recovered = null;
|
||||
const end = Date.now() + 40000;
|
||||
while (!recovered && Date.now() < end) {
|
||||
recovered = find(m, / STATUS '.*now=(?!0\.00)/, ready ? ready.t : tInject)[0] || null;
|
||||
if (!recovered) await sleep(300);
|
||||
}
|
||||
await sleep(12000);
|
||||
stop(m);
|
||||
if (!restarted) recovered = null;
|
||||
const statusMism = find(m, / STATUS '.*mismatched=[1-9]/);
|
||||
const mismCount = find(m, /WORKER MISMATCH/, 0, true).length;
|
||||
verdict('badfound', [
|
||||
{ ok: !!mism, what: 'the CPU re-check rejected the wrong hash' },
|
||||
{ ok: !!fault, what: 'three consecutive mismatches stopped the worker (X21)' },
|
||||
{ ok: mismCount <= 4, what: `no more than 4 wrong shares were accepted before the stop (${mismCount})` },
|
||||
{ ok: statusMism.length > 0, what: 'STATUS shows mismatched= above 0 for the app to read' },
|
||||
{ ok: !!restarted && !!recovered, what: 'the worker restarted and a rate above 0 followed' },
|
||||
], {
|
||||
inject_to_first_mismatch: mism ? s(mism.t - tInject) : 'n/a',
|
||||
inject_to_fault: fault ? s(fault.t - tInject) : 'n/a',
|
||||
fault_to_first_healthy_status: fault && recovered ? s(recovered.t - fault.t) : 'n/a',
|
||||
});
|
||||
};
|
||||
|
||||
S['silent'] = async () => {
|
||||
setMode('ok');
|
||||
const m = startMiner(240);
|
||||
await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(15000);
|
||||
const tInject = Date.now();
|
||||
setMode('silent');
|
||||
const fault = await waitFor(m, /WORKER FAULT no job completed/, 120000, tInject);
|
||||
if (fault) setMode('ok');
|
||||
const restarted = await waitFor(m, /worker restarted/, 30000, tInject);
|
||||
const ready = await waitFor(m, /worker: ready/, 30000, restarted ? restarted.t : tInject);
|
||||
let recovered = null;
|
||||
const end = Date.now() + 40000;
|
||||
while (!recovered && Date.now() < end) {
|
||||
recovered = find(m, / STATUS '.*now=(?!0\.00)/, ready ? ready.t : tInject)[0] || null;
|
||||
if (!recovered) await sleep(300);
|
||||
}
|
||||
await sleep(12000);
|
||||
stop(m);
|
||||
if (!restarted) recovered = null;
|
||||
const statusDuringSilence = fault ? find(m, / STATUS '/, tInject).filter(l => l.t < fault.t) : [];
|
||||
verdict('silent', [
|
||||
{ ok: !!fault, what: 'the stall guard fired on a worker that stopped answering' },
|
||||
{ ok: fault && fault.t - tInject >= 55000 && fault.t - tInject <= 90000, what: `the stall guard fired between 55 and 90 s after the silence began (${fault ? s(fault.t - tInject) : 'n/a'})` },
|
||||
{ ok: statusDuringSilence.length >= 4, what: `STATUS lines kept printing while the worker was silent (${statusDuringSilence.length}; the old loop printed none)` },
|
||||
{ ok: !!restarted && !!recovered, what: 'the worker restarted and a rate above 0 followed' },
|
||||
], {
|
||||
silence_to_fault: fault ? s(fault.t - tInject) : 'n/a',
|
||||
fault_to_first_healthy_status: fault && recovered ? s(recovered.t - fault.t) : 'n/a',
|
||||
silence_to_first_healthy_status: recovered ? s(recovered.t - tInject) : 'n/a',
|
||||
});
|
||||
};
|
||||
|
||||
S['prepare-flap'] = async () => {
|
||||
// a CPU block producer so the DAA moves and epochs turn every 60 blocks (lead 20)
|
||||
const cpu = spawn(MINER, ['mine', `grpc://127.0.0.1:${BASE}`, '3', '400', 'cpu', '--engine', 'igneum-pow', '--no-vote', '--payout-label', 'cpu', '--status-secs', '30'], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
started.push(cpu);
|
||||
log(`cpu block producer pid ${cpu.pid}`);
|
||||
// the block producer's log, for the block count (the node logs at warn level)
|
||||
const cf = join(SCRATCH, 'cpu-miner.log'); cpu.stdout.on('data', d => appendFileSync(cf, d)); cpu.stderr.on('data', d => appendFileSync(cf, d));
|
||||
setMode('preparefail');
|
||||
const m = startMiner(360);
|
||||
await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(300000);
|
||||
stop(m); try { cpu.kill('SIGTERM'); } catch {}
|
||||
const sent = find(m, /PREPARE sent for epoch seed (\S+) day (\d+)/);
|
||||
const held = find(m, /PREPARE held/);
|
||||
const failed = find(m, /could not prepare/, 0, true); // stderr
|
||||
const changes = find(m, /SEED CHANGE/);
|
||||
// per epoch (between SEED CHANGE lines): sends per pair and the smallest gap between any two sends
|
||||
const epochs = [];
|
||||
const bounds = [0, ...changes.map(l => l.t), Infinity];
|
||||
for (let i = 0; i + 1 < bounds.length; i++) {
|
||||
const inEpoch = sent.filter(l => l.t >= bounds[i] && l.t < bounds[i + 1]);
|
||||
const perPair = {};
|
||||
for (const l of inEpoch) { const k = /epoch seed (\S+) day (\d+)/.exec(l.text).slice(1).join('/'); perPair[k] = (perPair[k] || 0) + 1; }
|
||||
epochs.push({ sends: inEpoch.length, maxPerPair: Math.max(0, ...Object.values(perPair)) });
|
||||
}
|
||||
const gaps = sent.slice(1).map((l, i) => l.t - sent[i].t);
|
||||
verdict('prepare-flap', [
|
||||
{ ok: changes.length >= 2, what: `epochs turned (${changes.length} seed changes in 5 min)` },
|
||||
{ ok: sent.length >= 2, what: `prepares were sent (${sent.length})` },
|
||||
{ ok: failed.length >= 1, what: `the worker refused them (${failed.length} prepare-failed)` },
|
||||
{ ok: epochs.every(e => e.maxPerPair <= 2), what: `at most two sends per pair per epoch (${epochs.map(e => e.maxPerPair).join(', ')})` },
|
||||
{ ok: gaps.every(g => g >= 29500), what: `every send at least 30 s after the previous (min gap ${gaps.length ? s(Math.min(...gaps)) : 'n/a'})` },
|
||||
{ ok: held.length >= 1, what: `the miner said why it held a prepare (${held.length} lines)` },
|
||||
], { sends: sent.length, held: held.length, seed_changes: changes.length, blocks_by_the_cpu_producer: (() => { try { return (readFileSync(join(SCRATCH, 'cpu-miner.log'), 'utf8').match(/ACCEPTED block/g) || []).length; } catch { return 'n/a'; } })() });
|
||||
};
|
||||
|
||||
const names = ONLY.length ? ONLY : Object.keys(S);
|
||||
startNode({ IGNEUM_POW_EPOCH_BLOCKS: '60', IGNEUM_POW_EPOCH_LEAD: '20', IGNEUM_DEVNET_GENESIS_BITS: '0x1f100000' });
|
||||
await sleep(4000);
|
||||
for (const n of names) {
|
||||
if (!S[n]) { log(`unknown scenario ${n}`); continue; }
|
||||
log(`=== ${n}`);
|
||||
try { await S[n](); } catch (e) { log(`scenario ${n} threw: ${e.stack || e}`); results.push({ name: n, pass: false, checks: [{ ok: false, what: String(e) }], metrics: {} }); }
|
||||
await sleep(2000);
|
||||
}
|
||||
const report = { date: new Date().toISOString(), miner: MINER, node: NODE, scratch: SCRATCH, results };
|
||||
writeFileSync(join(SCRATCH, 'report.json'), JSON.stringify(report, null, 2));
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
process.exit(results.every(r => r.pass) ? 0 : 1);
|
||||
Loading…
Reference in a new issue