App engine: watchdog per card and per node, WORKER FAULT and mismatched= read from the miner; public GPU bench table and the reliability test harness
Review round 4 X21 and Josh's levers 5 and 6 (4 October 2026 evening): - src/watchdog.rs: CardWatch (no status line for 90 s, or hash rate 0 for 60 s while the node is synced: one restart, then the card is marked faulted with the reason in the UI and the log; the miner's own worker restarts suspend both rules until ready, so there are no double restarts; exit 43 counts as a watchdog restart) and NodeWatch (our node silent for 120 s is restarted in-process through the restart kind the remote jobs use, with a growing delay). State machines with no clock; 11 unit tests replay recorded STATUS and WORKER FAULT lines. - engine.rs reads STATUS mismatched=, faults= and the WORKER FAULT lines onto the card (faults, mismatched, message); a faulted card is not restarted until the user changes its settings or resumes mining; other cards keep mining. - site/miners.html (build.mjs, scrubbed like /bench, rows from site/miner-bench.json): card, generator version, best MH/s, MH per watt where measured, miner version, date, source, measured by the team or reported by the fleet. In the navigation beside the engineering log on every page and in the sitemap. One line says there is no other miner to compare with. - tools/reliability: fake-worker.mjs (the serve protocol, misbehaving on command), run.mjs (miner guards on a private test network, ports 29950+) and app-run.mjs (the app watchdog end to end, ports 29960+). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-josh
parent
33c430d93c
commit
f39e240d2c
17 changed files with 1423 additions and 30 deletions
|
|
@ -356,6 +356,8 @@ struct MinerSlot {
|
|||
prepared: bool, // the pack is exported (and the worker built) for the next start
|
||||
last_status: Option<Instant>,
|
||||
error_at: Option<Instant>,
|
||||
/// the per-card watchdog (src/watchdog.rs): one restart, then faulted
|
||||
watch: crate::watchdog::CardWatch,
|
||||
}
|
||||
|
||||
pub struct Engine {
|
||||
|
|
@ -416,6 +418,10 @@ pub struct Engine {
|
|||
last_stability: Instant,
|
||||
last_settings_save: Instant,
|
||||
last_error_event: Instant,
|
||||
/// the node watchdog (src/watchdog.rs): a silent node of ours is restarted
|
||||
node_watch: crate::watchdog::NodeWatch,
|
||||
/// the watchdogs' clock origin (they take seconds)
|
||||
t0: Instant,
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
|
|
@ -487,6 +493,8 @@ impl Engine {
|
|||
last_stability: now,
|
||||
last_settings_save: now,
|
||||
last_error_event: now - Duration::from_secs(600),
|
||||
node_watch: crate::watchdog::NodeWatch::new(),
|
||||
t0: now,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -494,6 +502,11 @@ impl Engine {
|
|||
self.shared.state.lock().unwrap()
|
||||
}
|
||||
|
||||
/// Seconds on the watchdogs' clock.
|
||||
fn secs(&self, now: Instant) -> f64 {
|
||||
now.duration_since(self.t0).as_secs_f64()
|
||||
}
|
||||
|
||||
pub fn run(mut self) {
|
||||
let setup_done = self.shared.settings.lock().unwrap().setup_done;
|
||||
self.shared.log(&format!(
|
||||
|
|
@ -608,6 +621,13 @@ impl Engine {
|
|||
self.shared.save_settings();
|
||||
self.st().mining.paused = false;
|
||||
self.shared.event("ok", "mining resumed");
|
||||
// a user action: faulted cards try again
|
||||
for m in self.miners.iter_mut() {
|
||||
if m.watch.faulted().is_some() {
|
||||
m.watch.event(0.0, crate::watchdog::Event::Reset);
|
||||
m.restart_at = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
if !self.running {
|
||||
self.start();
|
||||
}
|
||||
|
|
@ -616,6 +636,8 @@ impl Engine {
|
|||
self.shared.event("info", &format!("settings changed ({why}); the miners restart"));
|
||||
self.stop_miners("settings changed");
|
||||
for m in self.miners.iter_mut() {
|
||||
// a user action: a faulted card tries again
|
||||
m.watch.event(0.0, crate::watchdog::Event::Reset);
|
||||
m.restart_at = Some(Instant::now());
|
||||
}
|
||||
}
|
||||
|
|
@ -958,6 +980,7 @@ impl Engine {
|
|||
prepared: false,
|
||||
last_status: None,
|
||||
error_at: None,
|
||||
watch: crate::watchdog::CardWatch::new(),
|
||||
});
|
||||
}
|
||||
if self.miners.is_empty() {
|
||||
|
|
@ -1059,6 +1082,8 @@ impl Engine {
|
|||
self.miners[i].proc = Some(p);
|
||||
self.miners[i].restart_at = None;
|
||||
self.miners[i].last_status = None;
|
||||
let t = self.secs(Instant::now());
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::Started);
|
||||
}
|
||||
Err(e) => {
|
||||
self.shared.event("error", &format!("the miner could not start: {e}"));
|
||||
|
|
@ -1072,17 +1097,19 @@ impl Engine {
|
|||
if any {
|
||||
self.shared.log(&format!("stopping the miners ({why})"));
|
||||
}
|
||||
let t = self.secs(Instant::now());
|
||||
for m in self.miners.iter_mut() {
|
||||
if let Some(mut p) = m.proc.take() {
|
||||
p.write_stdin("quit\n");
|
||||
p.stop(8);
|
||||
}
|
||||
m.restart_at = None;
|
||||
m.watch.event(t, crate::watchdog::Event::Stopped);
|
||||
}
|
||||
let mut st = self.st();
|
||||
let paused = st.mining.paused;
|
||||
for c in st.mining.cards.iter_mut() {
|
||||
if c.enabled {
|
||||
if c.enabled && c.state != "faulted" {
|
||||
c.state = if paused { "off".into() } else { "waiting".into() };
|
||||
c.hash_now = 0.0;
|
||||
c.pid = 0;
|
||||
|
|
@ -1154,6 +1181,7 @@ impl Engine {
|
|||
if enabled {
|
||||
self.miners[pos].restart_at = Some(Instant::now());
|
||||
self.miners[pos].prepared = false;
|
||||
self.miners[pos].watch.event(0.0, crate::watchdog::Event::Reset);
|
||||
self.shared.event("info", &format!("{name}: {identities} identit{}; its worker restarts", if identities == 1 { "y" } else { "ies" }));
|
||||
} else {
|
||||
self.miners.remove(pos);
|
||||
|
|
@ -1477,13 +1505,7 @@ impl Engine {
|
|||
if self.node_external {
|
||||
self.shared.event("info", "remote job asked for a node restart, but the node is external; nothing done");
|
||||
} else {
|
||||
self.stop_miners("remote job: restart node");
|
||||
self.stop_node();
|
||||
self.node_restart_at = Some(Instant::now() + Duration::from_secs(3));
|
||||
for m in self.miners.iter_mut() {
|
||||
m.restart_at = Some(Instant::now());
|
||||
}
|
||||
self.st().node.message = "restart asked by a remote job".into();
|
||||
self.restart_node("restart asked by a remote job", Duration::from_secs(3));
|
||||
}
|
||||
}
|
||||
Action::RestartApp => {
|
||||
|
|
@ -1635,6 +1657,15 @@ impl Engine {
|
|||
st.node.message = "no answer from the RPC yet (still opening its database?)".into();
|
||||
}
|
||||
}
|
||||
// the node watchdog: our node silent for 120 s (no reading, nothing accepted) is restarted in-process
|
||||
if self.node.is_some() && !self.node_external && self.node_restart_at.is_none() {
|
||||
let silent_s = self.node_last_reading.map(|t| now.duration_since(t)).unwrap_or_else(|| self.node_started_at.elapsed()).as_secs_f64();
|
||||
if let Some(delay) = self.node_watch.tick(self.secs(now), true, silent_s, accepted_recent) {
|
||||
let n = self.node_watch.restarts_in_window();
|
||||
self.shared.event("error", &format!("no reading from the node for {} s; the watchdog restarts it in {delay} s (restart {n} in the last 10 minutes)", silent_s as u64));
|
||||
self.restart_node("no reading from the node for 120 s", Duration::from_secs(delay));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn tick_miners(&mut self, now: Instant) {
|
||||
|
|
@ -1648,12 +1679,17 @@ impl Engine {
|
|||
let code = p.exit_code.unwrap_or(-1);
|
||||
let tail = tail_of(&p.log_path, 3);
|
||||
self.miners[i].proc = None;
|
||||
let t = self.secs(now);
|
||||
let verdict = self.miners[i].watch.event(t, crate::watchdog::Event::Exited(code));
|
||||
if code == 42 {
|
||||
self.shared.event("build", "the hourly program changed; the worker restarts on the new one");
|
||||
if self.bins_worker_missing(card_idx) {
|
||||
self.miners[i].needs_rebuild = true;
|
||||
}
|
||||
self.miners[i].restart_at = Some(now);
|
||||
} else if verdict != crate::watchdog::Action::None {
|
||||
// exit 43: the miner gave up on its worker; once more, then the card is faulted
|
||||
self.watchdog_verdict(i, verdict, &tail);
|
||||
} else {
|
||||
self.miners[i].restarts += 1;
|
||||
let delay = jitter_secs(crate::platform::unix_now() + i as u64);
|
||||
|
|
@ -1669,17 +1705,34 @@ impl Engine {
|
|||
c.message = tail.last().cloned().unwrap_or_default();
|
||||
}
|
||||
}
|
||||
} else if let Some(t) = self.miners[i].last_status {
|
||||
if now.duration_since(t) > Duration::from_secs(self.shared.runtime.status_secs as u64 * 4) {
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
||||
if c.state == "mining" {
|
||||
c.message = "no status from the miner for a while".into();
|
||||
} else {
|
||||
// the watchdog: no status for 90 s, or a zero rate for 60 s while synced, is one restart, then faulted
|
||||
let t = self.secs(now);
|
||||
let verdict = self.miners[i].watch.tick(t, synced);
|
||||
if verdict != crate::watchdog::Action::None {
|
||||
if let Some(mut p) = self.miners[i].proc.take() {
|
||||
p.write_stdin("quit\n");
|
||||
p.stop(3);
|
||||
}
|
||||
self.watchdog_verdict(i, verdict, &[]);
|
||||
continue;
|
||||
}
|
||||
if let Some(t) = self.miners[i].last_status {
|
||||
if now.duration_since(t) > Duration::from_secs(self.shared.runtime.status_secs as u64 * 4) {
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
||||
if c.state == "mining" {
|
||||
c.message = "no status from the miner for a while".into();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if self.miners[i].watch.faulted().is_some() {
|
||||
// faulted by the watchdog: not restarted until the user changes the card's settings or resumes
|
||||
continue;
|
||||
}
|
||||
if self.job_hold {
|
||||
// a remote job has the GPU; the miners wait until it lets go (src/jobrun.rs)
|
||||
continue;
|
||||
|
|
@ -1711,6 +1764,58 @@ impl Engine {
|
|||
}
|
||||
}
|
||||
|
||||
/// Applies a watchdog verdict to slot `i` (its process already stopped or gone): a restart now with the reason on
|
||||
/// the card, or the card marked faulted with the reason in the UI and the log while the other cards keep mining.
|
||||
fn watchdog_verdict(&mut self, i: usize, verdict: crate::watchdog::Action, tail: &[String]) {
|
||||
use crate::watchdog::Action;
|
||||
let card_idx = self.miners[i].card;
|
||||
let name = self.st().mining.cards.get(card_idx).map(|c| c.name.clone()).unwrap_or_else(|| self.miners[i].label.clone());
|
||||
for l in tail {
|
||||
self.shared.log(&format!(" miner: {l}"));
|
||||
}
|
||||
match verdict {
|
||||
Action::Restart(reason) => {
|
||||
self.miners[i].restarts += 1;
|
||||
self.miners[i].restart_at = Some(Instant::now() + Duration::from_secs(1));
|
||||
self.shared.event("error", &format!("{name}: {reason}; the watchdog restarts the miner (restart {} of 1 before the card is marked faulted)", self.miners[i].watch.watchdog_restarts()));
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
||||
c.state = "restarting".into();
|
||||
c.restarts = self.miners[i].restarts;
|
||||
c.hash_now = 0.0;
|
||||
c.pid = 0;
|
||||
c.message = format!("watchdog: {reason}");
|
||||
}
|
||||
}
|
||||
Action::Fault(reason) => {
|
||||
self.miners[i].restart_at = None;
|
||||
self.shared.event("error", &format!("{name}: {reason}; the card is marked faulted and its miner is not restarted again (the other cards keep mining; change the card's settings or resume mining to try again)"));
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
||||
c.state = "faulted".into();
|
||||
c.hash_now = 0.0;
|
||||
c.pid = 0;
|
||||
c.restart_in_s = 0;
|
||||
c.message = format!("faulted: {reason}");
|
||||
}
|
||||
}
|
||||
Action::None => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops the miners and the node and schedules the node's start after `delay`; the miners follow once it is
|
||||
/// synced. The remote-job restart kind and the node watchdog share it.
|
||||
fn restart_node(&mut self, why: &str, delay: Duration) {
|
||||
self.stop_miners(why);
|
||||
self.stop_node();
|
||||
self.node_restart_at = Some(Instant::now() + delay);
|
||||
for m in self.miners.iter_mut() {
|
||||
m.restart_at = Some(Instant::now());
|
||||
}
|
||||
let mut st = self.st();
|
||||
st.node.state = "restarting".into();
|
||||
st.node.synced = false;
|
||||
st.node.message = format!("{why}; restart in {} s", delay.as_secs());
|
||||
}
|
||||
|
||||
fn bins_worker_missing(&self, card_idx: usize) -> bool {
|
||||
let st = self.st();
|
||||
match st.mining.cards.get(card_idx).map(|c| c.worker.as_str()) {
|
||||
|
|
@ -1944,7 +2049,7 @@ impl Engine {
|
|||
if synced && !was_synced {
|
||||
self.shared.event("ok", &format!("node synced: {blocks} blocks, {peers} peer(s)"));
|
||||
for m in self.miners.iter_mut() {
|
||||
if m.proc.is_none() && m.restart_at.is_none() {
|
||||
if m.proc.is_none() && m.restart_at.is_none() && m.watch.faulted().is_none() {
|
||||
m.restart_at = Some(now);
|
||||
}
|
||||
}
|
||||
|
|
@ -1970,19 +2075,38 @@ impl Engine {
|
|||
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
||||
c.message = "the node is not answering; the miner retries".into();
|
||||
}
|
||||
} else if text.contains("WORKER MISMATCH") || text.contains("worker error") || text.contains("worker exited") || text.contains("panicked") || text.contains("CUDA error") || text.contains("submit error") {
|
||||
} else if text.contains("WORKER MISMATCH") || text.contains("worker error") || text.contains("worker exited") || text.contains("worker killed by a guard") || text.contains("panicked") || text.contains("CUDA error") || text.contains("submit error") {
|
||||
let now = Instant::now();
|
||||
if now.duration_since(self.last_error_event) >= Duration::from_secs(30) {
|
||||
self.last_error_event = now;
|
||||
self.shared.event("error", &format!("{card_name}: {}", short(text, 160)));
|
||||
}
|
||||
self.miners[i].error_at = Some(now);
|
||||
if text.contains("worker exited") || text.contains("worker killed by a guard") {
|
||||
// the miner restarts its worker itself: the watchdog waits for `ready` instead of restarting the miner too
|
||||
let t = self.secs(now);
|
||||
let reason = short(text.split_once(' ').map(|(_, r)| r).unwrap_or(text), 160);
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart(&reason));
|
||||
}
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
||||
c.message = short(text, 160);
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if text.contains(" WORKER FAULT ") {
|
||||
// the miner's guards (interval, job time, cpu re-check, stall) killed the worker; it restarts it itself
|
||||
let reason = crate::watchdog::fault_reason(text).unwrap_or_else(|| short(text, 160));
|
||||
let t = self.secs(Instant::now());
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart(&reason));
|
||||
self.shared.event("error", &format!("{card_name}: worker fault: {}; the miner restarts the worker", short(&reason, 200)));
|
||||
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
||||
c.faults += 1;
|
||||
c.hash_now = 0.0;
|
||||
c.message = format!("worker fault: {}", short(&reason, 160));
|
||||
}
|
||||
return;
|
||||
}
|
||||
if text.contains(" ACCEPTED block") {
|
||||
self.last_accepted = Some(Instant::now());
|
||||
{
|
||||
|
|
@ -2009,27 +2133,33 @@ impl Engine {
|
|||
}
|
||||
drop(st);
|
||||
self.shared.event("block", &format!("block accepted by the node ({card_name})"));
|
||||
} else if text.contains(" STATUS '") {
|
||||
self.miners[i].last_status = Some(Instant::now());
|
||||
} else if let Some(s) = crate::watchdog::parse_status(text) {
|
||||
let now = Instant::now();
|
||||
self.miners[i].last_status = Some(now);
|
||||
let t = self.secs(now);
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::Status(&s));
|
||||
let avg = kv_f64(text, "hash").unwrap_or(0.0);
|
||||
// the v4 miner's interval rate: "now=12.34 MH/s wall (...)"; older miners: the average
|
||||
let now_rate = kv_f64(text, "now").unwrap_or(avg);
|
||||
let mut st = self.st();
|
||||
if let Some(c) = st.mining.cards.get_mut(card) {
|
||||
c.hash_avg = avg;
|
||||
c.hash_now = now_rate;
|
||||
// the v4 miner's interval rate: "now=12.34 MH/s wall (...)"; older miners: the average
|
||||
c.hash_now = s.hash_now;
|
||||
c.template_age_s = kv_f64(text, "template_age").unwrap_or(0.0);
|
||||
c.synced = kv(text, "synced") == Some("true");
|
||||
c.synced = s.synced;
|
||||
c.mismatched = s.mismatched;
|
||||
c.faults = c.faults.max(s.faults);
|
||||
c.last_status_age_s = 0.0;
|
||||
if c.state == "starting" || c.state == "ready" {
|
||||
c.state = "mining".into();
|
||||
}
|
||||
if c.state == "mining" {
|
||||
c.message = String::new();
|
||||
if c.state == "mining" && s.hash_now > 0.0 {
|
||||
c.message = if s.mismatched > 0 { format!("{} share(s) failed the CPU re-check this run", s.mismatched) } else { String::new() };
|
||||
}
|
||||
}
|
||||
} else if text.contains(" worker: ready ") {
|
||||
let prepare = text.contains(" prepare 1");
|
||||
let t = self.secs(Instant::now());
|
||||
self.miners[i].watch.event(t, crate::watchdog::Event::Ready);
|
||||
let mut st = self.st();
|
||||
if let Some(c) = st.mining.cards.get_mut(card) {
|
||||
c.ready = true;
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ mod update;
|
|||
mod jobs;
|
||||
mod jobrun;
|
||||
mod prover;
|
||||
mod watchdog;
|
||||
|
||||
use std::io::{BufRead, Write};
|
||||
use std::sync::mpsc::channel;
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ pub struct CardState {
|
|||
pub detail: String, // memory, cores
|
||||
pub device: String, // the worker's --device value (Windows)
|
||||
pub enabled: bool,
|
||||
pub state: String, // off | waiting | starting | ready | mining | restarting | failed
|
||||
pub state: String, // off | waiting | starting | ready | mining | restarting | failed | faulted (the watchdog gave up on it)
|
||||
pub hash_now: f64, // MH/s, the last interval
|
||||
pub hash_avg: f64, // MH/s since the start
|
||||
pub accepted: u64,
|
||||
|
|
@ -57,6 +57,10 @@ pub struct CardState {
|
|||
pub pid: u32,
|
||||
pub last_status_age_s: f64,
|
||||
pub message: String,
|
||||
/// `WORKER FAULT` lines seen (the miner killed and restarted its worker) and the miner's own `faults=` count
|
||||
pub faults: u64,
|
||||
/// shares that failed the miner's CPU re-check this run (`mismatched=` on the STATUS line)
|
||||
pub mismatched: u64,
|
||||
// power and heat (NVIDIA through nvidia-smi; 0 = unknown)
|
||||
pub power_w: f64, // draw now
|
||||
pub power_limit_w: f64, // the limit in force
|
||||
|
|
|
|||
482
app/igneum-app/src/watchdog.rs
Normal file
482
app/igneum-app/src/watchdog.rs
Normal file
|
|
@ -0,0 +1,482 @@
|
|||
//! The engine's watchdog, as state machines with no clock and no process (seconds come from the caller), so the
|
||||
//! rules can be unit-tested with recorded miner lines.
|
||||
//!
|
||||
//! Per card (`CardWatch`): a miner that prints no status line for 90 s, or reports a hash rate of 0 for 60 s while
|
||||
//! the node is synced, is restarted once; when that recurs before five minutes of healthy status, the card is marked
|
||||
//! faulted with the reason, its miner is not restarted again, and the other cards keep mining. The miner's own
|
||||
//! worker restarts (`WORKER FAULT`, `worker exited`) suspend both rules until the worker is ready again, so the app
|
||||
//! never restarts a miner that is already restarting its worker (no double restarts); if the worker is not back
|
||||
//! within 180 s the app steps in. Exit code 43 (the miner gave up on its worker after three guard trips) counts
|
||||
//! like a watchdog restart: once, then faulted.
|
||||
//!
|
||||
//! Per node (`NodeWatch`): a node of ours that answers no `watch` reading for 120 s is restarted by the app, with a
|
||||
//! growing delay when it repeats inside ten minutes.
|
||||
//!
|
||||
//! Review round 4, X21 (4 October 2026): the app now reads `mismatched=` and `faults=` from STATUS lines and the
|
||||
//! `WORKER FAULT` lines, and shows them on the card.
|
||||
|
||||
/// No status line from a running miner for this long: restart it.
|
||||
pub const NO_STATUS_S: f64 = 90.0;
|
||||
/// Hash rate 0 while the node is synced and the worker is ready for this long: restart the miner.
|
||||
pub const ZERO_RATE_S: f64 = 60.0;
|
||||
/// The miner is restarting its own worker: the app waits this long for `ready` before it steps in.
|
||||
pub const WORKER_RESTART_GRACE_S: f64 = 180.0;
|
||||
/// Continuous healthy status (rate above 0) that renews the one-restart budget.
|
||||
pub const HEALTHY_RESET_S: f64 = 300.0;
|
||||
/// `igneum-miner` exit code when its guards gave up on the worker (three trips in ten minutes).
|
||||
pub const MINER_GAVE_UP_CODE: i32 = 43;
|
||||
/// No `watch` reading from our node for this long: restart it.
|
||||
pub const NODE_SILENT_S: f64 = 120.0;
|
||||
/// Node restarts inside this window grow the delay before the next one.
|
||||
pub const NODE_WINDOW_S: f64 = 600.0;
|
||||
|
||||
/// What the watchdog reads from one `STATUS` line of `igneum-miner`.
|
||||
#[derive(Debug, Clone, PartialEq, Default)]
|
||||
pub struct Status {
|
||||
/// MH/s of the last interval (`now=`; older miners: the average `hash=`)
|
||||
pub hash_now: f64,
|
||||
pub mismatched: u64,
|
||||
pub faults: u64,
|
||||
pub restarts: u64,
|
||||
pub synced: bool,
|
||||
}
|
||||
|
||||
/// Parses a miner STATUS line; None for any other line.
|
||||
pub fn parse_status(text: &str) -> Option<Status> {
|
||||
if !text.contains(" STATUS '") {
|
||||
return None;
|
||||
}
|
||||
let avg = kv_f64(text, "hash").unwrap_or(0.0);
|
||||
Some(Status {
|
||||
hash_now: kv_f64(text, "now").unwrap_or(avg),
|
||||
mismatched: kv_u64(text, "mismatched").unwrap_or(0),
|
||||
faults: kv_u64(text, "faults").unwrap_or(0),
|
||||
restarts: kv_u64(text, "restarts").unwrap_or(0),
|
||||
synced: kv(text, "synced") == Some("true"),
|
||||
})
|
||||
}
|
||||
|
||||
/// The reason text of a `WORKER FAULT` line, without the timestamp and the trailing restart note.
|
||||
pub fn fault_reason(text: &str) -> Option<String> {
|
||||
let i = text.find("WORKER FAULT ")?;
|
||||
let rest = &text["WORKER FAULT ".len() + i..];
|
||||
let end = rest.find("; restarting the worker").or_else(|| rest.find("; killing the worker")).unwrap_or(rest.len());
|
||||
Some(rest[..end].trim().to_string())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Event<'a> {
|
||||
/// The app started the miner process.
|
||||
Started,
|
||||
/// The worker said ready.
|
||||
Ready,
|
||||
/// A STATUS line.
|
||||
Status(&'a Status),
|
||||
/// The miner killed its worker (a guard) or saw it exit; it restarts the worker itself.
|
||||
WorkerRestart(&'a str),
|
||||
/// The miner process ended with this code.
|
||||
Exited(i32),
|
||||
/// The app stopped the miner on purpose (pause, settings, node restart).
|
||||
Stopped,
|
||||
/// The user changed the card's settings: a faulted card may try again.
|
||||
Reset,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum Action {
|
||||
None,
|
||||
/// Stop the miner and start it again now, for this reason.
|
||||
Restart(String),
|
||||
/// Mark the card faulted with this reason; do not restart its miner.
|
||||
Fault(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct CardWatch {
|
||||
started_s: Option<f64>,
|
||||
last_status_s: Option<f64>,
|
||||
ready: bool,
|
||||
zero_since: Option<f64>,
|
||||
healthy_since: Option<f64>,
|
||||
worker_restart_since: Option<f64>,
|
||||
/// app-level restarts without five healthy minutes since
|
||||
restarts: u32,
|
||||
faulted: Option<String>,
|
||||
last_fault: String,
|
||||
}
|
||||
|
||||
impl CardWatch {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn faulted(&self) -> Option<&str> {
|
||||
self.faulted.as_deref()
|
||||
}
|
||||
|
||||
pub fn watchdog_restarts(&self) -> u32 {
|
||||
self.restarts
|
||||
}
|
||||
|
||||
fn arm(&mut self, now_s: f64) {
|
||||
self.started_s = Some(now_s);
|
||||
self.last_status_s = None;
|
||||
self.ready = false;
|
||||
self.zero_since = None;
|
||||
self.healthy_since = None;
|
||||
self.worker_restart_since = None;
|
||||
}
|
||||
|
||||
fn escalate(&mut self, reason: String) -> Action {
|
||||
self.started_s = None;
|
||||
self.last_status_s = None;
|
||||
self.ready = false;
|
||||
self.zero_since = None;
|
||||
self.healthy_since = None;
|
||||
self.worker_restart_since = None;
|
||||
if self.restarts >= 1 {
|
||||
let r = format!("{reason} (restarted once already)");
|
||||
self.faulted = Some(r.clone());
|
||||
Action::Fault(r)
|
||||
} else {
|
||||
self.restarts += 1;
|
||||
Action::Restart(reason)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn event(&mut self, now_s: f64, ev: Event<'_>) -> Action {
|
||||
match ev {
|
||||
Event::Started => {
|
||||
self.arm(now_s);
|
||||
Action::None
|
||||
}
|
||||
Event::Ready => {
|
||||
self.ready = true;
|
||||
self.worker_restart_since = None;
|
||||
Action::None
|
||||
}
|
||||
Event::Status(s) => {
|
||||
self.last_status_s = Some(now_s);
|
||||
if s.hash_now > 0.0 {
|
||||
self.zero_since = None;
|
||||
let since = *self.healthy_since.get_or_insert(now_s);
|
||||
if now_s - since >= HEALTHY_RESET_S {
|
||||
self.restarts = 0;
|
||||
}
|
||||
} else {
|
||||
self.healthy_since = None;
|
||||
if self.ready && self.worker_restart_since.is_none() {
|
||||
self.zero_since.get_or_insert(now_s);
|
||||
}
|
||||
}
|
||||
Action::None
|
||||
}
|
||||
Event::WorkerRestart(reason) => {
|
||||
self.last_fault = reason.to_string();
|
||||
self.worker_restart_since = Some(now_s);
|
||||
self.ready = false;
|
||||
self.zero_since = None;
|
||||
self.healthy_since = None;
|
||||
Action::None
|
||||
}
|
||||
Event::Exited(code) => {
|
||||
let running = self.started_s.is_some();
|
||||
self.started_s = None;
|
||||
if code == MINER_GAVE_UP_CODE && running {
|
||||
let why = if self.last_fault.is_empty() { "its guards tripped three times in ten minutes".to_string() } else { self.last_fault.clone() };
|
||||
self.escalate(format!("the miner gave up on its worker: {why}"))
|
||||
} else {
|
||||
Action::None
|
||||
}
|
||||
}
|
||||
Event::Stopped => {
|
||||
self.started_s = None;
|
||||
self.last_status_s = None;
|
||||
self.ready = false;
|
||||
self.zero_since = None;
|
||||
self.worker_restart_since = None;
|
||||
Action::None
|
||||
}
|
||||
Event::Reset => {
|
||||
*self = Self::default();
|
||||
Action::None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Called every engine tick while the miner process is alive.
|
||||
pub fn tick(&mut self, now_s: f64, node_synced: bool) -> Action {
|
||||
if self.faulted.is_some() {
|
||||
return Action::None;
|
||||
}
|
||||
let Some(started) = self.started_s else { return Action::None };
|
||||
if let Some(t) = self.worker_restart_since {
|
||||
// the miner is restarting its worker: its own guards own the card until the worker is ready
|
||||
if now_s - t > WORKER_RESTART_GRACE_S {
|
||||
return self.escalate(format!("the worker did not come back within {} s of the miner restarting it ({})", WORKER_RESTART_GRACE_S as u64, self.last_fault));
|
||||
}
|
||||
return Action::None;
|
||||
}
|
||||
let last = self.last_status_s.unwrap_or(started);
|
||||
if now_s - last > NO_STATUS_S {
|
||||
return self.escalate(format!("no status line from the miner for {} s", NO_STATUS_S as u64));
|
||||
}
|
||||
if !node_synced {
|
||||
// a zero rate while the node syncs is expected; the timer starts again once it is synced
|
||||
self.zero_since = None;
|
||||
}
|
||||
if node_synced && self.ready {
|
||||
if let Some(z) = self.zero_since {
|
||||
if now_s - z >= ZERO_RATE_S {
|
||||
return self.escalate(format!("hash rate 0 for {} s while the node is synced", ZERO_RATE_S as u64));
|
||||
}
|
||||
}
|
||||
}
|
||||
Action::None
|
||||
}
|
||||
}
|
||||
|
||||
/// The node watchdog: no reading for `NODE_SILENT_S` restarts our node, with a growing delay when it repeats.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct NodeWatch {
|
||||
restarts_s: Vec<f64>,
|
||||
}
|
||||
|
||||
impl NodeWatch {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// `silent_s`: seconds since the last reading (or since the node started, when it never answered). Returns the
|
||||
/// delay in seconds before the restart when one is due.
|
||||
pub fn tick(&mut self, now_s: f64, ours: bool, silent_s: f64, accepted_recent: bool) -> Option<u64> {
|
||||
if !ours || accepted_recent || silent_s < NODE_SILENT_S {
|
||||
return None;
|
||||
}
|
||||
self.restarts_s.retain(|t| now_s - *t <= NODE_WINDOW_S);
|
||||
self.restarts_s.push(now_s);
|
||||
let n = self.restarts_s.len();
|
||||
Some((3u64.saturating_mul(1u64 << (n.saturating_sub(1) * 2).min(8))).min(300))
|
||||
}
|
||||
|
||||
pub fn restarts_in_window(&self) -> usize {
|
||||
self.restarts_s.len()
|
||||
}
|
||||
}
|
||||
|
||||
fn kv<'a>(line: &'a str, key: &str) -> Option<&'a str> {
|
||||
let pat = format!(" {key}=");
|
||||
let i = line.find(&pat)? + pat.len();
|
||||
let rest = &line[i..];
|
||||
let end = rest.find(|c: char| c == ' ' || c == ',' || c == ')' || c == ';').unwrap_or(rest.len());
|
||||
Some(&rest[..end])
|
||||
}
|
||||
|
||||
fn kv_u64(line: &str, key: &str) -> Option<u64> {
|
||||
kv(line, key)?.parse().ok()
|
||||
}
|
||||
|
||||
fn kv_f64(line: &str, key: &str) -> Option<f64> {
|
||||
kv(line, key)?.trim_end_matches('s').parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
// Lines as igneum-miner 0.3.x prints them (devnet v4, 4 October 2026; identities and labels shortened).
|
||||
const STATUS_OK: &str = "1791138616.597 STATUS 'win-1' [worker]: 70s jobs=486 accepted=3 rejected=0 mismatched=0 extra=0 rate=0.04 blocks/s hash=123.90 MH/s wall (124.20 MH/s inside jobs) now=124.10 MH/s wall (124.30 MH/s inside jobs, 70 jobs, seed walk 0 calls) template_age=0.31s synced=true idle=0.2% (last 10s: 0.1%) queued=2 restarts=0 faults=0 identities=8 accepted_by_identity=1/0/1/0/0/1/0/0";
|
||||
const STATUS_ZERO: &str = "1791138626.597 STATUS 'win-1' [worker]: 80s jobs=486 accepted=3 rejected=0 mismatched=0 extra=0 rate=0.04 blocks/s hash=108.41 MH/s wall (124.20 MH/s inside jobs) now=0.00 MH/s wall (0.00 MH/s inside jobs, 0 jobs, seed walk 0 calls) template_age=0.31s synced=true idle=12.5% (last 10s: 100.0%) queued=2 restarts=0 faults=0 identities=8 accepted_by_identity=1/0/1/0/0/1/0/0";
|
||||
const STATUS_MISMATCH: &str = "1791138636.597 STATUS 'win-1' [worker]: 90s jobs=500 accepted=3 rejected=0 mismatched=3 extra=0 rate=0.03 blocks/s hash=110.00 MH/s wall (124.20 MH/s inside jobs) now=124.00 MH/s wall (124.30 MH/s inside jobs, 70 jobs, seed walk 0 calls) template_age=0.31s synced=true idle=0.2% (last 10s: 0.1%) queued=2 restarts=1 faults=1 identities=8 accepted_by_identity=1/0/1/0/0/1/0/0";
|
||||
const FAULT_LINE: &str = "1791138640.100 WORKER FAULT the last 10 s ran at 4300000.00 MH/s inside jobs against 3.30 MH/s over this worker's healthy intervals, over 10x: the worker is not running its kernel; restarting the worker (fault 1)";
|
||||
|
||||
#[test]
|
||||
fn parses_status_and_fault_lines() {
|
||||
let s = parse_status(STATUS_OK).unwrap();
|
||||
assert_eq!(s, Status { hash_now: 124.10, mismatched: 0, faults: 0, restarts: 0, synced: true });
|
||||
let m = parse_status(STATUS_MISMATCH).unwrap();
|
||||
assert_eq!((m.mismatched, m.faults, m.restarts), (3, 1, 1));
|
||||
assert_eq!(parse_status(STATUS_ZERO).unwrap().hash_now, 0.0);
|
||||
assert_eq!(parse_status("1791138640.100 worker: ready metal Apple_M5_Max prepare 1"), None);
|
||||
assert_eq!(fault_reason(FAULT_LINE).unwrap(), "the last 10 s ran at 4300000.00 MH/s inside jobs against 3.30 MH/s over this worker's healthy intervals, over 10x: the worker is not running its kernel");
|
||||
assert_eq!(fault_reason(STATUS_OK), None);
|
||||
}
|
||||
|
||||
fn healthy(w: &mut CardWatch, from: f64, to: f64) {
|
||||
let s = parse_status(STATUS_OK).unwrap();
|
||||
let mut t = from;
|
||||
while t <= to {
|
||||
assert_eq!(w.event(t, Event::Status(&s)), Action::None);
|
||||
assert_eq!(w.tick(t, true), Action::None);
|
||||
t += 10.0;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn healthy_miner_is_left_alone() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
healthy(&mut w, 10.0, 3600.0);
|
||||
assert_eq!(w.watchdog_restarts(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_rate_restarts_once_then_faults() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
healthy(&mut w, 10.0, 100.0);
|
||||
let z = parse_status(STATUS_ZERO).unwrap();
|
||||
for t in [110.0, 120.0, 130.0, 140.0, 150.0, 160.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
assert_eq!(w.tick(t, true), Action::None, "under 60 s at {t}");
|
||||
}
|
||||
w.event(170.0, Event::Status(&z));
|
||||
let a = w.tick(170.0, true);
|
||||
assert!(matches!(a, Action::Restart(ref r) if r.contains("hash rate 0 for 60 s")), "{a:?}");
|
||||
// the app restarted it; still zero: faulted, not restarted again
|
||||
w.event(172.0, Event::Started);
|
||||
w.event(174.0, Event::Ready);
|
||||
for t in [180.0, 190.0, 200.0, 210.0, 220.0, 230.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
assert_eq!(w.tick(t, true), Action::None);
|
||||
}
|
||||
w.event(240.0, Event::Status(&z));
|
||||
let a = w.tick(240.0, true);
|
||||
assert!(matches!(a, Action::Fault(ref r) if r.contains("restarted once already")), "{a:?}");
|
||||
assert!(w.faulted().is_some());
|
||||
// faulted stays: no more actions
|
||||
w.event(250.0, Event::Status(&z));
|
||||
assert_eq!(w.tick(260.0, true), Action::None);
|
||||
// the user changed the card's settings: a fresh start
|
||||
w.event(300.0, Event::Reset);
|
||||
assert!(w.faulted().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_rate_only_counts_with_a_synced_node_and_a_ready_worker() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
let z = parse_status(STATUS_ZERO).unwrap();
|
||||
// not ready yet (program loading): no zero timer
|
||||
for t in [10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
assert_eq!(w.tick(t, true), Action::None);
|
||||
}
|
||||
w.event(82.0, Event::Ready);
|
||||
// node not synced: no zero timer either
|
||||
for t in [90.0, 100.0, 110.0, 120.0, 130.0, 140.0, 150.0, 160.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
assert_eq!(w.tick(t, false), Action::None);
|
||||
}
|
||||
assert_eq!(w.tick(170.0, true), Action::None, "the timer starts at the first zero status after ready");
|
||||
for t in [180.0, 190.0, 200.0, 210.0, 220.0, 230.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
w.tick(t, true);
|
||||
}
|
||||
assert!(matches!(w.tick(240.0, true), Action::Restart(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_status_for_90s_restarts() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
healthy(&mut w, 10.0, 60.0);
|
||||
// the miner goes silent (a stopped process, a hung RPC)
|
||||
assert_eq!(w.tick(149.0, true), Action::None);
|
||||
let a = w.tick(151.0, true);
|
||||
assert!(matches!(a, Action::Restart(ref r) if r.contains("no status line")), "{a:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_status_from_the_start() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
assert_eq!(w.tick(89.0, true), Action::None);
|
||||
assert!(matches!(w.tick(91.0, true), Action::Restart(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_miners_own_worker_restart_is_not_doubled() {
|
||||
// The gfx1036 fault: the miner prints WORKER FAULT, kills the worker, restarts it 2 s later; STATUS lines in
|
||||
// between say now=0. The app must not restart the miner on top of that.
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
healthy(&mut w, 10.0, 570.0);
|
||||
w.event(580.0, Event::WorkerRestart(&fault_reason(FAULT_LINE).unwrap()));
|
||||
let z = parse_status(STATUS_ZERO).unwrap();
|
||||
for t in [580.0, 590.0, 600.0, 610.0, 620.0, 630.0, 640.0, 650.0, 660.0] {
|
||||
w.event(t, Event::Status(&z));
|
||||
assert_eq!(w.tick(t, true), Action::None, "the miner owns the restart at {t}");
|
||||
}
|
||||
w.event(665.0, Event::Ready);
|
||||
healthy(&mut w, 670.0, 900.0);
|
||||
assert_eq!(w.watchdog_restarts(), 0, "no app restart happened");
|
||||
// a worker restart that never comes back: the app steps in after the grace
|
||||
w.event(910.0, Event::WorkerRestart("no job completed for 60 s with 2 queued"));
|
||||
for t in (920..=1080).step_by(10) {
|
||||
w.event(t as f64, Event::Status(&z));
|
||||
assert_eq!(w.tick(t as f64, true), Action::None);
|
||||
}
|
||||
let a = w.tick(1091.0, true);
|
||||
assert!(matches!(a, Action::Restart(ref r) if r.contains("did not come back within 180 s")), "{a:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exit_43_once_then_faulted() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
healthy(&mut w, 10.0, 60.0);
|
||||
w.event(70.0, Event::WorkerRestart("cpu re-check: 3 consecutive mismatches (mismatched=3 in this run): the worker computes a wrong program"));
|
||||
let a = w.event(75.0, Event::Exited(43));
|
||||
assert!(matches!(a, Action::Restart(ref r) if r.contains("gave up") && r.contains("wrong program")), "{a:?}");
|
||||
w.event(80.0, Event::Started);
|
||||
let a = w.event(300.0, Event::Exited(43));
|
||||
assert!(matches!(a, Action::Fault(_)), "{a:?}");
|
||||
// an ordinary crash is the engine's own jittered restart, not the watchdog's
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
assert_eq!(w.event(5.0, Event::Exited(1)), Action::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn five_healthy_minutes_renew_the_budget() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
assert!(matches!(w.tick(100.0, true), Action::Restart(_)));
|
||||
w.event(101.0, Event::Started);
|
||||
w.event(103.0, Event::Ready);
|
||||
healthy(&mut w, 110.0, 420.0);
|
||||
assert_eq!(w.watchdog_restarts(), 0);
|
||||
// a second incident later is again a restart, not a fault
|
||||
assert!(matches!(w.tick(520.0, true), Action::Restart(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_deliberate_stop_is_not_a_fault() {
|
||||
let mut w = CardWatch::new();
|
||||
w.event(0.0, Event::Started);
|
||||
w.event(2.0, Event::Ready);
|
||||
w.event(30.0, Event::Stopped);
|
||||
assert_eq!(w.tick(500.0, true), Action::None);
|
||||
assert_eq!(w.event(500.0, Event::Exited(0)), Action::None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_watch_restarts_a_silent_node_with_growing_delay() {
|
||||
let mut n = NodeWatch::new();
|
||||
assert_eq!(n.tick(100.0, true, 119.0, false), None);
|
||||
assert_eq!(n.tick(100.0, false, 500.0, false), None, "an external node is never restarted");
|
||||
assert_eq!(n.tick(100.0, true, 500.0, true), None, "our block was accepted in the last minute: the node is alive");
|
||||
assert_eq!(n.tick(100.0, true, 120.0, false), Some(3));
|
||||
assert_eq!(n.tick(300.0, true, 120.0, false), Some(12));
|
||||
assert_eq!(n.tick(500.0, true, 120.0, false), Some(48));
|
||||
assert_eq!(n.restarts_in_window(), 3);
|
||||
assert_eq!(n.tick(2000.0, true, 120.0, false), Some(3), "the window passed");
|
||||
}
|
||||
}
|
||||
|
|
@ -340,12 +340,12 @@
|
|||
var tb = $('d-cards');
|
||||
if (!on.length) tb.innerHTML = '<div class="empty">' + (m.cards.length ? 'No card switched on. Open settings to pick one.' : 'No GPU worker. The node runs on its own.') + '</div>';
|
||||
else tb.innerHTML = on.map(function (cd) {
|
||||
var cls = cd.state === 'mining' ? 'mining' : (cd.state === 'failed' || cd.state === 'restarting') ? 'bad' : '';
|
||||
var cls = cd.state === 'mining' ? 'mining' : (cd.state === 'failed' || cd.state === 'restarting' || cd.state === 'faulted') ? 'bad' : '';
|
||||
var word = cd.state === 'restarting' ? ('restart in ' + cd.restart_in_s + ' s') : cd.state;
|
||||
var extra = cd.ids.length ? '<span>id <b>' + esc(cd.ids.join(' ')) + '</b></span>' : '';
|
||||
return '<div class="gpu-tile' + (cd.state === 'mining' ? '' : ' idle') + '"><div class="n">' + esc(cd.name) + ' <span class="kind ' + esc(cd.kind) + '">' + kindWord(cd.kind) + '</span></div>' +
|
||||
'<div class="h">' + cd.hash_now.toFixed(1) + '<span class="unit">MH/s</span></div>' +
|
||||
'<div class="m"><span>blocks <b>' + cd.accepted + '</b>' + (cd.rejected ? ' <span class="dim">/ ' + cd.rejected + ' rejected</span>' : '') + '</span><span>avg <b>' + cd.hash_avg.toFixed(1) + '</b></span><span>identities <b>' + cd.identities + '</b></span>' + extra + '</div>' +
|
||||
'<div class="m"><span>blocks <b>' + cd.accepted + '</b>' + (cd.rejected ? ' <span class="dim">/ ' + cd.rejected + ' rejected</span>' : '') + '</span><span>avg <b>' + cd.hash_avg.toFixed(1) + '</b></span><span>identities <b>' + cd.identities + '</b></span>' + extra + (cd.faults ? '<span class="dim">worker faults <b>' + cd.faults + '</b></span>' : '') + (cd.mismatched ? '<span class="dim">re-check misses <b>' + cd.mismatched + '</b></span>' : '') + '</div>' +
|
||||
telemetryHtml(cd) +
|
||||
'<span class="st ' + cls + '"><i></i>' + esc(word) + (cd.prepared ? ' · next program ready' : '') + '</span>' + (cd.message ? '<div class="msg">' + esc(cd.message) + '</div>' : '') + '</div>';
|
||||
}).join('');
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ footer{border-top:1px solid var(--line);padding-block:32px 48px;font-size:13px;c
|
|||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<header><a class="brand" href="/"><svg viewBox="0 0 100 100" width="26" height="26" aria-hidden="true"><polygon points="50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34" fill="#F2541B"></polygon><polygon points="50,42 59,58 50,82 41,58" fill="#0C0C0E"></polygon></svg><b>IGNEUM</b></a><nav><a href="/">Home</a><a href="/litepaper">Litepaper</a><a href="/live">Live devnet</a><a href="/bench">Engineering log</a><a href="/evidence">Evidence</a><a href="https://github.com/igneum-network/spec" rel="noopener">GitHub, spec and vectors</a></nav></header>
|
||||
<header><a class="brand" href="/"><svg viewBox="0 0 100 100" width="26" height="26" aria-hidden="true"><polygon points="50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34" fill="#F2541B"></polygon><polygon points="50,42 59,58 50,82 41,58" fill="#0C0C0E"></polygon></svg><b>IGNEUM</b></a><nav><a href="/">Home</a><a href="/litepaper">Litepaper</a><a href="/live">Live devnet</a><a href="/bench">Engineering log</a><a href="/miners">Miners</a><a href="/evidence">Evidence</a><a href="https://github.com/igneum-network/spec" rel="noopener">GitHub, spec and vectors</a></nav></header>
|
||||
<h1>Engineering log</h1>
|
||||
<p class="note">Every measurement the project has made, newest at the bottom, written by the people and agents who ran it, with the commands and hardware. Prototype numbers are not mining numbers and say so.</p>
|
||||
<div class="layout">
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ footer{border-top:1px solid var(--line);padding-block:32px 48px;font-size:13px;c
|
|||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<header><a class="brand" href="/">${mark}<b>IGNEUM</b></a><nav><a href="/">Home</a><a href="/litepaper">Litepaper</a><a href="/live">Live devnet</a><a href="/bench">Engineering log</a><a href="/evidence">Evidence</a><a href="https://github.com/igneum-network/spec" rel="noopener">GitHub, spec and vectors</a></nav></header>
|
||||
<header><a class="brand" href="/">${mark}<b>IGNEUM</b></a><nav><a href="/">Home</a><a href="/litepaper">Litepaper</a><a href="/live">Live devnet</a><a href="/bench">Engineering log</a><a href="/miners">Miners</a><a href="/evidence">Evidence</a><a href="https://github.com/igneum-network/spec" rel="noopener">GitHub, spec and vectors</a></nav></header>
|
||||
<h1>${esc(heading)}</h1>
|
||||
<p class="note">${note}</p>
|
||||
<div class="layout">
|
||||
|
|
@ -207,4 +207,31 @@ if (existsSync(join(docs, 'bench-log.md'))) {
|
|||
writeFileSync(jp, JSON.stringify(j, null, 2) + '\n');
|
||||
built.push('journey.json (' + j.log.length + ' entries)');
|
||||
}
|
||||
// the public bench table (/miners): one row per card, generator version and miner version, from site/miner-bench.json
|
||||
// (the bench-log numbers that exist today, and rows later jobs append); rendered through the same scrub as /bench
|
||||
{
|
||||
const bj = JSON.parse(readFileSync(join(here, 'miner-bench.json'), 'utf8'));
|
||||
const rows = bj.rows.slice().sort((a, b) => (a.card < b.card ? -1 : a.card > b.card ? 1 : a.generator < b.generator ? -1 : a.generator > b.generator ? 1 : b.mh_s - a.mh_s));
|
||||
const fmt = (n) => Number(n).toLocaleString('en-GB', { maximumFractionDigits: 1 });
|
||||
const cell = (r) => [
|
||||
r.card, r.generator, fmt(r.mh_s), r.mh_per_w == null ? 'not measured' : fmt(r.mh_per_w), r.miner, r.date, r.source, r.by + (r.note ? '. ' + r.note : ''),
|
||||
];
|
||||
const table = '<div class="tbl"><table><thead><tr>' + ['Card', 'Generator', 'Best MH/s', 'MH per watt', 'Miner', 'Date', 'Source', 'Who measured it'].map(h => `<th>${h}</th>`).join('') + '</tr></thead><tbody>' +
|
||||
rows.map(r => '<tr>' + cell(r).map(c => `<td>${esc(String(c))}</td>`).join('') + '</tr>').join('') + '</tbody></table></div>';
|
||||
const body = scrubBench([
|
||||
'<h2 id="table">The table</h2>',
|
||||
'<p>One row per card, generator version and miner version. The rate is the best one measured. Integrated GPUs are not listed. Prototype rows are bench numbers from before the devnet and say so in the miner column.</p>',
|
||||
table,
|
||||
'<h2 id="how">How a row gets here</h2>',
|
||||
'<p>Every row names the engineering log entry or the job it came from. "Measured by the team" means our own hardware and our own log. "Reported by the fleet" means a machine we do not own, read from the status lines its miner uploads.</p>',
|
||||
'<p>MH per watt needs the card\'s power draw during the run. The app reads it on NVIDIA cards through the driver. Rows get the figure when a run records it.</p>',
|
||||
'<p>There is no other Igneum miner to compare with yet, so this table compares cards, not miners.</p>',
|
||||
`<p>Rows: ${rows.length}. Source file: <code>site/miner-bench.json</code> in the repository.</p>`,
|
||||
].join('\n'));
|
||||
const toc = [{ lvl: 2, t: 'The table', id: 'table' }, { lvl: 2, t: 'How a row gets here', id: 'how' }];
|
||||
writeFileSync(join(here, 'miners.html'), page('Igneum GPU bench table', 'Measured Igneum hash rates per GPU: card, generator version, best MH/s, MH per watt where measured, miner version, date and the log entry each number came from.', body, toc,
|
||||
'Measured hash rates per card on the Igneum lottery hash, with the generator version, the miner version, the date and the log entry behind each number.',
|
||||
{ path: '/miners', heading: 'GPU bench table' }));
|
||||
built.push('miners.html (' + rows.length + ' rows)');
|
||||
}
|
||||
console.log('built: ' + built.join(', '));
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ footer{border-top:1px solid var(--line);padding-block:32px 48px;font-size:13px;c
|
|||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<header><a class="brand" href="/"><svg viewBox="0 0 100 100" width="26" height="26" aria-hidden="true"><polygon points="50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34" fill="#F2541B"></polygon><polygon points="50,42 59,58 50,82 41,58" fill="#0C0C0E"></polygon></svg><b>IGNEUM</b></a><nav><a href="/">Home</a><a href="/litepaper">Litepaper</a><a href="/live">Live devnet</a><a href="/bench">Engineering log</a><a href="/evidence" aria-current="page">Evidence</a><a href="https://github.com/igneum-network/spec" rel="noopener">GitHub, spec and vectors</a></nav></header>
|
||||
<header><a class="brand" href="/"><svg viewBox="0 0 100 100" width="26" height="26" aria-hidden="true"><polygon points="50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34" fill="#F2541B"></polygon><polygon points="50,42 59,58 50,82 41,58" fill="#0C0C0E"></polygon></svg><b>IGNEUM</b></a><nav><a href="/">Home</a><a href="/litepaper">Litepaper</a><a href="/live">Live devnet</a><a href="/bench">Engineering log</a><a href="/miners">Miners</a><a href="/evidence" aria-current="page">Evidence</a><a href="https://github.com/igneum-network/spec" rel="noopener">GitHub, spec and vectors</a></nav></header>
|
||||
<h1>Evidence</h1>
|
||||
<p class="note">Every claim the homepage and the litepaper make, one row each, with one of five labels: <b>designed</b> (a decision, no code), <b>implemented</b> (code with passing test vectors), <b>tested by the team</b> (measured by the project on a named machine, in the engineering log), <b>reproduced externally</b> (a third party ran the published command and got the published result) and <b>reviewed independently</b> (a named outside reviewer published a finding on that version). Nothing on this chain has been reproduced externally or reviewed independently; every row says so. A label belongs to the exact version in the row, and an audit of one version never covers a newer one. The 12-node cloud network of 4 October 2026 is the project's own, so its rows are tested by the team, not reproduced externally.</p>
|
||||
<div class="labels">
|
||||
|
|
|
|||
|
|
@ -246,6 +246,7 @@ footer .wrap{padding-block:48px 32px}
|
|||
<a href="#economics">Economics</a>
|
||||
<a href="/litepaper">Litepaper</a>
|
||||
<a href="/bench">Log</a>
|
||||
<a href="/miners">Miners</a>
|
||||
<a href="/evidence">Evidence</a>
|
||||
<a href="/live">Live</a>
|
||||
<a href="#mine" class="btn primary">Get the miner</a>
|
||||
|
|
@ -485,6 +486,7 @@ footer .wrap{padding-block:48px 32px}
|
|||
<a href="/litepaper#randomx">Igneum vs RandomX</a>
|
||||
<a href="/litepaper#shoulders">Built on the shoulders</a>
|
||||
<a href="/bench">Engineering log</a>
|
||||
<a href="/miners">GPU bench table</a>
|
||||
</nav>
|
||||
<nav class="foot-col" aria-label="Follow">
|
||||
<div class="eyebrow">Follow</div>
|
||||
|
|
|
|||
|
|
@ -145,7 +145,7 @@ body.all .pager{display:none}
|
|||
<svg class="mark" viewBox="0 0 100 100" width="40" height="40" role="img" aria-label="Igneum mark"><polygon points="50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34" fill="var(--ember)"></polygon><polygon points="50,42 59,58 50,82 41,58" fill="var(--ground)"></polygon></svg>
|
||||
<span class="word">IGNEUM</span>
|
||||
</div>
|
||||
<div style="margin-top:10px;display:flex;gap:18px;flex-wrap:wrap"><a href="/" style="font-size:14px">← igneum.network</a><a href="/live" style="font-size:14px">Live devnet →</a></div>
|
||||
<div style="margin-top:10px;display:flex;gap:18px;flex-wrap:wrap"><a href="/" style="font-size:14px">← igneum.network</a><a href="/live" style="font-size:14px">Live devnet →</a><a href="/miners" style="font-size:14px">GPU bench table →</a></div>
|
||||
<div class="eyebrow" style="margin-top:26px">Litepaper · version 0.1</div>
|
||||
<h1>Mined by GPUs.<br>Proven by fire.</h1>
|
||||
<div class="tag">A proof-of-work chain whose miners also prove every block, run Ethereum's apps, and built so no chip can ever take your place.</div>
|
||||
|
|
|
|||
|
|
@ -153,6 +153,7 @@ footer{border-top:1px solid var(--line);padding-block:48px 32px}
|
|||
<a href="/#economics">Economics</a>
|
||||
<a href="/litepaper">Litepaper</a>
|
||||
<a href="/bench">Log</a>
|
||||
<a href="/miners">Miners</a>
|
||||
<a href="/live" class="here" aria-current="page">Live</a>
|
||||
<a href="/#mine" class="btn primary">Get the miner</a>
|
||||
</div>
|
||||
|
|
@ -227,6 +228,7 @@ footer{border-top:1px solid var(--line);padding-block:48px 32px}
|
|||
<a href="/litepaper#limits">What Igneum does not claim</a>
|
||||
<a href="/litepaper#randomx">Igneum vs RandomX</a>
|
||||
<a href="/bench">Engineering log</a>
|
||||
<a href="/miners">GPU bench table</a>
|
||||
</nav>
|
||||
<nav class="foot-col" aria-label="Follow">
|
||||
<div class="eyebrow">Follow</div>
|
||||
|
|
|
|||
71
site/miner-bench.json
Normal file
71
site/miner-bench.json
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
{
|
||||
"_about": "Rows of the public bench table at /miners (site/build.mjs). One row per card, generator version and miner version: the best measured rate. Later jobs append rows. Fields: card, generator (v1 | v2), mh_s (best measured MH/s), mh_per_w (MH per watt, null when the power was not measured), miner (the miner or package version), date (YYYY-MM-DD), source (the docs/bench-log.md heading, or the job id), by ('measured by the team' | 'reported by the fleet'), note (short, optional). No machine names, no addresses, no owner names: the build scrubs the page and fails on any that survive.",
|
||||
"rows": [
|
||||
{
|
||||
"card": "NVIDIA RTX 5090 (32 GB)",
|
||||
"generator": "v1",
|
||||
"mh_s": 228.95,
|
||||
"mh_per_w": null,
|
||||
"miner": "proto-cuda bench (prototype, not mining)",
|
||||
"date": "2026-10-03",
|
||||
"source": "bench log: 3 October 2026, RTX 5090, memory-hard dataset (pack igneum-genesis-mh)",
|
||||
"by": "measured by the team",
|
||||
"note": "genesis program, 104 loads per hash, 1 GiB dataset"
|
||||
},
|
||||
{
|
||||
"card": "NVIDIA RTX 5090 (32 GB)",
|
||||
"generator": "v1",
|
||||
"mh_s": 185.3,
|
||||
"mh_per_w": null,
|
||||
"miner": "proto-cuda bench (prototype, not mining)",
|
||||
"date": "2026-10-03",
|
||||
"source": "bench log: 3 October 2026, RTX 5090 first run, dataset sweep and second program",
|
||||
"by": "measured by the team",
|
||||
"note": "hourly program, 128 loads per hash, 1 GiB dataset"
|
||||
},
|
||||
{
|
||||
"card": "NVIDIA RTX 5090 (32 GB)",
|
||||
"generator": "v2",
|
||||
"mh_s": 124.2,
|
||||
"mh_per_w": null,
|
||||
"miner": "Igneum Miner 0.3.0 package, prebuilt NVRTC worker",
|
||||
"date": "2026-10-04",
|
||||
"source": "bench log: 4 October 2026, the gfx1036 worker fault and what the Mac could and could not reproduce",
|
||||
"by": "measured by the team",
|
||||
"note": "live devnet v4, 128 loads per hash, CPU re-check clean, 0 rejected"
|
||||
},
|
||||
{
|
||||
"card": "Apple M5 Max (40 GPU cores, Metal)",
|
||||
"generator": "v1",
|
||||
"mh_s": 45.2,
|
||||
"mh_per_w": null,
|
||||
"miner": "proto-metal bench (prototype, not mining)",
|
||||
"date": "2026-10-03",
|
||||
"source": "bench log: 3 October 2026, RTX 5090 first run (the Apple row of the same table)",
|
||||
"by": "measured by the team",
|
||||
"note": "genesis program, 1 GiB dataset"
|
||||
},
|
||||
{
|
||||
"card": "Apple M5 Max (40 GPU cores, Metal)",
|
||||
"generator": "v2",
|
||||
"mh_s": 26.7,
|
||||
"mh_per_w": null,
|
||||
"miner": "igneum-miner devnet v4, Metal worker with prepare",
|
||||
"date": "2026-10-04",
|
||||
"source": "bench log: 4 October 2026, first hourly program swap on the live devnet: compile-ahead, no pause, two cards",
|
||||
"by": "measured by the team",
|
||||
"note": "live devnet v4, unbroken through the hour boundary"
|
||||
},
|
||||
{
|
||||
"card": "Apple silicon laptop (model not reported)",
|
||||
"generator": "v2",
|
||||
"mh_s": 24.3,
|
||||
"mh_per_w": null,
|
||||
"miner": "Igneum Miner 0.3.1 (DMG)",
|
||||
"date": "2026-10-04",
|
||||
"source": "bench log: 4 October 2026, first outside machine on the devnet: an Apple silicon laptop through the Igneum Miner app",
|
||||
"by": "reported by the fleet",
|
||||
"note": "21.0 MH/s average over 7 minutes, 24.3 MH/s at the moment of the report, 33 accepted blocks"
|
||||
}
|
||||
]
|
||||
}
|
||||
85
site/miners.html
Normal file
85
site/miners.html
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover">
|
||||
<title>Igneum GPU bench table</title>
|
||||
<meta name="description" content="Measured Igneum hash rates per GPU: card, generator version, best MH/s, MH per watt where measured, miner version, date and the log entry each number came from.">
|
||||
<link rel="canonical" href="https://igneum.network/miners">
|
||||
<meta name="theme-color" content="#0C0C0E">
|
||||
<meta property="og:type" content="website">
|
||||
<meta property="og:site_name" content="Igneum">
|
||||
<meta property="og:title" content="Igneum GPU bench table">
|
||||
<meta property="og:description" content="Measured Igneum hash rates per GPU: card, generator version, best MH/s, MH per watt where measured, miner version, date and the log entry each number came from.">
|
||||
<meta property="og:url" content="https://igneum.network/miners">
|
||||
<meta property="og:image" content="https://igneum.network/og.png?v=2">
|
||||
<meta property="og:image:width" content="1200">
|
||||
<meta property="og:image:height" content="630">
|
||||
<meta property="og:image:alt" content="Igneum. Mined by GPUs. Proven by fire.">
|
||||
<meta name="twitter:card" content="summary_large_image">
|
||||
<meta name="twitter:title" content="Igneum GPU bench table">
|
||||
<meta name="twitter:description" content="Measured Igneum hash rates per GPU: card, generator version, best MH/s, MH per watt where measured, miner version, date and the log entry each number came from.">
|
||||
<meta name="twitter:image" content="https://igneum.network/og.png?v=2">
|
||||
<meta name="twitter:image:alt" content="Igneum. Mined by GPUs. Proven by fire.">
|
||||
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1024 1024'%3E%3Crect width='1024' height='1024' fill='%230C0C0E'/%3E%3Cg transform='translate(166.95 166.95) scale(6.901)'%3E%3Cpolygon points='50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34' fill='%23F2541B'/%3E%3Cpolygon points='50,42 59,58 50,82 41,58' fill='%230C0C0E'/%3E%3C/g%3E%3C/svg%3E" type="image/svg+xml">
|
||||
<link rel="icon" href="/favicon.ico" sizes="48x48">
|
||||
<link rel="icon" href="/favicon-32.png" type="image/png" sizes="32x32">
|
||||
<link rel="icon" href="/icon-192.png" type="image/png" sizes="192x192">
|
||||
<link rel="icon" href="/icon-512.png" type="image/png" sizes="512x512">
|
||||
<link rel="apple-touch-icon" href="/apple-touch-icon.png" sizes="180x180">
|
||||
<link rel="manifest" href="/site.webmanifest">
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Unbounded:wght@500;700;900&family=IBM+Plex+Sans:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500&display=swap">
|
||||
<style>
|
||||
:root{--obsidian:#0C0C0E;--graphite:#16161A;--line:#2A2A30;--ember:#F2541B;--molten:#FFB35C;--bone:#F4F1EC;--ash:#9A9A9E;--ink-2:#C9C7C2;
|
||||
/* one system for every page: container, gutter, section rhythm, card and tile boxes, heading scale (same tokens as the homepage) */
|
||||
--max:1200px;--gutter:clamp(16px,4vw,32px);--sec:clamp(56px,8vw,96px);--head:clamp(40px,6vw,64px);
|
||||
--card-pad:clamp(18px,3vw,28px);--card-r:18px;--tile-pad:18px 20px;--tile-r:14px;--gap:24px;--gap-tile:12px;
|
||||
--fs-h1:clamp(32px,5.5vw,56px);--fs-h2:clamp(28px,4.2vw,44px);--fs-h3:clamp(18px,2vw,22px);--fs-tile:clamp(20px,2.2vw,26px)}
|
||||
*{box-sizing:border-box}body{margin:0;background:var(--obsidian);color:var(--bone);font-family:'IBM Plex Sans',system-ui,sans-serif;font-size:16px;line-height:1.6;padding:0}
|
||||
a{color:var(--ember);text-decoration:none}a:hover{text-decoration:underline}
|
||||
.wrap{max-width:var(--max);margin:0 auto;padding-inline:var(--gutter)}
|
||||
header{display:flex;flex-wrap:wrap;justify-content:space-between;align-items:center;gap:16px;padding-block:18px;border-bottom:1px solid var(--line)}
|
||||
.brand{display:flex;align-items:center;gap:10px;color:var(--bone)}.brand b{font-family:'Unbounded',sans-serif;font-weight:900;letter-spacing:.06em;font-size:18px}
|
||||
header nav{display:flex;flex-wrap:wrap;gap:18px;font-size:14px}header nav a{color:var(--ink-2)}
|
||||
h1{font-family:'Unbounded',sans-serif;font-weight:900;font-size:var(--fs-h1);line-height:1.05;margin:var(--head) 0 12px}
|
||||
.note{color:var(--ash);font-size:15px;max-width:72ch;margin-bottom:32px}
|
||||
.layout{display:grid;grid-template-columns:minmax(0,1fr);gap:32px;padding-bottom:var(--sec)}
|
||||
@media(min-width:960px){.layout{grid-template-columns:240px minmax(0,1fr);gap:56px}}
|
||||
.toc{align-self:start;font-size:14px}@media(min-width:960px){.toc{position:sticky;top:24px}}
|
||||
.toc ol{list-style:none;margin:0;padding:0;display:flex;flex-direction:column;gap:2px}.toc a{display:block;padding:7px 10px;border-radius:8px;color:var(--ink-2)}.toc a:hover{background:var(--graphite);text-decoration:none;color:var(--bone)}
|
||||
article{max-width:78ch;min-width:0}
|
||||
article h1{display:none}
|
||||
article h2{font-family:'Unbounded',sans-serif;font-weight:700;font-size:var(--fs-h3);line-height:1.3;margin:44px 0 12px;padding-top:24px;border-top:1px solid var(--line)}
|
||||
article h2:first-of-type{margin-top:0;padding-top:0;border-top:0}
|
||||
article h3{font-family:'Unbounded',sans-serif;font-weight:500;font-size:16px;margin:26px 0 8px}
|
||||
article h4{font-size:15px;margin:20px 0 6px;color:var(--molten)}
|
||||
article p{margin:0 0 14px}article ul,article ol{margin:0 0 14px;padding-left:22px}article li{margin-bottom:6px}
|
||||
article p,article li,article h2,article h3{overflow-wrap:anywhere}
|
||||
code{font-family:'IBM Plex Mono',monospace;font-size:.92em;background:var(--graphite);padding:1px 5px;border-radius:4px;overflow-wrap:anywhere}
|
||||
pre{margin:0 0 16px;padding:16px 18px;background:var(--graphite);border-radius:var(--card-r);font-family:'IBM Plex Mono',monospace;font-size:13px;line-height:1.55;overflow-x:auto}
|
||||
.tbl{overflow-x:auto;margin:18px 0 24px;border:1px solid var(--line);border-radius:var(--card-r);background:var(--graphite)}
|
||||
table{border-collapse:collapse;width:100%;min-width:560px;font-size:15px}th,td{padding:12px 14px;text-align:left;vertical-align:top;border-bottom:1px solid var(--line)}
|
||||
th{font-family:'IBM Plex Mono',monospace;font-size:12px;letter-spacing:.12em;text-transform:uppercase;color:var(--ash);font-weight:500;background:var(--obsidian)}tr:last-child td{border-bottom:0}
|
||||
footer{border-top:1px solid var(--line);padding-block:32px 48px;font-size:13px;color:var(--ash)}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<header><a class="brand" href="/"><svg viewBox="0 0 100 100" width="26" height="26" aria-hidden="true"><polygon points="50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34" fill="#F2541B"></polygon><polygon points="50,42 59,58 50,82 41,58" fill="#0C0C0E"></polygon></svg><b>IGNEUM</b></a><nav><a href="/">Home</a><a href="/litepaper">Litepaper</a><a href="/live">Live devnet</a><a href="/bench">Engineering log</a><a href="/miners">Miners</a><a href="/evidence">Evidence</a><a href="https://github.com/igneum-network/spec" rel="noopener">GitHub, spec and vectors</a></nav></header>
|
||||
<h1>GPU bench table</h1>
|
||||
<p class="note">Measured hash rates per card on the Igneum lottery hash, with the generator version, the miner version, the date and the log entry behind each number.</p>
|
||||
<div class="layout">
|
||||
<nav class="toc" aria-label="Contents"><ol><li><a href="#table">The table</a></li><li><a href="#how">How a row gets here</a></li></ol></nav>
|
||||
<article><h2 id="table">The table</h2>
|
||||
<p>One row per card, generator version and miner version. The rate is the best one measured. Integrated GPUs are not listed. Prototype rows are bench numbers from before the devnet and say so in the miner column.</p>
|
||||
<div class="tbl"><table><thead><tr><th>Card</th><th>Generator</th><th>Best MH/s</th><th>MH per watt</th><th>Miner</th><th>Date</th><th>Source</th><th>Who measured it</th></tr></thead><tbody><tr><td>Apple M5 Max (40 GPU cores, Metal)</td><td>v1</td><td>45.2</td><td>not measured</td><td>proto-metal bench (prototype, not mining)</td><td>2026-10-03</td><td>bench log: 3 October 2026, RTX 5090 first run (the Apple row of the same table)</td><td>measured by the team. genesis program, 1 GiB dataset</td></tr><tr><td>Apple M5 Max (40 GPU cores, Metal)</td><td>v2</td><td>26.7</td><td>not measured</td><td>igneum-miner devnet v4, Metal worker with prepare</td><td>2026-10-04</td><td>bench log: 4 October 2026, first hourly program swap on the live devnet: compile-ahead, no pause, two cards</td><td>measured by the team. live devnet v4, unbroken through the hour boundary</td></tr><tr><td>Apple silicon laptop (model not reported)</td><td>v2</td><td>24.3</td><td>not measured</td><td>Igneum Miner 0.3.1 (DMG)</td><td>2026-10-04</td><td>bench log: 4 October 2026, first outside machine on the devnet: an Apple silicon laptop through the Igneum Miner app</td><td>reported by the fleet. 21.0 MH/s average over 7 minutes, 24.3 MH/s at the moment of the report, 33 accepted blocks</td></tr><tr><td>NVIDIA RTX 5090 (32 GB)</td><td>v1</td><td>229</td><td>not measured</td><td>proto-cuda bench (prototype, not mining)</td><td>2026-10-03</td><td>bench log: 3 October 2026, RTX 5090, memory-hard dataset (pack igneum-genesis-mh)</td><td>measured by the team. genesis program, 104 loads per hash, 1 GiB dataset</td></tr><tr><td>NVIDIA RTX 5090 (32 GB)</td><td>v1</td><td>185.3</td><td>not measured</td><td>proto-cuda bench (prototype, not mining)</td><td>2026-10-03</td><td>bench log: 3 October 2026, RTX 5090 first run, dataset sweep and second program</td><td>measured by the team. hourly program, 128 loads per hash, 1 GiB dataset</td></tr><tr><td>NVIDIA RTX 5090 (32 GB)</td><td>v2</td><td>124.2</td><td>not measured</td><td>Igneum Miner 0.3.0 package, prebuilt NVRTC worker</td><td>2026-10-04</td><td>bench log: 4 October 2026, the gfx1036 worker fault and what the Apple M5 Max could and could not reproduce</td><td>measured by the team. live devnet v4, 128 loads per hash, CPU re-check clean, 0 rejected</td></tr></tbody></table></div>
|
||||
<h2 id="how">How a row gets here</h2>
|
||||
<p>Every row names the engineering log entry or the job it came from. "Measured by the team" means our own hardware and our own log. "Reported by the fleet" means a machine we do not own, read from the status lines its miner uploads.</p>
|
||||
<p>MH per watt needs the card's power draw during the run. The app reads it on NVIDIA cards through the driver. Rows get the figure when a run records it.</p>
|
||||
<p>There is no other Igneum miner to compare with yet, so this table compares cards, not miners.</p>
|
||||
<p>Rows: 6. Source file: <code>site/miner-bench.json</code> in the repository.</p></article>
|
||||
</div>
|
||||
<footer>© 2026 Igneum. Generated from the repository at build time. Nothing on this page is an offer to sell anything.</footer>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -4,4 +4,5 @@
|
|||
<url><loc>https://igneum.network/litepaper</loc><lastmod>2026-10-03</lastmod><changefreq>monthly</changefreq><priority>0.9</priority></url>
|
||||
<url><loc>https://igneum.network/live</loc><lastmod>2026-10-03</lastmod><changefreq>daily</changefreq><priority>0.7</priority></url>
|
||||
<url><loc>https://igneum.network/bench</loc><lastmod>2026-10-03</lastmod><changefreq>weekly</changefreq><priority>0.6</priority></url>
|
||||
<url><loc>https://igneum.network/miners</loc><lastmod>2026-10-04</lastmod><changefreq>weekly</changefreq><priority>0.6</priority></url>
|
||||
</urlset>
|
||||
|
|
|
|||
217
tools/reliability/app-run.mjs
Executable file
217
tools/reliability/app-run.mjs
Executable file
|
|
@ -0,0 +1,217 @@
|
|||
#!/usr/bin/env node
|
||||
// The app engine's watchdog measured end to end on a Mac: a scratch Igneum Miner engine with its own private node
|
||||
// (ports 29960 and 29961, devnet suffix 9960, no peers, unsynced mining allowed, so the node counts as synced) and
|
||||
// fake-worker.mjs standing in for the Metal worker, driven through the control file and with signals.
|
||||
//
|
||||
// node tools/reliability/app-run.mjs --app <igneum-app> --miner <igneum-miner> [--node <igneumd>]
|
||||
//
|
||||
// Steps, in one engine run (each states what must happen and what must not):
|
||||
// own-restart the worker reports jobs done in 0.3 ms: the miner's guard restarts the worker; the app shows the
|
||||
// fault and does NOT restart the miner (same pid, no app restart)
|
||||
// zero-once the worker completes jobs with 0 hashes: hash rate 0 for 60 s while synced, the app restarts the
|
||||
// miner once; the worker is healthy again: mining resumes
|
||||
// zero-faulted the same again inside five minutes: the card is marked faulted with the reason, its miner is not
|
||||
// restarted, the node keeps running; "resume" clears it
|
||||
// no-status the miner process is stopped with SIGSTOP: no status line for 90 s, the app restarts it
|
||||
// node-silent the node is stopped with SIGSTOP: no reading for 120 s, the app restarts the node in-process and
|
||||
// the miner comes back once it is synced
|
||||
// Never touches the live devnet. The app's own log is in the scratch directory.
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdirSync, rmSync, writeFileSync, existsSync, symlinkSync, readFileSync, appendFileSync, chmodSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const args = process.argv.slice(2);
|
||||
const opt = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
|
||||
const APP = opt('--app');
|
||||
const MINER = opt('--miner');
|
||||
const NODE = opt('--node', join(here, '../../vendor/igneum-node/target-integration/release/igneumd'));
|
||||
const ONLY = opt('--only', '').split(',').filter(Boolean);
|
||||
const SCRATCH = process.env.SCRATCH || `/tmp/igneum-reliability-app-${process.pid}`;
|
||||
const RPC = 29960, P2P = 29961, SUFFIX = 9960;
|
||||
for (const [k, v] of Object.entries({ APP, MINER, NODE })) if (!v || !existsSync(v)) { console.error(`missing ${k} (${v})`); process.exit(2); }
|
||||
|
||||
const t0 = Date.now();
|
||||
const since = () => ((Date.now() - t0) / 1000).toFixed(1);
|
||||
const log = (...a) => console.log(new Date().toISOString().slice(11, 23), `t=${since()}s`, ...a);
|
||||
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
const s = (ms) => (ms / 1000).toFixed(1) + ' s';
|
||||
|
||||
rmSync(SCRATCH, { recursive: true, force: true });
|
||||
const bin = join(SCRATCH, 'stage', 'bin'); mkdirSync(bin, { recursive: true });
|
||||
symlinkSync(NODE, join(bin, 'igneumd'));
|
||||
symlinkSync(MINER, join(bin, 'igneum-miner'));
|
||||
chmodSync(join(here, 'fake-worker.mjs'), 0o755);
|
||||
symlinkSync(join(here, 'fake-worker.mjs'), join(bin, 'igneum-bench'));
|
||||
const data = join(SCRATCH, 'data'); mkdirSync(join(data, 'app'), { recursive: true });
|
||||
const logs = join(SCRATCH, 'logs'); mkdirSync(logs, { recursive: true });
|
||||
const CTL = join(SCRATCH, 'ctl');
|
||||
const setMode = (m) => { writeFileSync(CTL, m + '\n'); log(`fake worker mode -> ${m}`); };
|
||||
setMode('ok');
|
||||
writeFileSync(join(data, 'app', 'settings.json'), JSON.stringify({
|
||||
setup_done: true, address: '0x4242424242424242424242424242424242424242', address_source: 'pasted', key_saved: true, identities: 1,
|
||||
cards: { 'apple::Fake GPU': { enabled: true, identities: 1, power_pct: 0 } }, vote: false, paused: false, accepted_total: 0,
|
||||
auto_update: false, remote_jobs: false, prove: false,
|
||||
}, null, 2));
|
||||
|
||||
const env = {
|
||||
...process.env, IGNEUM_APP_DATA: data, IGNEUM_APP_LOGS: logs, IGNEUM_APP_BIN: bin, IGNEUM_APP_RPC_PORT: String(RPC), IGNEUM_APP_P2P_PORT: String(P2P),
|
||||
IGNEUM_APP_PEERS: '', IGNEUM_APP_UNSYNCED: '1', IGNEUM_APP_DEVNET_SUFFIX: String(SUFFIX), IGNEUM_APP_STATUS_SECS: '10', FAKE_WORKER_CTL: CTL,
|
||||
};
|
||||
const app = spawn(APP, ['--no-open'], { stdio: ['pipe', 'pipe', 'pipe'], env });
|
||||
const appOut = join(SCRATCH, 'app.out');
|
||||
app.stdout.on('data', d => appendFileSync(appOut, d)); app.stderr.on('data', d => appendFileSync(appOut, d));
|
||||
let appExit = null;
|
||||
app.on('exit', c => { appExit = c; log(`app exited ${c}`); });
|
||||
process.on('exit', () => { try { app.kill('SIGKILL'); } catch {} });
|
||||
process.on('SIGINT', () => process.exit(130));
|
||||
log(`app pid ${app.pid}, scratch ${SCRATCH}`);
|
||||
|
||||
let url = null;
|
||||
for (let i = 0; i < 100 && !url; i++) { try { url = readFileSync(join(data, 'app', 'app.url'), 'utf8').trim(); } catch { await sleep(200); } }
|
||||
if (!url) { log('no app.url'); process.exit(1); }
|
||||
async function state() {
|
||||
try { const r = await fetch(url + 'api/state'); return await r.json(); } catch { return null; }
|
||||
}
|
||||
const card = (st) => (st && st.mining && st.mining.cards && st.mining.cards[0]) || {};
|
||||
const events = []; // (t, card state, message, hash, pid, restarts, faults, node state) transitions, for the record
|
||||
let last = '';
|
||||
async function poll() {
|
||||
const st = await state();
|
||||
if (!st) return null;
|
||||
const c = card(st);
|
||||
const key = [c.state, c.message, c.pid, c.restarts, c.faults, st.node.state, c.hash_now > 0 ? '>0' : '0'].join('|');
|
||||
if (key !== last) { last = key; events.push({ t: Date.now(), key }); log(`card ${c.state} pid ${c.pid} restarts ${c.restarts} faults ${c.faults} hash ${Number(c.hash_now).toFixed(1)} node ${st.node.state} ${c.message ? '"' + c.message + '"' : ''}`); }
|
||||
return st;
|
||||
}
|
||||
async function until(pred, timeoutMs, what) {
|
||||
const end = Date.now() + timeoutMs;
|
||||
while (Date.now() < end) { const st = await poll(); if (st && pred(st, card(st))) return st; if (appExit !== null) return null; await sleep(1000); }
|
||||
log(`timeout waiting for: ${what}`);
|
||||
return null;
|
||||
}
|
||||
const mining = (st, c) => c.state === 'mining' && c.hash_now > 0 && st.node.synced;
|
||||
const results = [];
|
||||
function verdict(name, checks, metrics) {
|
||||
const pass = checks.every(c => c.ok);
|
||||
results.push({ name, pass, checks, metrics });
|
||||
log(`${pass ? 'PASS' : 'FAIL'} ${name}`);
|
||||
for (const c of checks) log(` ${c.ok ? 'ok ' : 'FAIL'} ${c.what}`);
|
||||
for (const [k, v] of Object.entries(metrics)) log(` ${k}: ${v}`);
|
||||
}
|
||||
async function steady(ms) { const end = Date.now() + ms; while (Date.now() < end) { await poll(); await sleep(1000); } }
|
||||
|
||||
const S = {};
|
||||
S['own-restart'] = async () => {
|
||||
const st0 = await until(mining, 180000, 'first mining');
|
||||
const pid0 = card(st0).pid, r0 = card(st0).restarts;
|
||||
const tInject = Date.now();
|
||||
setMode('fast');
|
||||
const faulted = await until((st, c) => c.faults >= 1, 90000, 'the worker fault to reach the card');
|
||||
const tFault = Date.now();
|
||||
setMode('ok');
|
||||
const back = await until((st, c) => mining(st, c) && c.faults >= 1, 90000, 'mining again after the worker restart');
|
||||
const tBack = Date.now();
|
||||
await steady(20000);
|
||||
const st1 = await poll();
|
||||
verdict('own-restart', [
|
||||
{ ok: !!st0, what: 'the card mined with a rate above 0 on the fake worker' },
|
||||
{ ok: !!faulted && /worker fault/.test(card(faulted).message || ''), what: `the card showed the WORKER FAULT reason (${JSON.stringify(card(faulted || {}).message)})` },
|
||||
{ ok: !!back, what: 'mining resumed after the miner restarted its own worker' },
|
||||
{ ok: st1 && card(st1).pid === pid0 && card(st1).restarts === r0, what: `the app did not restart the miner (pid ${pid0} -> ${card(st1 || {}).pid}, app restarts ${r0} -> ${card(st1 || {}).restarts})` },
|
||||
], { inject_to_fault_on_card: s(tFault - tInject), fault_to_mining_again: s(tBack - tFault) });
|
||||
};
|
||||
S['zero-once'] = async () => {
|
||||
const st0 = await until(mining, 60000, 'mining');
|
||||
const pid0 = card(st0).pid, r0 = card(st0).restarts;
|
||||
const tInject = Date.now();
|
||||
setMode('zero');
|
||||
const rs = await until((st, c) => c.restarts > r0 || /watchdog/.test(c.message || ''), 150000, 'the watchdog restart');
|
||||
const tRestart = Date.now();
|
||||
setMode('ok');
|
||||
const back = await until((st, c) => mining(st, c) && c.pid !== pid0, 120000, 'mining on the restarted miner');
|
||||
const tBack = Date.now();
|
||||
await steady(15000);
|
||||
const st1 = await poll();
|
||||
verdict('zero-once', [
|
||||
{ ok: !!rs && /hash rate 0/.test(card(rs).message || ''), what: `the watchdog restarted the miner for a zero rate (${JSON.stringify(card(rs || {}).message)})` },
|
||||
{ ok: rs && tRestart - tInject >= 55000 && tRestart - tInject <= 100000, what: `between 55 and 100 s after the rate went to 0 (${s(tRestart - tInject)})` },
|
||||
{ ok: !!back, what: 'mining resumed on a new miner process' },
|
||||
{ ok: st1 && card(st1).restarts === r0 + 1, what: `exactly one app restart (${r0} -> ${card(st1 || {}).restarts})` },
|
||||
], { zero_to_restart: s(tRestart - tInject), restart_to_mining: s(tBack - tRestart), zero_to_mining: s(tBack - tInject) });
|
||||
};
|
||||
S['zero-faulted'] = async () => {
|
||||
const st0 = await until(mining, 60000, 'mining');
|
||||
const r0 = card(st0).restarts;
|
||||
const tInject = Date.now();
|
||||
setMode('zero');
|
||||
const f = await until((st, c) => c.state === 'faulted', 150000, 'the card to be marked faulted');
|
||||
const tFault = Date.now();
|
||||
await steady(45000);
|
||||
const st1 = await poll();
|
||||
setMode('ok');
|
||||
app.stdin.write('resume\n');
|
||||
const back = await until(mining, 90000, 'mining after resume');
|
||||
verdict('zero-faulted', [
|
||||
{ ok: !!f && /restarted once already/.test(card(f).message || ''), what: `the card was marked faulted with the reason (${JSON.stringify(card(f || {}).message)})` },
|
||||
{ ok: st1 && card(st1).state === 'faulted' && card(st1).pid === 0 && card(st1).restarts === r0, what: `45 s later: still faulted, no miner process, no further restart (restarts ${r0} -> ${card(st1 || {}).restarts})` },
|
||||
{ ok: st1 && st1.node.synced && appExit === null, what: 'the node kept running and the app stayed up' },
|
||||
{ ok: !!back, what: 'resume cleared the fault and mining resumed' },
|
||||
], { zero_to_faulted: s(tFault - tInject) });
|
||||
};
|
||||
S['no-status'] = async () => {
|
||||
const st0 = await until(mining, 60000, 'mining');
|
||||
const pid0 = card(st0).pid, r0 = card(st0).restarts;
|
||||
const tInject = Date.now();
|
||||
process.kill(pid0, 'SIGSTOP');
|
||||
log(`SIGSTOP miner pid ${pid0}`);
|
||||
const rs = await until((st, c) => c.restarts > r0 || /no status line/.test(c.message || ''), 150000, 'the watchdog restart');
|
||||
const tRestart = Date.now();
|
||||
const back = await until((st, c) => mining(st, c) && c.pid !== pid0 && c.pid > 0, 120000, 'mining on the restarted miner');
|
||||
const tBack = Date.now();
|
||||
let gone = false; try { process.kill(pid0, 0); } catch { gone = true; }
|
||||
if (!gone) { try { process.kill(pid0, 'SIGKILL'); } catch {} }
|
||||
verdict('no-status', [
|
||||
{ ok: !!rs && /no status line/.test(card(rs).message || ''), what: `the watchdog restarted the miner for missing status lines (${JSON.stringify(card(rs || {}).message)})` },
|
||||
{ ok: rs && tRestart - tInject >= 85000 && tRestart - tInject <= 130000, what: `between 85 and 130 s after the miner went quiet (${s(tRestart - tInject)})` },
|
||||
{ ok: !!back, what: 'mining resumed on a new miner process' },
|
||||
{ ok: gone, what: 'the stopped miner process was killed' },
|
||||
], { quiet_to_restart: s(tRestart - tInject), restart_to_mining: s(tBack - tRestart) });
|
||||
};
|
||||
S['node-silent'] = async () => {
|
||||
const st0 = await until(mining, 60000, 'mining');
|
||||
const npid = st0.node.pid, starts0 = st0.node.starts;
|
||||
const tInject = Date.now();
|
||||
process.kill(npid, 'SIGSTOP');
|
||||
log(`SIGSTOP node pid ${npid}`);
|
||||
const rs = await until((st) => st.node.state === 'restarting' || st.node.starts > starts0, 200000, 'the node restart');
|
||||
const tRestart = Date.now();
|
||||
const synced = await until((st) => st.node.starts > starts0 && st.node.synced, 120000, 'the restarted node to sync');
|
||||
const tSynced = Date.now();
|
||||
const back = await until(mining, 120000, 'mining again');
|
||||
const tBack = Date.now();
|
||||
let gone = false; try { process.kill(npid, 0); } catch { gone = true; }
|
||||
if (!gone) { try { process.kill(npid, 'SIGKILL'); } catch {} }
|
||||
verdict('node-silent', [
|
||||
{ ok: !!rs, what: 'the app restarted the node' },
|
||||
{ ok: rs && tRestart - tInject >= 115000 && tRestart - tInject <= 170000, what: `between 115 and 170 s after the node went quiet (${s(tRestart - tInject)})` },
|
||||
{ ok: !!synced, what: `the new node synced (starts ${starts0} -> ${synced ? synced.node.starts : '?'})` },
|
||||
{ ok: !!back, what: 'mining resumed' },
|
||||
{ ok: gone, what: 'the stopped node process was killed' },
|
||||
], { quiet_to_restart: s(tRestart - tInject), restart_to_synced: s(tSynced - tRestart), quiet_to_mining: s(tBack - tInject) });
|
||||
};
|
||||
|
||||
const names = ONLY.length ? ONLY : Object.keys(S);
|
||||
for (const n of names) {
|
||||
if (!S[n]) { log(`unknown step ${n}`); continue; }
|
||||
log(`=== ${n}`);
|
||||
try { await S[n](); } catch (e) { log(`step ${n} threw: ${e.stack || e}`); results.push({ name: n, pass: false, checks: [{ ok: false, what: String(e) }], metrics: {} }); }
|
||||
}
|
||||
app.stdin.write('quit\n');
|
||||
await sleep(8000);
|
||||
const report = { date: new Date().toISOString(), app: APP, miner: MINER, node: NODE, scratch: SCRATCH, results, transitions: events.map(e => ({ t: new Date(e.t).toISOString(), key: e.key })) };
|
||||
writeFileSync(join(SCRATCH, 'report.json'), JSON.stringify(report, null, 2));
|
||||
console.log(JSON.stringify({ ...report, transitions: undefined }, null, 2));
|
||||
process.exit(results.every(r => r.pass) ? 0 : 1);
|
||||
74
tools/reliability/fake-worker.mjs
Executable file
74
tools/reliability/fake-worker.mjs
Executable file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env node
|
||||
// A GPU worker stand-in that speaks the miner's serve protocol (ready / job / done / found / prepare / prepared /
|
||||
// quit) and misbehaves on command, so the miner's fault guards and the app's watchdog can be measured on a Mac with
|
||||
// no faulty GPU at hand. It never hashes: every `done` is invented, every `found` is wrong on purpose.
|
||||
//
|
||||
// FAKE_WORKER_CTL=<file> the file's first word is the mode, re-read before every job:
|
||||
// ok every job: `done` after 100 ms with the job's nonce count as hashes (about 167 MH/s at 2^24 nonces)
|
||||
// slow `done` after 5,000 ms (a self-heal build inside a job, a game on the GPU)
|
||||
// fast `done` after 0.3 ms with the full count: a runtime answering success while running nothing
|
||||
// (the 4 October 2026 gfx1036 fault)
|
||||
// zero `done` after 100 ms with 0 hashes: the worker completes jobs and produces nothing
|
||||
// silent never answers a job (the queue fills and nothing comes back)
|
||||
// badfound a `found` with a wrong hash before every `done`: a worker on a wrong program
|
||||
// preparefail every `prepare` is answered `prepare-failed`; jobs run as in ok
|
||||
// exit the process exits with code 7 at the next job
|
||||
// The mode is read from argv `--mode <m>` when no control file is set. `--serve` is accepted and ignored.
|
||||
// Lines on stderr are prefixed `fake-worker:` and never match a miner pattern.
|
||||
|
||||
import { readFileSync, existsSync, writeSync } from 'node:fs';
|
||||
import { createInterface } from 'node:readline';
|
||||
|
||||
const ctl = process.env.FAKE_WORKER_CTL;
|
||||
const argMode = (() => { const i = process.argv.indexOf('--mode'); return i >= 0 ? process.argv[i + 1] : 'ok'; })();
|
||||
function mode() {
|
||||
if (ctl && existsSync(ctl)) {
|
||||
const m = readFileSync(ctl, 'utf8').trim().split(/\s+/)[0];
|
||||
if (m) return m;
|
||||
}
|
||||
return argMode;
|
||||
}
|
||||
// synchronous writes: a piped stdout is asynchronous in Node and process.exit would drop pending lines
|
||||
const out = (s) => { writeSync(1, s + '\n'); };
|
||||
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
|
||||
out('ready fake Fake_GPU dataset-log2 28 batch 16777216 prepare 1');
|
||||
|
||||
const queue = [];
|
||||
let running = false;
|
||||
async function pump() {
|
||||
if (running) return;
|
||||
running = true;
|
||||
while (queue.length) {
|
||||
const f = queue.shift();
|
||||
const [id, , , nonceStart, nonces] = f;
|
||||
const m = mode();
|
||||
if (m === 'exit') process.exit(7);
|
||||
if (m === 'silent') { await sleep(1_000_000); continue; }
|
||||
const t0 = performance.now();
|
||||
let hashes = Number(nonces);
|
||||
let ms = 100;
|
||||
if (m === 'slow') ms = 5000;
|
||||
else if (m === 'fast') ms = 0.3;
|
||||
else if (m === 'zero') hashes = 0;
|
||||
await sleep(ms);
|
||||
if (m === 'badfound') out(`found ${id} ${BigInt(nonceStart) + 17n} 0000000000000001`);
|
||||
out(`done ${id} ${hashes} ${(performance.now() - t0).toFixed(2)}`);
|
||||
}
|
||||
running = false;
|
||||
}
|
||||
|
||||
const rl = createInterface({ input: process.stdin });
|
||||
rl.on('line', (line) => {
|
||||
const f = line.trim().split(/\s+/);
|
||||
switch (f[0]) {
|
||||
case 'job': queue.push(f.slice(1)); pump(); break;
|
||||
case 'prepare':
|
||||
if (mode() === 'preparefail') out(`prepare-failed ${f[1]} ${f[2]} fake worker refuses`);
|
||||
else setTimeout(() => out(`prepared ${f[1]} ${f[2]} 50 fake resident 2 programs 2 datasets`), 50);
|
||||
break;
|
||||
case 'quit': process.exit(0);
|
||||
default: break;
|
||||
}
|
||||
});
|
||||
rl.on('close', () => process.exit(0));
|
||||
297
tools/reliability/run.mjs
Executable file
297
tools/reliability/run.mjs
Executable file
|
|
@ -0,0 +1,297 @@
|
|||
#!/usr/bin/env node
|
||||
// Miner fault guards measured on a private test network (ports 29950 to 29953, devnet suffix 9950, data under the
|
||||
// scratch directory): one igneumd, one CPU block producer when a scenario needs epochs, and the miner under test
|
||||
// driving fake-worker.mjs, which misbehaves on command through a control file. Never touches the live devnet.
|
||||
//
|
||||
// node tools/reliability/run.mjs --miner <igneum-miner> [--node <igneumd>] [--only slow-first,fake-fast,...]
|
||||
//
|
||||
// Scenarios (each a fresh miner process; the fake worker's mode is switched by rewriting the control file):
|
||||
// slow-first the first 25 s of jobs take 5 s each, then the true rate (100x): the old guard tripped on every
|
||||
// interval from the second on (M26); expected now: no fault, STATUS every 10 s throughout
|
||||
// fake-fast 3 healthy intervals, then jobs "done" in 0.3 ms: one fault, a STATUS line with faults=1,
|
||||
// the worker back after the 2 s delay; the mode returns to ok at the fault
|
||||
// fake-fast-stays the same, but the fault persists: three trips at growing delay, then exit 43
|
||||
// badfound wrong hashes on every found: three consecutive CPU re-check mismatches stop the worker (X21)
|
||||
// silent the worker stops answering: STATUS lines keep coming, the stall guard fires at 60 s
|
||||
// prepare-flap with epochs every 60 DAA (the block producer), the worker refuses every prepare: at most two
|
||||
// sends per pair per epoch, 30 s apart (M27)
|
||||
//
|
||||
// A watcher is trusted only once it fires on a known-good and a known-bad case: every scenario states what must
|
||||
// appear AND what must not, and the run fails if the fault detector saw nothing in the scenarios that fault.
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdirSync, rmSync, writeFileSync, existsSync, appendFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const args = process.argv.slice(2);
|
||||
const opt = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
|
||||
const MINER = opt('--miner');
|
||||
const NODE = opt('--node', join(here, '../../vendor/igneum-node/target-integration/release/igneumd'));
|
||||
const ONLY = opt('--only', '').split(',').filter(Boolean);
|
||||
const SCRATCH = process.env.SCRATCH || `/tmp/igneum-reliability-${process.pid}`;
|
||||
const BASE = 29950, SUFFIX = 9950;
|
||||
const FAKE = join(here, 'fake-worker.mjs');
|
||||
if (!MINER || !existsSync(MINER)) { console.error('need --miner <igneum-miner with the guards>'); process.exit(2); }
|
||||
if (!existsSync(NODE)) { console.error(`missing node ${NODE}`); process.exit(2); }
|
||||
|
||||
const t0 = Date.now();
|
||||
const since = () => ((Date.now() - t0) / 1000).toFixed(1);
|
||||
const log = (...a) => console.log(new Date().toISOString().slice(11, 23), `t=${since()}s`, ...a);
|
||||
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
const started = [];
|
||||
process.on('exit', () => { for (const p of started) { try { p.kill('SIGKILL'); } catch {} } });
|
||||
process.on('SIGINT', () => process.exit(130));
|
||||
|
||||
rmSync(SCRATCH, { recursive: true, force: true }); mkdirSync(SCRATCH, { recursive: true });
|
||||
const CTL = join(SCRATCH, 'ctl');
|
||||
const setMode = (m) => { writeFileSync(CTL, m + '\n'); log(`fake worker mode -> ${m}`); };
|
||||
|
||||
function startNode(env = {}) {
|
||||
const dir = join(SCRATCH, 'node'); mkdirSync(dir, { recursive: true });
|
||||
const a = ['--devnet', `--devnet-suffix=${SUFFIX}`, '--nodnsseed', '--disable-upnp', '--nologfiles', '--enable-unsynced-mining', '--outpeers=0',
|
||||
`--appdir=${dir}`, `--rpclisten=127.0.0.1:${BASE}`, `--rpclisten-json=127.0.0.1:${BASE + 2}`, `--evm-rpclisten=127.0.0.1:${BASE + 3}`,
|
||||
`--listen=127.0.0.1:${BASE + 1}`, '--loglevel=warn', '--yes'];
|
||||
const p = spawn(NODE, a, { stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, ...env } });
|
||||
started.push(p);
|
||||
const f = join(SCRATCH, 'node.log');
|
||||
p.stdout.on('data', d => appendFileSync(f, d)); p.stderr.on('data', d => appendFileSync(f, d));
|
||||
log(`node pid ${p.pid} grpc ${BASE}`);
|
||||
return p;
|
||||
}
|
||||
|
||||
/// The miner under test: lines are kept with the receipt time (the miner's own unix stamp when the line carries one).
|
||||
function startMiner(secs, extra = []) {
|
||||
const a = ['mine', `grpc://127.0.0.1:${BASE}`, '1', String(secs), 'wt', '--worker', FAKE, '--status-secs', '10', '--no-vote', '--payout-label', 'wt', ...extra];
|
||||
const p = spawn(MINER, a, { stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, FAKE_WORKER_CTL: CTL } });
|
||||
started.push(p);
|
||||
const m = { proc: p, lines: [], exit: null };
|
||||
const f = join(SCRATCH, `miner-${Date.now()}.log`);
|
||||
const take = (err) => (chunk) => {
|
||||
for (const text of chunk.toString().split('\n')) {
|
||||
if (!text) continue;
|
||||
const stamp = /^(\d{10})\.(\d{3}) /.exec(text);
|
||||
const t = stamp ? Number(stamp[1]) * 1000 + Number(stamp[2]) : Date.now();
|
||||
m.lines.push({ t, err, text });
|
||||
appendFileSync(f, (err ? '! ' : '') + text + '\n');
|
||||
}
|
||||
};
|
||||
p.stdout.on('data', take(false)); p.stderr.on('data', take(true));
|
||||
p.on('exit', (code) => { m.exit = code; log(`miner exited ${code}`); });
|
||||
log(`miner pid ${p.pid} log ${f}`);
|
||||
return m;
|
||||
}
|
||||
const find = (m, re, after = 0) => m.lines.filter(l => l.t >= after && re.test(l.text));
|
||||
async function waitFor(m, re, timeoutMs, after = 0) {
|
||||
const end = Date.now() + timeoutMs;
|
||||
while (Date.now() < end) { const h = find(m, re, after); if (h.length) return h[0]; if (m.exit !== null) return null; await sleep(200); }
|
||||
return null;
|
||||
}
|
||||
const stop = (m) => { try { m.proc.kill('SIGTERM'); } catch {} };
|
||||
const s = (ms) => (ms / 1000).toFixed(1) + ' s';
|
||||
const results = [];
|
||||
function verdict(name, checks, metrics) {
|
||||
const pass = checks.every(c => c.ok);
|
||||
results.push({ name, pass, checks, metrics });
|
||||
log(`${pass ? 'PASS' : 'FAIL'} ${name}`);
|
||||
for (const c of checks) log(` ${c.ok ? 'ok ' : 'FAIL'} ${c.what}`);
|
||||
for (const [k, v] of Object.entries(metrics)) log(` ${k}: ${v}`);
|
||||
}
|
||||
|
||||
const S = {};
|
||||
|
||||
S['slow-first'] = async () => {
|
||||
setMode('slow');
|
||||
const m = startMiner(120);
|
||||
const ready = await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(25000);
|
||||
setMode('ok');
|
||||
await sleep(60000);
|
||||
stop(m);
|
||||
const faults = find(m, /WORKER FAULT/);
|
||||
const status = find(m, / STATUS '/);
|
||||
const gaps = status.slice(1).map((l, i) => l.t - status[i].t);
|
||||
verdict('slow-first', [
|
||||
{ ok: !!ready, what: 'worker reported ready' },
|
||||
{ ok: faults.length === 0, what: `no WORKER FAULT after a slow first interval (saw ${faults.length})` },
|
||||
{ ok: status.length >= 7, what: `STATUS lines kept coming (${status.length} in 85 s)` },
|
||||
{ ok: gaps.every(g => g < 25000), what: `no STATUS gap over 25 s (max ${s(Math.max(0, ...gaps))})` },
|
||||
], { status_lines: status.length, faults: faults.length });
|
||||
};
|
||||
|
||||
S['fake-fast'] = async () => {
|
||||
setMode('ok');
|
||||
const m = startMiner(180);
|
||||
await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(36000);
|
||||
const tInject = Date.now();
|
||||
setMode('fast');
|
||||
const fault = await waitFor(m, /WORKER FAULT/, 60000, tInject);
|
||||
if (fault) setMode('ok');
|
||||
const killed = await waitFor(m, /worker killed by a guard/, 20000, tInject);
|
||||
const restarted = await waitFor(m, /worker restarted/, 30000, tInject);
|
||||
const ready = await waitFor(m, /worker: ready/, 30000, restarted ? restarted.t : tInject);
|
||||
// the first STATUS after the restart with a rate above 0
|
||||
let recovered = null;
|
||||
const end = Date.now() + 40000;
|
||||
while (!recovered && Date.now() < end) {
|
||||
recovered = find(m, / STATUS '.*now=(?!0\.00)/, ready ? ready.t : tInject)[0] || null;
|
||||
if (!recovered) await sleep(300);
|
||||
}
|
||||
await sleep(15000);
|
||||
stop(m);
|
||||
const statusAfterFault = fault ? find(m, / STATUS '.*faults=[1-9]/, fault.t) : [];
|
||||
const faultsTotal = find(m, /WORKER FAULT/).filter(l => !l.err).length;
|
||||
const restarts = find(m, /worker restarted/).length;
|
||||
verdict('fake-fast', [
|
||||
{ ok: !!fault, what: 'the guard fired on jobs done in 0.3 ms' },
|
||||
{ ok: statusAfterFault.length > 0, what: 'a STATUS line with faults=1 was printed after the trip (the old guard printed none)' },
|
||||
{ ok: !!killed && !!restarted, what: 'the worker was killed by the guard and restarted' },
|
||||
{ ok: !!recovered, what: 'a STATUS line with a rate above 0 followed the restart' },
|
||||
{ ok: faultsTotal === 1 && restarts === 1, what: `exactly one fault and one restart (faults ${faultsTotal}, restarts ${restarts})` },
|
||||
], {
|
||||
inject_to_fault: fault ? s(fault.t - tInject) : 'n/a',
|
||||
fault_to_restarted: fault && restarted ? s(restarted.t - fault.t) : 'n/a',
|
||||
fault_to_ready: fault && ready ? s(ready.t - fault.t) : 'n/a',
|
||||
fault_to_first_healthy_status: fault && recovered ? s(recovered.t - fault.t) : 'n/a',
|
||||
inject_to_first_healthy_status: recovered ? s(recovered.t - tInject) : 'n/a',
|
||||
});
|
||||
};
|
||||
|
||||
S['fake-fast-stays'] = async () => {
|
||||
setMode('ok');
|
||||
const m = startMiner(600);
|
||||
await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(36000);
|
||||
const tInject = Date.now();
|
||||
setMode('fast');
|
||||
const end = Date.now() + 300000;
|
||||
while (m.exit === null && Date.now() < end) await sleep(500);
|
||||
const faults = find(m, /WORKER FAULT/).filter(l => !l.err);
|
||||
const delays = find(m, /restarting it in (\d+) s/).map(l => Number(/restarting it in (\d+) s/.exec(l.text)[1]));
|
||||
const gaveUp = find(m, /exiting with code 43/);
|
||||
verdict('fake-fast-stays', [
|
||||
{ ok: faults.length >= 3, what: `three or more faults (${faults.length})` },
|
||||
{ ok: delays.length >= 2 && delays[0] === 2 && delays[1] === 4, what: `restart delays grow 2, 4 s (saw ${delays.join(', ')})` },
|
||||
{ ok: m.exit === 43 && gaveUp.length === 1, what: `the miner gave up with exit 43 (exit ${m.exit})` },
|
||||
], {
|
||||
inject_to_exit: s((m.lines.at(-1)?.t || Date.now()) - tInject),
|
||||
fault_times: faults.map(l => s(l.t - tInject)).join(', '),
|
||||
});
|
||||
};
|
||||
|
||||
S['badfound'] = async () => {
|
||||
setMode('ok');
|
||||
const m = startMiner(180);
|
||||
await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(15000);
|
||||
const tInject = Date.now();
|
||||
setMode('badfound');
|
||||
const mism = await waitFor(m, /WORKER MISMATCH/, 30000, tInject);
|
||||
const fault = await waitFor(m, /WORKER FAULT cpu re-check: 3 consecutive mismatches/, 60000, tInject);
|
||||
if (fault) setMode('ok');
|
||||
const restarted = await waitFor(m, /worker restarted/, 30000, tInject);
|
||||
const ready = await waitFor(m, /worker: ready/, 30000, restarted ? restarted.t : tInject);
|
||||
let recovered = null;
|
||||
const end = Date.now() + 40000;
|
||||
while (!recovered && Date.now() < end) {
|
||||
recovered = find(m, / STATUS '.*now=(?!0\.00)/, ready ? ready.t : tInject)[0] || null;
|
||||
if (!recovered) await sleep(300);
|
||||
}
|
||||
await sleep(12000);
|
||||
stop(m);
|
||||
const statusMism = find(m, / STATUS '.*mismatched=[1-9]/);
|
||||
const mismCount = find(m, /WORKER MISMATCH/).length;
|
||||
verdict('badfound', [
|
||||
{ ok: !!mism, what: 'the CPU re-check rejected the wrong hash' },
|
||||
{ ok: !!fault, what: 'three consecutive mismatches stopped the worker (X21)' },
|
||||
{ ok: mismCount <= 4, what: `no more than 4 wrong shares were accepted before the stop (${mismCount})` },
|
||||
{ ok: statusMism.length > 0, what: 'STATUS shows mismatched= above 0 for the app to read' },
|
||||
{ ok: !!restarted && !!recovered, what: 'the worker restarted and a rate above 0 followed' },
|
||||
], {
|
||||
inject_to_first_mismatch: mism ? s(mism.t - tInject) : 'n/a',
|
||||
inject_to_fault: fault ? s(fault.t - tInject) : 'n/a',
|
||||
fault_to_first_healthy_status: fault && recovered ? s(recovered.t - fault.t) : 'n/a',
|
||||
});
|
||||
};
|
||||
|
||||
S['silent'] = async () => {
|
||||
setMode('ok');
|
||||
const m = startMiner(240);
|
||||
await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(15000);
|
||||
const tInject = Date.now();
|
||||
setMode('silent');
|
||||
const fault = await waitFor(m, /WORKER FAULT no job completed/, 120000, tInject);
|
||||
if (fault) setMode('ok');
|
||||
const restarted = await waitFor(m, /worker restarted/, 30000, tInject);
|
||||
const ready = await waitFor(m, /worker: ready/, 30000, restarted ? restarted.t : tInject);
|
||||
let recovered = null;
|
||||
const end = Date.now() + 40000;
|
||||
while (!recovered && Date.now() < end) {
|
||||
recovered = find(m, / STATUS '.*now=(?!0\.00)/, ready ? ready.t : tInject)[0] || null;
|
||||
if (!recovered) await sleep(300);
|
||||
}
|
||||
await sleep(12000);
|
||||
stop(m);
|
||||
const statusDuringSilence = fault ? find(m, / STATUS '/, tInject).filter(l => l.t < fault.t) : [];
|
||||
verdict('silent', [
|
||||
{ ok: !!fault, what: 'the stall guard fired on a worker that stopped answering' },
|
||||
{ ok: fault && fault.t - tInject >= 55000 && fault.t - tInject <= 90000, what: `the stall guard fired between 55 and 90 s after the silence began (${fault ? s(fault.t - tInject) : 'n/a'})` },
|
||||
{ ok: statusDuringSilence.length >= 4, what: `STATUS lines kept printing while the worker was silent (${statusDuringSilence.length}; the old loop printed none)` },
|
||||
{ ok: !!restarted && !!recovered, what: 'the worker restarted and a rate above 0 followed' },
|
||||
], {
|
||||
silence_to_fault: fault ? s(fault.t - tInject) : 'n/a',
|
||||
fault_to_first_healthy_status: fault && recovered ? s(recovered.t - fault.t) : 'n/a',
|
||||
silence_to_first_healthy_status: recovered ? s(recovered.t - tInject) : 'n/a',
|
||||
});
|
||||
};
|
||||
|
||||
S['prepare-flap'] = async () => {
|
||||
// a CPU block producer so the DAA moves and epochs turn every 60 blocks (lead 20)
|
||||
const cpu = spawn(MINER, ['mine', `grpc://127.0.0.1:${BASE}`, '2', '400', 'cpu', '--engine', 'igneum-pow', '--no-vote', '--payout-label', 'cpu'], { stdio: ['ignore', 'ignore', 'ignore'] });
|
||||
started.push(cpu);
|
||||
log(`cpu block producer pid ${cpu.pid}`);
|
||||
setMode('preparefail');
|
||||
const m = startMiner(360);
|
||||
await waitFor(m, /worker: ready/, 20000);
|
||||
await sleep(300000);
|
||||
stop(m); try { cpu.kill('SIGTERM'); } catch {}
|
||||
const sent = find(m, /PREPARE sent for epoch seed (\S+) day (\d+)/);
|
||||
const held = find(m, /PREPARE held/);
|
||||
const failed = find(m, /could not prepare/);
|
||||
const changes = find(m, /SEED CHANGE/);
|
||||
// per epoch (between SEED CHANGE lines): sends per pair and the smallest gap between any two sends
|
||||
const epochs = [];
|
||||
const bounds = [0, ...changes.map(l => l.t), Infinity];
|
||||
for (let i = 0; i + 1 < bounds.length; i++) {
|
||||
const inEpoch = sent.filter(l => l.t >= bounds[i] && l.t < bounds[i + 1]);
|
||||
const perPair = {};
|
||||
for (const l of inEpoch) { const k = /epoch seed (\S+) day (\d+)/.exec(l.text).slice(1).join('/'); perPair[k] = (perPair[k] || 0) + 1; }
|
||||
epochs.push({ sends: inEpoch.length, maxPerPair: Math.max(0, ...Object.values(perPair)) });
|
||||
}
|
||||
const gaps = sent.slice(1).map((l, i) => l.t - sent[i].t);
|
||||
verdict('prepare-flap', [
|
||||
{ ok: changes.length >= 2, what: `epochs turned (${changes.length} seed changes in 5 min)` },
|
||||
{ ok: sent.length >= 2, what: `prepares were sent (${sent.length})` },
|
||||
{ ok: failed.length >= 1, what: `the worker refused them (${failed.length} prepare-failed)` },
|
||||
{ ok: epochs.every(e => e.maxPerPair <= 2), what: `at most two sends per pair per epoch (${epochs.map(e => e.maxPerPair).join(', ')})` },
|
||||
{ ok: gaps.every(g => g >= 29500), what: `every send at least 30 s after the previous (min gap ${gaps.length ? s(Math.min(...gaps)) : 'n/a'})` },
|
||||
{ ok: held.length >= 1, what: `the miner said why it held a prepare (${held.length} lines)` },
|
||||
], { sends: sent.length, held: held.length, seed_changes: changes.length });
|
||||
};
|
||||
|
||||
const names = ONLY.length ? ONLY : Object.keys(S);
|
||||
startNode({ IGNEUM_POW_EPOCH_BLOCKS: '60', IGNEUM_POW_EPOCH_LEAD: '20', IGNEUM_DEVNET_GENESIS_BITS: '0x1f010000' });
|
||||
await sleep(4000);
|
||||
for (const n of names) {
|
||||
if (!S[n]) { log(`unknown scenario ${n}`); continue; }
|
||||
log(`=== ${n}`);
|
||||
try { await S[n](); } catch (e) { log(`scenario ${n} threw: ${e.stack || e}`); results.push({ name: n, pass: false, checks: [{ ok: false, what: String(e) }], metrics: {} }); }
|
||||
await sleep(2000);
|
||||
}
|
||||
const report = { date: new Date().toISOString(), miner: MINER, node: NODE, scratch: SCRATCH, results };
|
||||
writeFileSync(join(SCRATCH, 'report.json'), JSON.stringify(report, null, 2));
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
process.exit(results.every(r => r.pass) ? 0 : 1);
|
||||
Loading…
Reference in a new issue