7eed16a29a
The sweep (main's item 1): 199 tracked text files, 783 lines. The founder's full name, first name and possessive become "the founder" (sentence starts capitalised); the lowercase operating-system user name in WSL paths and commands becomes <user>; the second owner login becomes "the second owner login"; the three earlier businesses and the two other brands become "the other business", "the earlier entity", "the earlier business" and "another brand"; the Chrome profile rule names the igneum.network profile, not the profile's label. The standing commit login igneum-labs is not a founder term here: the fresh-repository step renames it in the history (docs/plans/history-rewrite.md, tools/repo/fresh-repo.sh). The patterns never appear in plain text in the tree (a plaintext list would be the hit): tools/ci/founder-strings.b64 (perl regex, tab, a sample per row) is read by tools/ci/founder-strings-check.sh (every tracked text file, perl, known-failed first: the self-test plants each row's sample in a fixture and the hit must name the file), by tools/community/discord-hooks.mjs (the guard's founder and business rows; the test takes its fixtures from the samples) and by tools/repo/fresh-repo.sh (the business names of the rewrite rules). site/forbidden-strings.txt carries the same patterns as b64: lines, decoded case-insensitive by site/scrub.mjs and tools/ci/launch-gates-check.mjs (whose fixture now plants an encoded made-up name). The check runs in the gate's tree checks on every merge. Not in this commit, by main's word: the 105 commit messages and 40 personal-identity commits that need the history rewrite (listed, not run), and the secrets found by gitleaks over the history (reported with owners). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4699 lines
238 KiB
Rust
4699 lines
238 KiB
Rust
//! The supervisor: starts the node, waits until it is synced, starts one miner per card, reads every line they print
|
|
//! into the state the dashboard shows, restarts what crashes, and stops everything in order (miners first, then the
|
|
//! node) on quit. The command lines are the ones today's launchers use (packaging/mac/app/igneum-miner.sh and
|
|
//! proto-cuda/windows-app/igneum-common.ps1).
|
|
|
|
use crate::config::{Packaged, Runtime, Settings};
|
|
use crate::detect::Bins;
|
|
use crate::keys;
|
|
use crate::procs::{self, Line, Proc, Source};
|
|
use crate::state::{CardState, Rings, State};
|
|
use serde_json::{json, Value};
|
|
use std::io::Write;
|
|
use std::path::PathBuf;
|
|
use std::sync::mpsc::{channel, Receiver, Sender};
|
|
use std::sync::{Arc, Mutex};
|
|
use std::time::{Duration, Instant};
|
|
|
|
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
|
const POW_EPOCH_BLOCKS: u64 = 3_600;
|
|
|
|
#[derive(Clone, Debug, PartialEq)]
|
|
pub struct CardChoice {
|
|
pub key: String,
|
|
pub enabled: bool,
|
|
pub identities: u32,
|
|
pub power_pct: Option<u32>,
|
|
}
|
|
|
|
pub enum Cmd {
|
|
Detect,
|
|
ApplyCards(Vec<CardChoice>),
|
|
/// one enumeration of the cards finished (the first, or a re-detection: src/hotplug.rs)
|
|
Detected(crate::detect::Detection),
|
|
Start,
|
|
Pause,
|
|
Resume,
|
|
RestartMiners(String),
|
|
CheckUpdate,
|
|
InstallUpdate,
|
|
AutoUpdate(bool),
|
|
OpenUpdateFile,
|
|
/// the over-the-air updater's threads report here (src/ota.rs)
|
|
Ota(crate::ota::Event),
|
|
/// the remote-job runner's threads report here (src/jobrun.rs)
|
|
Job(crate::jobrun::Event),
|
|
JobsAllow(bool),
|
|
JobsCheck,
|
|
WorkerBuilt(usize, Result<PathBuf, String>),
|
|
/// local minus the latest block's timestamp, seconds (from the node's EVM RPC)
|
|
ClockSample(f64),
|
|
/// local minus an HTTPS Date header, seconds; None when the check failed
|
|
ClockHttps(Option<f64>),
|
|
ClockCheck,
|
|
ClockSync,
|
|
ClockSynced(Result<String, String>),
|
|
/// the elevated nvidia-smi -pl step finished: (what was asked, how it ran, the limits read back per device)
|
|
PowerApplied(String, Result<(), String>, std::collections::HashMap<String, f64>),
|
|
/// the window host ran the elevated command line (Windows): Ok or the reason
|
|
ElevatedDone(Result<(), String>),
|
|
/// the user asked for the cap again (the Retry button)
|
|
ApplyPower,
|
|
/// the efficiency sweep (src/sweep.rs): start on one card (by key), stop, pin or unpin a card's cap, the toggle
|
|
SweepStart(String),
|
|
SweepStop,
|
|
SweepPin(String, bool),
|
|
SweepEnable(bool),
|
|
/// Settings > Power control (config.rs power_control): on asks for administrator rights once, at that moment
|
|
PowerControl(bool),
|
|
/// Ember Tune's probe of a card (src/ember.rs): the vendor's clock range, the driver, and how settings reach the
|
|
/// card (direct, the elevated helper, the AMD helper, or measure only)
|
|
TuneProbe(usize, Result<TuneProbe, String>),
|
|
/// the elevated helper process ended (Ok at `quit`, Err when the prompt was cancelled or it failed)
|
|
SweepHelperDone(Result<(), String>),
|
|
/// a tune request (by number) was carried out: what the vendor tool printed, or why it refused
|
|
TuneSet(u64, Result<String, String>),
|
|
/// POST /api/tune-progress from a measurement engine beside this app: the card's live tune state
|
|
TuneProgress(Value),
|
|
/// Ember 2: Settings > goal, electricity price, the hill-climb switch
|
|
TuneGoal(Option<String>, Option<f64>, Option<bool>),
|
|
/// restart the node with the verifier decided again (src/verifier.rs): the trust setting changed, or the
|
|
/// prover found a host that was not there when the node started
|
|
RestartNode(String),
|
|
/// quit, with its source (the log names it: C35, 5 October 2026, two unexplained quits)
|
|
Quit(&'static str),
|
|
}
|
|
|
|
pub struct Shared {
|
|
pub token: String,
|
|
pub state: Mutex<State>,
|
|
pub rings: Mutex<Rings>,
|
|
pub settings: Mutex<Settings>,
|
|
pub settings_path: PathBuf,
|
|
pub wallet_path: PathBuf,
|
|
pub runtime: Runtime,
|
|
pub packaged: Packaged,
|
|
cmd_tx: Mutex<Sender<Cmd>>,
|
|
pub started: Instant,
|
|
engine_log: Mutex<Option<std::fs::File>>,
|
|
/// the app log's path: the one the uploader sends (main.rs names it; the engine must not guess the stamp)
|
|
pub log_path: PathBuf,
|
|
port: std::sync::atomic::AtomicU16,
|
|
/// set once `state_json` has logged a serialisation error (one line, not one per poll)
|
|
state_error_logged: std::sync::atomic::AtomicBool,
|
|
}
|
|
|
|
impl Shared {
|
|
pub fn new(token: String, runtime: Runtime, packaged: Packaged, settings: Settings, cmd_tx: Sender<Cmd>, engine_log: Option<std::fs::File>, log_path: PathBuf) -> Shared {
|
|
let settings_path = runtime.app_dir.join("settings.json");
|
|
let wallet_path = runtime.app_dir.join("wallet.json");
|
|
let mut st = State { version: VERSION.into(), ..Default::default() };
|
|
st.network = runtime.network.clone();
|
|
st.chain = if runtime.network == "devnet" { "devnet v4".into() } else { format!("{} (private)", runtime.network) };
|
|
st.host = runtime.host.clone();
|
|
st.display_name = if settings.display_name.is_empty() { runtime.host.clone() } else { settings.display_name.clone() };
|
|
st.machine_id = runtime.machine_id.clone();
|
|
st.node_dir = runtime.node_dir.display().to_string();
|
|
st.log_dir = runtime.log_dir.display().to_string();
|
|
st.setup_done = settings.setup_done;
|
|
st.phase = if settings.setup_done { "dashboard".into() } else { "welcome".into() };
|
|
st.node.state = "stopped".into();
|
|
st.mining.state = "idle".into();
|
|
st.mining.paused = settings.paused;
|
|
st.mining.accepted_total = settings.accepted_total;
|
|
st.mining.fee_total = settings.fee_total;
|
|
st.address = address_state(&settings, &wallet_path);
|
|
st.settings = crate::state::SettingsState { identities: settings.identities, vote: settings.vote, start_at_login: crate::platform::start_at_login_is_on(), auto_update: settings.auto_update, remote_jobs: settings.remote_jobs, prove: settings.prove, sweep: settings.sweep, power_control: settings.power_control, power_note: String::new(), tuning_off: false, tuning_note: String::new(), tune_goal: settings.tune_goal.clone(), power_price_pence: settings.power_price_pence, tune_climb: settings.tune_climb, tune_period_s: crate::ember::PERIOD_S, dev_fee: settings.dev_fee, proof_verify_trust: settings.proof_verify_trust };
|
|
st.dev_fee = crate::state::DevFeeState { on: settings.dev_fee, percent: if settings.dev_fee { 1 } else { 0 }, address: String::new(), line: String::new() };
|
|
st.live_page = packaged.live_page.clone();
|
|
st.finality.message = "waiting for the miner".into();
|
|
st.clock.hint = crate::platform::clock_hint().into();
|
|
st.clock.severity = "none".into();
|
|
st.program.message = "waiting for the node".into();
|
|
Shared {
|
|
token,
|
|
state: Mutex::new(st),
|
|
rings: Mutex::new(Rings::new()),
|
|
settings: Mutex::new(settings),
|
|
settings_path,
|
|
wallet_path,
|
|
runtime,
|
|
packaged,
|
|
cmd_tx: Mutex::new(cmd_tx),
|
|
started: Instant::now(),
|
|
engine_log: Mutex::new(engine_log),
|
|
log_path,
|
|
port: std::sync::atomic::AtomicU16::new(0),
|
|
state_error_logged: std::sync::atomic::AtomicBool::new(false),
|
|
}
|
|
}
|
|
|
|
/// The first line of every upload, so the console's Machines tab parses it without guessing:
|
|
/// IGNEUM-APP version=<v> machine=<id8> platform=<os> node=<igneumd version>. Logged at every engine start too,
|
|
/// so the restart after an OTA apply carries it again.
|
|
pub fn upload_header(&self) -> String {
|
|
let node = self.state.lock().unwrap().node.version.clone();
|
|
format!("IGNEUM-APP version={} machine={} platform={} node={}", VERSION, self.runtime.id8(), crate::manifest::platform_name(), if node.is_empty() { "unknown".to_string() } else { node.replace(' ', "_") })
|
|
}
|
|
|
|
pub fn set_port(&self, p: u16) {
|
|
self.port.store(p, std::sync::atomic::Ordering::Relaxed);
|
|
}
|
|
pub fn port(&self) -> u16 {
|
|
self.port.load(std::sync::atomic::Ordering::Relaxed)
|
|
}
|
|
|
|
pub fn send(&self, c: Cmd) {
|
|
let _ = self.cmd_tx.lock().unwrap().send(c);
|
|
}
|
|
|
|
pub fn log(&self, text: &str) {
|
|
let text = &crate::platform::redact(text);
|
|
let stamp = crate::platform::unix_now_f();
|
|
if let Some(f) = self.engine_log.lock().unwrap().as_mut() {
|
|
let _ = writeln!(f, "{stamp:.0} {text}");
|
|
}
|
|
self.rings.lock().unwrap().log("app", false, text);
|
|
}
|
|
|
|
pub fn event(&self, kind: &str, text: &str) {
|
|
let text = &crate::platform::redact(text);
|
|
self.rings.lock().unwrap().event(kind, text);
|
|
self.log(&format!("[{kind}] {text}"));
|
|
}
|
|
|
|
pub fn state_json(&self) -> Value {
|
|
let mut st = self.state.lock().unwrap().clone();
|
|
st.now = crate::platform::unix_now_f();
|
|
st.uptime_s = self.started.elapsed().as_secs();
|
|
st.events = self.rings.lock().unwrap().events.iter().cloned().collect();
|
|
match serde_json::to_value(st) {
|
|
Ok(v) => v,
|
|
Err(e) => {
|
|
// never a silent "{}": the dashboard and every PC playbook read this reply
|
|
let msg = format!("state_json: {e}");
|
|
if !self.state_error_logged.swap(true, std::sync::atomic::Ordering::Relaxed) {
|
|
self.log(&format!("[error] {msg}"));
|
|
}
|
|
json!({ "error": msg, "version": env!("CARGO_PKG_VERSION") })
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A short state for the window host (menu bar, tray).
|
|
pub fn wrapper_state(&self) -> Value {
|
|
let st = self.state.lock().unwrap();
|
|
json!({
|
|
"phase": st.phase, "mining": st.mining.state, "paused": st.mining.paused,
|
|
"hash_total": st.mining.hash_total, "accepted_total": st.mining.accepted_total,
|
|
"node": st.node.state, "blocks": st.node.blocks, "peers": st.node.peers, "quitting": st.quitting,
|
|
"update": st.update.available,
|
|
})
|
|
}
|
|
|
|
fn save_settings(&self) {
|
|
self.settings.lock().unwrap().save(&self.settings_path);
|
|
}
|
|
|
|
/// First-run setup: the payout address (a generated key, or one the user pasted) and the identity count.
|
|
pub fn setup(&self, mode: &str, address: Option<&str>, identities: Option<u32>) -> Result<Value, String> {
|
|
let mut out = json!({ "ok": true });
|
|
{
|
|
let mut s = self.settings.lock().unwrap();
|
|
if let Some(n) = identities {
|
|
s.identities = n;
|
|
}
|
|
match mode {
|
|
"paste" => {
|
|
let a = address.ok_or("address missing")?.trim().to_string();
|
|
if !keys::valid_address(&a) {
|
|
return Err("an address is 0x followed by 40 hex characters".into());
|
|
}
|
|
s.address = a.to_ascii_lowercase();
|
|
s.address_source = "pasted".into();
|
|
s.key_saved = true;
|
|
}
|
|
_ => {
|
|
let wallet = match keys::load(&self.wallet_path) {
|
|
Some(w) => w,
|
|
None => {
|
|
let w = keys::generate();
|
|
keys::save(&self.wallet_path, &w).map_err(|e| format!("could not write the wallet file: {e}"))?;
|
|
w
|
|
}
|
|
};
|
|
s.address = wallet.address.clone();
|
|
s.address_source = "generated".into();
|
|
s.key_saved = false;
|
|
out["private_key"] = json!(wallet.private_key);
|
|
}
|
|
}
|
|
s.save(&self.settings_path);
|
|
out["address"] = json!(s.address);
|
|
out["display"] = json!(keys::checksum(&s.address));
|
|
out["source"] = json!(s.address_source);
|
|
out["wallet_file"] = json!(self.wallet_path.display().to_string());
|
|
let mut st = self.state.lock().unwrap();
|
|
st.address = address_state(&s, &self.wallet_path);
|
|
st.settings.identities = s.identities;
|
|
}
|
|
self.event("ok", &format!("rewards go to {}", self.state.lock().unwrap().address.display));
|
|
Ok(out)
|
|
}
|
|
|
|
pub fn key_saved(&self) {
|
|
self.settings.lock().unwrap().key_saved = true;
|
|
self.save_settings();
|
|
self.state.lock().unwrap().address.key_saved = true;
|
|
}
|
|
|
|
pub fn reveal_key(&self) -> Result<Value, String> {
|
|
let w = keys::load(&self.wallet_path).ok_or("no wallet file: the address was pasted, not made here")?;
|
|
self.log("the private key was shown in the window (settings > reveal)");
|
|
Ok(json!({ "ok": true, "address": w.address, "display": keys::checksum(&w.address), "private_key": w.private_key, "wallet_file": self.wallet_path.display().to_string() }))
|
|
}
|
|
|
|
/// Proving v1 step 1 (5 October 2026): once per install, after the cards are known, the prover goes on by itself
|
|
/// when this machine can prove (src/provedefault.rs: an NVIDIA card with 12 GB or more, WSL2 on Windows, Linux
|
|
/// native, Apple silicon off until measured). An explicit on is never switched off; the line goes to the log and
|
|
/// to the Proving tile. Older installs apply it at their first start on this version.
|
|
pub fn apply_prove_default(&self) {
|
|
let (applied, already_on) = {
|
|
let s = self.settings.lock().unwrap();
|
|
(s.prove_default_applied, s.prove)
|
|
};
|
|
if applied {
|
|
return;
|
|
}
|
|
let cards = self.state.lock().unwrap().mining.cards.clone();
|
|
let wsl = if cfg!(windows) { Some(crate::wslhost::distro_answers()) } else { None };
|
|
let d = crate::provedefault::decide(&cards, std::env::consts::OS, wsl, crate::detect::total_ram_mb());
|
|
let on = d.on || already_on;
|
|
{
|
|
let mut s = self.settings.lock().unwrap();
|
|
s.prove = on;
|
|
s.prove_default_applied = true;
|
|
s.save(&self.settings_path);
|
|
}
|
|
{
|
|
let mut st = self.state.lock().unwrap();
|
|
st.settings.prove = on;
|
|
st.proving.enabled = on;
|
|
st.proving.default_note = d.line.clone();
|
|
if !on {
|
|
st.proving.status = "off".into();
|
|
}
|
|
}
|
|
self.log(&format!("prover default: {}{}", d.line, if already_on && !d.on { " (left on: it was switched on by hand)" } else { "" }));
|
|
}
|
|
|
|
/// The prover service switch (src/prover.rs); the thread picks it up within 10 s.
|
|
pub fn set_prove(&self, on: bool) -> Result<Value, String> {
|
|
{
|
|
let mut s = self.settings.lock().unwrap();
|
|
s.prove = on;
|
|
}
|
|
self.save_settings();
|
|
let mut st = self.state.lock().unwrap();
|
|
st.settings.prove = on;
|
|
st.proving.enabled = on;
|
|
if !on {
|
|
st.proving.status = "off".into();
|
|
}
|
|
Ok(json!({ "ok": true }))
|
|
}
|
|
|
|
pub fn apply_settings(&self, identities: Option<u32>, vote: Option<bool>, login: Option<bool>, address: Option<&str>, display_name: Option<&str>, dev_fee: Option<bool>, proof_verify_trust: Option<bool>) -> Result<Value, String> {
|
|
let mut restart = Vec::new();
|
|
let mut restart_node: Option<String> = None;
|
|
{
|
|
let mut s = self.settings.lock().unwrap();
|
|
if let Some(n) = display_name {
|
|
let n: String = n.trim().chars().take(40).collect();
|
|
s.display_name = n.clone();
|
|
self.state.lock().unwrap().display_name = if n.is_empty() { self.runtime.host.clone() } else { n };
|
|
}
|
|
if let Some(n) = identities {
|
|
if n != s.identities {
|
|
s.identities = n;
|
|
restart.push(format!("{n} identities"));
|
|
}
|
|
}
|
|
if let Some(v) = vote {
|
|
if v != s.vote {
|
|
s.vote = v;
|
|
restart.push(if v { "voting on".into() } else { "voting off".into() });
|
|
}
|
|
}
|
|
if let Some(v) = dev_fee {
|
|
if v != s.dev_fee {
|
|
s.dev_fee = v;
|
|
restart.push(if v { "dev fee on (1 block in 100)".into() } else { "dev fee off".into() });
|
|
}
|
|
}
|
|
if let Some(v) = proof_verify_trust {
|
|
if v != s.proof_verify_trust {
|
|
s.proof_verify_trust = v;
|
|
restart_node = Some(if v { "proof trust mode on (devnet only)".into() } else { "proof trust mode off".into() });
|
|
}
|
|
}
|
|
if let Some(a) = address {
|
|
let a = a.trim().to_ascii_lowercase();
|
|
if !a.is_empty() && a != s.address {
|
|
if !keys::valid_address(&a) {
|
|
return Err("an address is 0x followed by 40 hex characters".into());
|
|
}
|
|
let generated = keys::load(&self.wallet_path).map(|w| w.address == a).unwrap_or(false);
|
|
s.address = a.clone();
|
|
s.address_source = if generated { "generated".into() } else { "pasted".into() };
|
|
s.key_saved = true;
|
|
restart.push(format!("rewards to {}", keys::checksum(&a)));
|
|
}
|
|
}
|
|
s.save(&self.settings_path);
|
|
let mut st = self.state.lock().unwrap();
|
|
st.settings.identities = s.identities;
|
|
st.settings.vote = s.vote;
|
|
st.settings.dev_fee = s.dev_fee;
|
|
st.settings.proof_verify_trust = s.proof_verify_trust;
|
|
st.dev_fee.on = s.dev_fee;
|
|
st.dev_fee.percent = if s.dev_fee { 1 } else { 0 };
|
|
st.address = address_state(&s, &self.wallet_path);
|
|
}
|
|
if let Some(on) = login {
|
|
crate::platform::set_start_at_login(on)?;
|
|
self.state.lock().unwrap().settings.start_at_login = crate::platform::start_at_login_is_on();
|
|
self.event("info", if on { "starts at login from now on" } else { "no longer starts at login" });
|
|
}
|
|
if !restart.is_empty() {
|
|
self.send(Cmd::RestartMiners(restart.join(", ")));
|
|
}
|
|
if let Some(why) = restart_node.clone() {
|
|
self.send(Cmd::RestartNode(why));
|
|
}
|
|
Ok(json!({ "ok": true, "restart": !restart.is_empty(), "restart_node": restart_node.is_some() }))
|
|
}
|
|
}
|
|
|
|
fn address_state(s: &Settings, wallet_path: &std::path::Path) -> crate::state::AddressState {
|
|
crate::state::AddressState {
|
|
value: s.address.clone(),
|
|
display: if s.address.is_empty() { String::new() } else { keys::checksum(&s.address) },
|
|
source: s.address_source.clone(),
|
|
key_saved: s.key_saved,
|
|
wallet_file: if s.address_source == "generated" { wallet_path.display().to_string() } else { String::new() },
|
|
}
|
|
}
|
|
|
|
/// The node's consensus params digest out of its own line ("Consensus params digest: <64 hex> (exchanged in the p2p
|
|
/// handshake; ...)"). None for any other line, including the WARN lines that quote a peer's digest.
|
|
pub fn digest_from_line(line: &str) -> Option<String> {
|
|
let rest = line.split("Consensus params digest:").nth(1)?.trim_start();
|
|
let hex: String = rest.chars().take_while(|c| c.is_ascii_hexdigit()).collect();
|
|
(hex.len() == 64).then(|| hex.to_ascii_lowercase())
|
|
}
|
|
|
|
/// The planned rule changes in an override file: every `<name>_activation_daa` key, lowest height first, with plain
|
|
/// words for the name ("fees_v1" -> "Fees v1"). Keys that are not activation heights are left out.
|
|
pub fn switches_of(v: &Value) -> Vec<crate::state::ConsensusSwitch> {
|
|
let mut out: Vec<crate::state::ConsensusSwitch> = v
|
|
.as_object()
|
|
.map(|o| {
|
|
o.iter()
|
|
.filter_map(|(k, val)| {
|
|
let stem = k.strip_suffix("_activation_daa")?;
|
|
let daa = val.as_u64()?;
|
|
let mut words: Vec<String> = stem.split('_').map(|w| w.to_string()).collect();
|
|
if let Some(first) = words.first_mut() {
|
|
let mut c = first.chars();
|
|
if let Some(f) = c.next() {
|
|
*first = f.to_ascii_uppercase().to_string() + c.as_str();
|
|
}
|
|
}
|
|
Some(crate::state::ConsensusSwitch { key: k.clone(), name: words.join(" "), daa })
|
|
})
|
|
.collect()
|
|
})
|
|
.unwrap_or_default();
|
|
out.sort_by_key(|s| (s.daa, s.key.clone()));
|
|
out
|
|
}
|
|
|
|
/// One value out of `key=value` tokens of a log line.
|
|
fn kv<'a>(line: &'a str, key: &str) -> Option<&'a str> {
|
|
let pat = format!(" {key}=");
|
|
let i = line.find(&pat)? + pat.len();
|
|
let rest = &line[i..];
|
|
let end = rest.find(|c: char| c == ' ' || c == ',' || c == ')' || c == ';').unwrap_or(rest.len());
|
|
Some(&rest[..end])
|
|
}
|
|
|
|
fn kv_u64(line: &str, key: &str) -> Option<u64> {
|
|
kv(line, key)?.parse().ok()
|
|
}
|
|
|
|
fn kv_f64(line: &str, key: &str) -> Option<f64> {
|
|
kv(line, key)?.trim_end_matches('s').parse().ok()
|
|
}
|
|
|
|
fn fmt_uptime(s: u64) -> String {
|
|
format!("{:02}:{:02}:{:02}", s / 3600, s % 3600 / 60, s % 60)
|
|
}
|
|
|
|
fn port_open(port: u16) -> bool {
|
|
std::net::TcpStream::connect_timeout(&std::net::SocketAddr::from(([127, 0, 0, 1], port)), Duration::from_millis(800)).is_ok()
|
|
}
|
|
|
|
fn jitter_secs(seed: u64) -> u64 {
|
|
// 5 to 60 s, from the clock: enough spread for a restart, no rng needed
|
|
5 + (seed.wrapping_mul(2654435761) >> 7) % 56
|
|
}
|
|
|
|
/// The `--prepare-packs` directory the miner gets, relative to the app data folder (its cwd), in the platform's
|
|
/// separator: `packs\prepare` on Windows, `packs/prepare` elsewhere. Every worker gets it (program class v3).
|
|
pub(crate) fn prepare_packs_arg() -> String {
|
|
if cfg!(windows) { "packs\\prepare".to_string() } else { "packs/prepare".to_string() }
|
|
}
|
|
|
|
/// Seconds after a resume before every enabled card must be mining (a worker takes 10 to 60 s to its first STATUS
|
|
/// line with a hash rate; the pack export before it a few seconds more).
|
|
pub(crate) const RESUME_CHECK_SECS: u64 = 90;
|
|
|
|
/// What the resume rule reads from a miner slot.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub(crate) struct ResumeSlot {
|
|
/// the watchdog marked the card faulted
|
|
pub faulted: bool,
|
|
/// a worker process is alive on the slot
|
|
pub live: bool,
|
|
}
|
|
|
|
/// The resume rule: every slot without a live worker is re-armed, faulted or not. (The rule before 5 October 2026
|
|
/// re-armed only faulted slots; `stop_miners("paused")` had cleared every slot's `restart_at`, so a healthy paused
|
|
/// card never restarted: PC 2 at 21:25:11Z, the Mac that afternoon.)
|
|
pub(crate) fn slots_to_rearm_on_resume(slots: &[ResumeSlot]) -> Vec<usize> {
|
|
slots.iter().enumerate().filter(|(_, s)| !s.live).map(|(i, _)| i).collect()
|
|
}
|
|
|
|
/// The check RESUME_CHECK_SECS after a resume: every enabled, present, non-faulted card must report a hash rate
|
|
/// above 0 or its name is returned with its state (the caller logs one line per card).
|
|
pub(crate) fn resume_check(cards: &[CardState]) -> Vec<String> {
|
|
cards
|
|
.iter()
|
|
.filter(|c| c.enabled && c.present() && c.state != "faulted" && c.hash_now <= 0.0)
|
|
.map(|c| format!("{} is not mining {RESUME_CHECK_SECS} s after resume (state {}, pid {}{})", c.name, c.state, c.pid, if c.message.is_empty() { String::new() } else { format!(", {}", c.message) }))
|
|
.collect()
|
|
}
|
|
|
|
struct MinerSlot {
|
|
card: usize,
|
|
label: String,
|
|
proc: Option<Proc>,
|
|
worker: PathBuf,
|
|
restart_at: Option<Instant>,
|
|
starts: u32,
|
|
restarts: u32,
|
|
building: bool,
|
|
needs_rebuild: bool,
|
|
prepared: bool, // the pack is exported (and the worker built) for the next start
|
|
last_status: Option<Instant>,
|
|
error_at: Option<Instant>,
|
|
/// the per-card watchdog (src/watchdog.rs): one restart, then faulted
|
|
watch: crate::watchdog::CardWatch,
|
|
/// pack refusals (src/watchdog.rs): the pack is exported again before the restart, capped per epoch
|
|
pack_rebuilds: crate::watchdog::PackRebuilds,
|
|
}
|
|
|
|
pub struct Engine {
|
|
shared: Arc<Shared>,
|
|
bins: Bins,
|
|
rx: Receiver<Cmd>,
|
|
lines_rx: Receiver<Line>,
|
|
lines_tx: Sender<Line>,
|
|
node: Option<Proc>,
|
|
node_external: bool,
|
|
node_restart_at: Option<Instant>,
|
|
/// resume rule (5 October 2026): when due, every enabled card must be mining or its name goes to the log
|
|
resume_check_at: Option<Instant>,
|
|
node_started_at: Instant,
|
|
node_starts: u32,
|
|
node_restarts: u32,
|
|
node_log: Option<PathBuf>,
|
|
node_last_reading: Option<Instant>,
|
|
/// the proof verifier decided for the node (src/verifier.rs); None = decide at the next node start
|
|
verifier: Option<crate::verifier::Verifier>,
|
|
sync_prev: Option<u64>,
|
|
sync_stable_since: Option<Instant>,
|
|
last_sync_check: Instant,
|
|
no_peer_warned: bool,
|
|
watch: Option<Proc>,
|
|
watch_retry_at: Instant,
|
|
miners: Vec<MinerSlot>,
|
|
running: bool,
|
|
quitting: bool,
|
|
keep_awake: Option<crate::platform::KeepAwake>,
|
|
last_awake: Instant,
|
|
last_status: Instant,
|
|
last_upload: Instant,
|
|
upload_thread: Option<std::thread::JoinHandle<()>>,
|
|
ota: crate::ota::Updater,
|
|
jobs: crate::jobrun::Jobs,
|
|
/// a remote job has the miners stopped; they restart when the runner lets go
|
|
job_hold: bool,
|
|
stamp: String,
|
|
label_base: String,
|
|
wrapper: bool,
|
|
last_state_print: Instant,
|
|
detected: bool,
|
|
/// hot-plug (src/hotplug.rs): an enumeration thread is running; one more was asked for meanwhile; when the
|
|
/// next periodic one is due; when the app log last carried the card list
|
|
detect_busy: bool,
|
|
detect_again: bool,
|
|
detect_next: Instant,
|
|
last_cards_line: Instant,
|
|
clock_samples: Vec<f64>,
|
|
clock_node_at: Option<Instant>,
|
|
clock_node_behind: f64,
|
|
clock_https: Option<(f64, Instant)>,
|
|
clock_next_https: Instant,
|
|
clock_last_sample: Instant,
|
|
last_accepted: Option<Instant>,
|
|
telemetry: Option<Proc>,
|
|
telemetry_retry_at: Instant,
|
|
amd_telemetry: Option<Proc>,
|
|
amd_telemetry_retry_at: Instant,
|
|
power_busy: bool,
|
|
power_restore_pending: bool,
|
|
/// an elevated step handed to the window host: (command line, what, requested watts per device, since)
|
|
power_via_host: Option<(String, String, std::collections::HashMap<String, f64>, Instant)>,
|
|
/// the manifest's consensus override changed: restart the node at a safe moment
|
|
node_override_restart: bool,
|
|
/// this node build refused the override file (an older igneumd without the field): start without it
|
|
node_override_unusable: bool,
|
|
stability: std::collections::HashMap<usize, Stability>,
|
|
last_stability: Instant,
|
|
last_settings_save: Instant,
|
|
last_error_event: Instant,
|
|
// the efficiency sweep (src/sweep.rs): one card at a time
|
|
sweep: Option<crate::ember::Run>,
|
|
/// who asked for the quit (the log's `quit:` line names it)
|
|
quit_source: &'static str,
|
|
/// the over-the-air updater never runs: IGNEUM_APP_NO_OTA=1 or --sweep (a second engine beside the installed app)
|
|
no_ota: bool,
|
|
/// the Igneum Power Helper task is registered (src/powertask.rs): once, ever; None = not asked yet
|
|
power_task: Option<bool>,
|
|
/// the running tune helper is the task (quit ends it; no SweepHelperDone comes from a thread)
|
|
sweep_helper_is_task: bool,
|
|
/// the request number the vendor tool last carried out (the run's acknowledgement)
|
|
tune_acked: Option<u64>,
|
|
/// cards whose confirm check found a better neighbour: the full plan runs next
|
|
tune_full_due: std::collections::HashSet<usize>,
|
|
/// a sweep waiting for the cap-mode probe: (card index, forced)
|
|
sweep_pending: Option<(usize, bool)>,
|
|
/// how caps are set: Some(true) directly, Some(false) through the elevated helper, None = not probed yet
|
|
sweep_direct: Option<bool>,
|
|
/// the elevated helper is running (its thread reports SweepHelperDone when it ends)
|
|
sweep_helper: bool,
|
|
sweep_seq: u64,
|
|
/// cards asked for by hand (Sweep now) or by --sweep, by key
|
|
sweep_queue: Vec<String>,
|
|
sweep_next_check: Instant,
|
|
/// after an abort, a card waits before the scheduler tries it again
|
|
sweep_retry: std::collections::HashMap<usize, Instant>,
|
|
sweep_attempts: std::collections::HashMap<usize, u32>,
|
|
/// the node watchdog (src/watchdog.rs): a silent node of ours is restarted
|
|
node_watch: crate::watchdog::NodeWatch,
|
|
/// the watchdogs' clock origin (they take seconds)
|
|
t0: Instant,
|
|
}
|
|
|
|
impl Engine {
|
|
pub fn new(shared: Arc<Shared>, bins: Bins, rx: Receiver<Cmd>, wrapper: bool) -> Engine {
|
|
let no_ota = shared.runtime.sweep_only || std::env::var("IGNEUM_APP_NO_OTA").map(|v| v == "1").unwrap_or(false);
|
|
let (lines_tx, lines_rx) = channel();
|
|
let now = Instant::now();
|
|
let stamp = {
|
|
let t = crate::platform::unix_now();
|
|
// yyyymmdd-hhmmss in UTC, without a date crate
|
|
let days = t / 86400;
|
|
let (y, m, d) = civil_from_days(days as i64);
|
|
format!("{y:04}{m:02}{d:02}-{:02}{:02}{:02}", t % 86400 / 3600, t % 3600 / 60, t % 60)
|
|
};
|
|
// labels never carry the hostname: two cloned PCs with the same COMPUTERNAME would sign with the same keys
|
|
let label_base = format!("{}-{}", if cfg!(target_os = "macos") { "mac" } else { "win" }, shared.runtime.id8());
|
|
let ota = crate::ota::Updater::new(&shared);
|
|
let jobs = crate::jobrun::Jobs::new(&shared);
|
|
Engine {
|
|
shared,
|
|
bins,
|
|
rx,
|
|
lines_rx,
|
|
lines_tx,
|
|
node: None,
|
|
node_external: false,
|
|
node_restart_at: None,
|
|
resume_check_at: None,
|
|
node_started_at: now,
|
|
node_starts: 0,
|
|
node_restarts: 0,
|
|
node_log: None,
|
|
node_last_reading: None,
|
|
verifier: None,
|
|
sync_prev: None,
|
|
sync_stable_since: None,
|
|
last_sync_check: now,
|
|
no_peer_warned: false,
|
|
watch: None,
|
|
watch_retry_at: now,
|
|
miners: Vec::new(),
|
|
running: false,
|
|
quitting: false,
|
|
keep_awake: None,
|
|
last_awake: now,
|
|
last_status: now,
|
|
last_upload: now,
|
|
upload_thread: None,
|
|
ota,
|
|
jobs,
|
|
job_hold: false,
|
|
stamp,
|
|
label_base,
|
|
wrapper,
|
|
last_state_print: now,
|
|
detected: false,
|
|
detect_busy: false,
|
|
detect_again: false,
|
|
detect_next: now + Duration::from_secs(crate::hotplug::POLL_S),
|
|
last_cards_line: now,
|
|
clock_samples: Vec::new(),
|
|
clock_node_at: None,
|
|
clock_node_behind: 0.0,
|
|
clock_https: None,
|
|
clock_next_https: now + Duration::from_secs(5),
|
|
clock_last_sample: now,
|
|
last_accepted: None,
|
|
telemetry: None,
|
|
telemetry_retry_at: now,
|
|
amd_telemetry: None,
|
|
amd_telemetry_retry_at: now,
|
|
power_busy: false,
|
|
power_restore_pending: false,
|
|
power_via_host: None,
|
|
node_override_restart: false,
|
|
node_override_unusable: false,
|
|
stability: std::collections::HashMap::new(),
|
|
last_stability: now,
|
|
last_settings_save: now,
|
|
last_error_event: now - Duration::from_secs(600),
|
|
sweep: None,
|
|
quit_source: "unknown",
|
|
no_ota,
|
|
power_task: None,
|
|
sweep_helper_is_task: false,
|
|
tune_acked: None,
|
|
tune_full_due: std::collections::HashSet::new(),
|
|
sweep_pending: None,
|
|
sweep_direct: None,
|
|
sweep_helper: false,
|
|
sweep_seq: 0,
|
|
sweep_queue: Vec::new(),
|
|
sweep_next_check: now + Duration::from_secs(20),
|
|
sweep_retry: std::collections::HashMap::new(),
|
|
sweep_attempts: std::collections::HashMap::new(),
|
|
node_watch: crate::watchdog::NodeWatch::new(),
|
|
t0: now,
|
|
}
|
|
}
|
|
|
|
fn st(&self) -> std::sync::MutexGuard<'_, State> {
|
|
self.shared.state.lock().unwrap()
|
|
}
|
|
|
|
/// Seconds on the watchdogs' clock.
|
|
fn secs(&self, now: Instant) -> f64 {
|
|
now.duration_since(self.t0).as_secs_f64()
|
|
}
|
|
|
|
pub fn run(mut self) {
|
|
let setup_done = self.shared.settings.lock().unwrap().setup_done;
|
|
self.shared.log(&format!(
|
|
"Igneum Miner {} on {} (machine id {}), {} (run {}-{})",
|
|
VERSION,
|
|
self.shared.runtime.host,
|
|
self.shared.runtime.machine_id,
|
|
self.st().chain,
|
|
self.label_base,
|
|
self.stamp
|
|
));
|
|
self.shared.log("Nobody from Igneum will ever ask for your seed. Nothing is bought or sold on this network.");
|
|
self.shared.log(&format!(
|
|
"node: RPC 127.0.0.1:{}, p2p 0.0.0.0:{}, peers {}, data {}",
|
|
self.shared.runtime.rpc_port,
|
|
self.shared.runtime.p2p_port,
|
|
if self.shared.runtime.peers.is_empty() { "none".to_string() } else { self.shared.runtime.peers.join(" ") },
|
|
self.shared.runtime.node_dir.display()
|
|
));
|
|
let v = crate::detect::node_version(&self.bins.node);
|
|
self.st().node.version = v.clone();
|
|
self.shared.log(&self.shared.upload_header());
|
|
// which intake and which downloads folder this build reports to and checks (fingerprints, never the values;
|
|
// rotation phase 2 reads this line from every machine's upload: docs/plans/rotation-phase-2.md)
|
|
self.shared.log(&self.shared.packaged.describe());
|
|
// the prover service (proving v0): its own thread, idle until the setting is on
|
|
crate::prover::start(self.shared.clone(), self.bins.dir.clone());
|
|
self.shared.log(&format!("node binary: {} ({v})", self.bins.node.display()));
|
|
if self.ota.needs_rollback() {
|
|
// this version died twice before it was healthy: the helper puts the previous one back
|
|
match self.ota.launch_rollback(&self.shared, self.host_pid()) {
|
|
Ok(()) => {
|
|
self.quitting = true;
|
|
self.st().quitting = true;
|
|
}
|
|
Err(e) => self.shared.event("error", &format!("rollback not possible: {e}")),
|
|
}
|
|
}
|
|
self.shared.send(Cmd::Detect);
|
|
if self.shared.runtime.sweep_only {
|
|
self.shared.log("--sweep: the efficiency sweep runs on every supported card as soon as it mines; the table goes to stdout and this log; the engine quits after it");
|
|
}
|
|
if self.no_ota {
|
|
self.shared.log("updates: off for this engine (IGNEUM_APP_NO_OTA or --sweep): it never downloads or installs, whatever the manifest says (C35)");
|
|
}
|
|
if (setup_done || self.shared.runtime.sweep_only) && !self.quitting {
|
|
self.shared.send(Cmd::Start);
|
|
}
|
|
loop {
|
|
while let Ok(c) = self.rx.try_recv() {
|
|
self.command(c);
|
|
}
|
|
while let Ok(l) = self.lines_rx.try_recv() {
|
|
self.line(l);
|
|
}
|
|
if self.quitting {
|
|
self.shutdown();
|
|
return;
|
|
}
|
|
self.tick();
|
|
std::thread::sleep(Duration::from_millis(250));
|
|
}
|
|
}
|
|
|
|
fn command(&mut self, c: Cmd) {
|
|
match c {
|
|
Cmd::Detect => {
|
|
// the first run, /api/detect, the host's "detect" on a device change, and the minute poll all land
|
|
// here; one enumeration at a time, and a request during one runs once more after it
|
|
if self.detect_busy {
|
|
self.detect_again = true;
|
|
return;
|
|
}
|
|
self.detect_busy = true;
|
|
if !self.detected {
|
|
self.st().detecting = true;
|
|
}
|
|
let bins = self.bins.clone();
|
|
let shared = self.shared.clone();
|
|
std::thread::spawn(move || {
|
|
let d = crate::detect::detect(&bins);
|
|
shared.send(Cmd::Detected(d));
|
|
});
|
|
}
|
|
Cmd::Detected(d) => {
|
|
self.detect_busy = false;
|
|
self.detect_next = Instant::now() + Duration::from_secs(crate::hotplug::POLL_S);
|
|
if self.detected {
|
|
self.merge_detection(d);
|
|
} else {
|
|
self.first_detection(d);
|
|
}
|
|
if self.detect_again {
|
|
self.detect_again = false;
|
|
self.shared.send(Cmd::Detect);
|
|
}
|
|
// proving v1 step 1: the install-time prover default, once the cards are known
|
|
self.shared.apply_prove_default();
|
|
}
|
|
Cmd::ApplyCards(choices) => self.apply_cards(choices),
|
|
Cmd::Start => self.start(),
|
|
Cmd::Pause => {
|
|
if !self.running {
|
|
return;
|
|
}
|
|
self.shared.settings.lock().unwrap().paused = true;
|
|
self.shared.save_settings();
|
|
self.st().mining.paused = true;
|
|
self.stop_miners("paused");
|
|
self.shared.event("info", "mining paused; the node keeps running");
|
|
}
|
|
Cmd::Resume => {
|
|
self.shared.settings.lock().unwrap().paused = false;
|
|
self.shared.save_settings();
|
|
self.st().mining.paused = false;
|
|
self.shared.event("ok", "mining resumed");
|
|
// the resume rule (5 October 2026, PC 2 at 21:25:11Z and the Mac that afternoon): stop_miners("paused")
|
|
// cleared every slot's restart_at and this arm re-armed only FAULTED slots, so a card whose worker had
|
|
// simply been stopped stayed "off" at 0 MH/s until the app was relaunched. Now every slot without a
|
|
// live worker is re-armed (a faulted one reset first), its pack is exported again before the start
|
|
// (prepared = false: the hour may have turned while paused), and a check 90 s later names any
|
|
// enabled card that is not mining (`resume_check`).
|
|
let views: Vec<ResumeSlot> = self.miners.iter().map(|m| ResumeSlot { faulted: m.watch.faulted().is_some(), live: m.proc.is_some() }).collect();
|
|
let now = Instant::now();
|
|
for i in slots_to_rearm_on_resume(&views) {
|
|
let m = &mut self.miners[i];
|
|
if m.watch.faulted().is_some() {
|
|
m.watch.event(0.0, crate::watchdog::Event::Reset);
|
|
}
|
|
m.restart_at = Some(now);
|
|
m.prepared = false;
|
|
}
|
|
self.resume_check_at = Some(now + Duration::from_secs(RESUME_CHECK_SECS));
|
|
if !self.running {
|
|
self.start();
|
|
}
|
|
}
|
|
Cmd::RestartMiners(why) => {
|
|
self.shared.event("info", &format!("settings changed ({why}); the miners restart"));
|
|
self.stop_miners("settings changed");
|
|
for m in self.miners.iter_mut() {
|
|
// a user action: a faulted card tries again
|
|
m.watch.event(0.0, crate::watchdog::Event::Reset);
|
|
m.restart_at = Some(Instant::now());
|
|
}
|
|
}
|
|
Cmd::RestartNode(why) => {
|
|
self.verifier = None;
|
|
if self.node_external {
|
|
self.shared.event("info", &format!("{why}; the node is external, so the app cannot restart it"));
|
|
} else if self.node.is_some() || self.node_restart_at.is_some() {
|
|
self.shared.event("info", &format!("{why}; the node restarts"));
|
|
self.restart_node(&why, Duration::from_secs(2));
|
|
}
|
|
}
|
|
Cmd::CheckUpdate => {
|
|
if self.no_ota {
|
|
self.shared.event("info", "updates are off for this engine (a measurement run: IGNEUM_APP_NO_OTA or --sweep)");
|
|
} else {
|
|
self.ota.check_now(&self.shared);
|
|
}
|
|
}
|
|
Cmd::InstallUpdate => self.ota.install_now(&self.shared),
|
|
Cmd::AutoUpdate(on) => self.ota.set_auto(&self.shared, on),
|
|
Cmd::OpenUpdateFile => {
|
|
if let Err(e) = self.ota.open_file() {
|
|
self.shared.event("error", &format!("could not open the update file: {e}"));
|
|
}
|
|
}
|
|
Cmd::Ota(ev) => self.ota.event(&self.shared, ev),
|
|
Cmd::Job(ev) => {
|
|
if let Some(a) = self.jobs.event(&self.shared, ev) {
|
|
self.job_action(a);
|
|
}
|
|
}
|
|
Cmd::JobsAllow(on) => self.jobs.set_allowed(&self.shared, on),
|
|
Cmd::JobsCheck => {
|
|
if let Some(a) = self.jobs.check_now(&self.shared) {
|
|
self.job_action(a);
|
|
}
|
|
}
|
|
Cmd::WorkerBuilt(card, result) => {
|
|
if let Some(m) = self.miners.iter_mut().find(|m| m.card == card) {
|
|
m.building = false;
|
|
match result {
|
|
Ok(p) => {
|
|
if p != m.worker {
|
|
self.shared.event("build", "GPU worker built from source");
|
|
}
|
|
m.worker = p;
|
|
m.needs_rebuild = false;
|
|
m.prepared = true;
|
|
m.restart_at = Some(Instant::now());
|
|
}
|
|
Err(e) => {
|
|
self.shared.event("error", &format!("worker build failed: {e}"));
|
|
m.restart_at = Some(Instant::now() + Duration::from_secs(300));
|
|
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
|
c.state = "failed".into();
|
|
c.message = e;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Cmd::ClockSample(d) => {
|
|
self.clock_samples.push(d);
|
|
if self.clock_samples.len() > 9 {
|
|
self.clock_samples.remove(0);
|
|
}
|
|
self.resolve_clock();
|
|
}
|
|
Cmd::ClockHttps(r) => {
|
|
if let Some(d) = r {
|
|
self.clock_https = Some((d, Instant::now()));
|
|
self.shared.log(&format!("clock check (https): local time is {:+.1} s against dl.igneum.network", d));
|
|
} else {
|
|
self.shared.log("clock check (https): no answer from dl.igneum.network");
|
|
}
|
|
self.resolve_clock();
|
|
}
|
|
Cmd::ClockCheck => {
|
|
self.clock_next_https = Instant::now() + Duration::from_secs(600);
|
|
if let Some(f) = std::env::var("IGNEUM_APP_FAKE_SKEW").ok().and_then(|v| v.parse::<f64>().ok()) {
|
|
self.command(Cmd::ClockHttps(Some(f)));
|
|
return;
|
|
}
|
|
let shared = self.shared.clone();
|
|
std::thread::spawn(move || {
|
|
let r = crate::update::https_time("https://dl.igneum.network/").map(|t| crate::platform::unix_now_f() - t);
|
|
shared.send(Cmd::ClockHttps(r));
|
|
});
|
|
}
|
|
Cmd::ClockSync => {
|
|
if self.st().clock.syncing {
|
|
return;
|
|
}
|
|
self.st().clock.syncing = true;
|
|
self.shared.event("info", "asking the system to set its clock from a time server");
|
|
let shared = self.shared.clone();
|
|
std::thread::spawn(move || {
|
|
let r = crate::platform::sync_clock();
|
|
shared.send(Cmd::ClockSynced(r));
|
|
});
|
|
}
|
|
Cmd::ClockSynced(r) => {
|
|
let mut st = self.st();
|
|
st.clock.syncing = false;
|
|
match &r {
|
|
Ok(t) => st.clock.sync_result = t.clone(),
|
|
Err(e) => st.clock.sync_result = format!("could not set the clock: {e}"),
|
|
}
|
|
drop(st);
|
|
match r {
|
|
Ok(t) => self.shared.event("ok", &format!("clock: {t}; checking again")),
|
|
Err(e) => self.shared.event("error", &format!("clock: {e}")),
|
|
}
|
|
// the sources answer again: forget the old samples and re-check
|
|
self.clock_samples.clear();
|
|
self.clock_node_at = None;
|
|
self.clock_https = None;
|
|
self.resolve_clock();
|
|
self.clock_next_https = Instant::now() + Duration::from_secs(6);
|
|
}
|
|
Cmd::PowerApplied(what, r, readback) => {
|
|
self.power_task = None; // the one elevated step may have registered the task: ask again next time
|
|
self.power_busy = false;
|
|
self.power_via_host = None;
|
|
// the truth is what nvidia-smi reads back, not whether the prompt said yes
|
|
let mut applied = Vec::new();
|
|
let mut missing = Vec::new();
|
|
{
|
|
let mut st = self.st();
|
|
for c in st.mining.cards.iter_mut().filter(|c| c.vendor == "nvidia" && c.enabled && c.present() && c.power_default_w > 0.0) {
|
|
let want = requested_watts(c);
|
|
let got = readback.get(&c.device).copied().unwrap_or(c.power_limit_w);
|
|
if got > 0.0 {
|
|
c.power_limit_w = got;
|
|
}
|
|
if got > 0.0 && (got - want).abs() < 1.0 {
|
|
c.power_applied = true;
|
|
c.power_note = format!("power cap: {} W applied", want as u64);
|
|
applied.push(format!("{} {} W", c.name, want as u64));
|
|
} else {
|
|
c.power_applied = false;
|
|
c.power_note = format!("power cap NOT applied (needs the administrator prompt): card reports {} W, wanted {} W", got as u64, want as u64);
|
|
missing.push(format!("{} reports {} W, wanted {} W", c.name, got as u64, want as u64));
|
|
}
|
|
}
|
|
}
|
|
if !applied.is_empty() {
|
|
self.shared.event("ok", &format!("GPU power cap in force: {}", applied.join(", ")));
|
|
}
|
|
if !missing.is_empty() {
|
|
let why = match &r {
|
|
Ok(()) => "the step ran but the card did not take it".to_string(),
|
|
Err(e) => e.clone(),
|
|
};
|
|
self.shared.event("error", &format!("GPU power cap NOT applied ({why}): {}. Retry from the card tile.", missing.join("; ")));
|
|
}
|
|
if applied.is_empty() && missing.is_empty() {
|
|
self.shared.log(&format!("power cap: nothing to read back for {what}"));
|
|
}
|
|
match power_control_after_prompt(&r) {
|
|
Some(note) => self.power_control_off(note),
|
|
None if !applied.is_empty() => self.st().settings.power_note = "administrator rights given; the cap and the sweep run".into(),
|
|
None => {}
|
|
}
|
|
}
|
|
Cmd::ElevatedDone(r) => {
|
|
if let Some((line, what, want, _)) = self.power_via_host.take() {
|
|
self.shared.log(&format!("window host ran the elevated step ({}): {line}", match &r { Ok(()) => "ok".to_string(), Err(e) => e.clone() }));
|
|
self.finish_power(what, r, want);
|
|
}
|
|
}
|
|
Cmd::ApplyPower => {
|
|
if self.sweep.is_some() || self.sweep_pending.is_some() {
|
|
self.shared.event("info", "a sweep is setting the caps right now; retry once it is done");
|
|
return;
|
|
}
|
|
for c in self.st().mining.cards.iter_mut().filter(|c| c.vendor == "nvidia") {
|
|
c.power_applied = false; // force the step again
|
|
}
|
|
self.apply_power_limits("retry");
|
|
}
|
|
Cmd::SweepStart(key) => {
|
|
let (name, ok, why) = {
|
|
let st = self.st();
|
|
match st.mining.cards.iter().find(|c| c.key == key) {
|
|
Some(c) if !matches!(c.vendor.as_str(), "nvidia" | "amd" | "apple") => (c.name.clone(), false, "no power or clock control for this card".to_string()),
|
|
Some(c) if !c.enabled => (c.name.clone(), false, "the card is switched off".to_string()),
|
|
Some(c) => (c.name.clone(), true, String::new()),
|
|
None => (key.clone(), false, "no such card".to_string()),
|
|
}
|
|
};
|
|
if !ok {
|
|
self.shared.event("error", &format!("{name}: no tune: {why}"));
|
|
} else if !self.sweep_queue.contains(&key) {
|
|
self.sweep_queue.push(key);
|
|
self.sweep_next_check = Instant::now();
|
|
self.shared.event("info", &format!("{name}: tune queued; it starts once the card has mined for {} s with over {} s to the hour boundary", crate::ember::STABLE_S, crate::ember::NEEDS_S));
|
|
}
|
|
}
|
|
Cmd::SweepStop => {
|
|
self.sweep_queue.clear();
|
|
if self.sweep.is_some() || self.sweep_pending.is_some() {
|
|
self.sweep_abort("stopped from the dashboard");
|
|
}
|
|
}
|
|
Cmd::SweepPin(key, pinned) => {
|
|
let name = {
|
|
let mut settings = self.shared.settings.lock().unwrap();
|
|
let mut st = self.st();
|
|
let Some(c) = st.mining.cards.iter_mut().find(|c| c.key == key) else { return };
|
|
c.pinned = pinned;
|
|
let e = settings.cards.entry(key.clone()).or_insert_with(|| crate::config::CardPref { enabled: c.enabled, identities: c.identities, ..Default::default() });
|
|
e.pinned = pinned;
|
|
if pinned {
|
|
e.power_pct = c.power_pct;
|
|
}
|
|
c.name.clone()
|
|
};
|
|
self.shared.save_settings();
|
|
self.shared.event("info", &if pinned { format!("{name}: cap pinned; the sweep records but does not change it") } else { format!("{name}: the sweep chooses the cap again from its next run") });
|
|
}
|
|
Cmd::SweepEnable(on) => {
|
|
let allowed = self.elevation_allowed();
|
|
self.shared.settings.lock().unwrap().sweep = on;
|
|
self.shared.save_settings();
|
|
self.st().settings.sweep = on;
|
|
self.shared.event("info", if on && allowed { "Ember Tune on: every card is tuned for MH per watt once after install, then weekly" } else if on { "Ember Tune on: AMD cards are tuned; NVIDIA cards measure only until Power control is on in Settings" } else { "Ember Tune off; Tune now on a card still runs one" });
|
|
}
|
|
Cmd::PowerControl(on) => {
|
|
{
|
|
let mut s = self.shared.settings.lock().unwrap();
|
|
s.power_control = on;
|
|
s.sweep = on; // implied
|
|
}
|
|
self.shared.save_settings();
|
|
{
|
|
let mut st = self.st();
|
|
st.settings.power_control = on;
|
|
st.settings.sweep = on;
|
|
st.settings.power_note = if on { "asking Windows for administrator rights once".into() } else { String::new() };
|
|
}
|
|
if on {
|
|
self.shared.event("info", "power control on: Windows asks for administrator rights once; the cap and the sweep need them");
|
|
for c in self.st().mining.cards.iter_mut().filter(|c| c.vendor == "nvidia") {
|
|
c.power_applied = false; // the one prompt sets every cap now
|
|
}
|
|
self.apply_power_limits("power control on");
|
|
if !self.power_busy {
|
|
self.st().settings.power_note = "on; no NVIDIA card needs a cap right now".into();
|
|
}
|
|
} else {
|
|
self.power_control_off("power control off; the cap and the sweep do not run and nothing asks for administrator rights");
|
|
}
|
|
}
|
|
Cmd::TuneProbe(idx, r) => self.sweep_probe_known(idx, r),
|
|
Cmd::SweepHelperDone(r) => {
|
|
self.sweep_helper = false;
|
|
if let Err(e) = &r {
|
|
self.shared.log(&format!("sweep helper: {e}"));
|
|
if self.sweep.is_some() || self.sweep_pending.is_some() {
|
|
self.sweep_abort(&format!("the elevated helper did not run ({e})"));
|
|
}
|
|
} else if self.sweep.is_some() {
|
|
self.sweep_abort("the elevated helper exited early");
|
|
} else {
|
|
self.shared.log("sweep helper: exited");
|
|
}
|
|
}
|
|
Cmd::TuneProgress(v) => self.tune_progress(&v),
|
|
Cmd::TuneGoal(goal, price, climb) => {
|
|
{
|
|
let mut s = self.shared.settings.lock().unwrap();
|
|
if let Some(g) = &goal {
|
|
s.tune_goal = g.clone();
|
|
}
|
|
if let Some(p) = price {
|
|
s.power_price_pence = p;
|
|
}
|
|
if let Some(c) = climb {
|
|
s.tune_climb = c;
|
|
}
|
|
}
|
|
self.shared.save_settings();
|
|
let s = self.shared.settings.lock().unwrap().clone();
|
|
let mut st = self.st();
|
|
st.settings.tune_goal = s.tune_goal.clone();
|
|
st.settings.power_price_pence = s.power_price_pence;
|
|
st.settings.tune_climb = s.tune_climb;
|
|
drop(st);
|
|
self.shared.event("info", &format!("tune goal: {}{}{}", s.tune_goal, if s.power_price_pence > 0.0 { format!(", electricity {:.1} p/kWh", s.power_price_pence) } else { String::new() }, if s.tune_climb { ", hill-climb on" } else { "" }));
|
|
}
|
|
Cmd::TuneSet(seq, r) => match r {
|
|
Ok(text) => {
|
|
self.tune_acked = Some(seq);
|
|
if !text.contains("All done") && text != "helper" {
|
|
self.shared.log(&format!("tune: request {seq} answered: {}", short(&text.replace('\n', " "), 200)));
|
|
}
|
|
}
|
|
Err(text) => {
|
|
self.shared.log(&format!("tune: request {seq} refused: {}", short(&text.replace('\n', " "), 300)));
|
|
if self.sweep.as_ref().map(|r| r.seq == seq).unwrap_or(false) {
|
|
self.sweep_abort("the card refused the setting (see the log)");
|
|
}
|
|
}
|
|
},
|
|
Cmd::Quit(source) => {
|
|
self.shared.log(&format!("quit requested by {source}"));
|
|
self.quit_source = source;
|
|
self.quitting = true;
|
|
self.st().quitting = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---- start, node ------------------------------------------------------------------------------------------
|
|
|
|
/// The first enumeration: the list as detected, the saved choices applied, the setup screen's notes.
|
|
fn first_detection(&mut self, d: crate::detect::Detection) {
|
|
self.detected = true;
|
|
let crate::detect::Detection { cards, notes, dropped, .. } = d;
|
|
let names: Vec<String> = cards.iter().map(|c| format!("{} ({}{}{})", c.name, c.worker, if c.code != c.name { format!(", {}", c.code) } else { String::new() }, if c.problem.is_empty() { String::new() } else { format!(", {}", c.problem) })).collect();
|
|
self.shared.log(&format!("GPUs: {}", if names.is_empty() { "none usable".to_string() } else { names.join("; ") }));
|
|
for n in ¬es {
|
|
self.shared.log(&format!("detection: {n}"));
|
|
}
|
|
for n in &dropped {
|
|
self.shared.log(&format!("detection: duplicate OpenCL platform entry {n}"));
|
|
}
|
|
let prefs = self.shared.settings.lock().unwrap().cards.clone();
|
|
let mut st = self.st();
|
|
st.detecting = false;
|
|
st.detect_message = notes.join(". ");
|
|
st.mining.cards = cards;
|
|
for c in st.mining.cards.iter_mut() {
|
|
if let Some(p) = crate::hotplug::pref_for(&prefs, c) {
|
|
crate::hotplug::apply_pref(c, p);
|
|
}
|
|
}
|
|
let supported: Vec<String> = st.mining.cards.iter().filter(|c| c.sweep_supported && c.enabled).map(|c| c.key.clone()).collect();
|
|
let unsupported: Vec<String> = st.mining.cards.iter().filter(|c| !c.sweep_supported).map(|c| format!("{}: {}", c.name, c.sweep_note)).collect();
|
|
let line = crate::hotplug::cards_line(&st.mining.cards);
|
|
drop(st);
|
|
self.shared.log(&line);
|
|
self.last_cards_line = Instant::now();
|
|
for u in &unsupported {
|
|
self.shared.log(&format!("efficiency sweep {u}"));
|
|
}
|
|
if self.shared.runtime.sweep_only {
|
|
if supported.is_empty() {
|
|
self.sweep_say("SWEEP none reason=no_supported_card");
|
|
self.shared.send(Cmd::Quit("the --sweep run (no supported card)"));
|
|
} else {
|
|
self.sweep_queue = supported;
|
|
}
|
|
}
|
|
if self.running {
|
|
self.plan_miners();
|
|
}
|
|
}
|
|
|
|
/// A later enumeration (src/hotplug.rs): new cards start, lost cards stop, faulty cards are marked; a list
|
|
/// that changed nothing touches nothing (no running worker ever restarts because of a re-detection).
|
|
fn merge_detection(&mut self, d: crate::detect::Detection) {
|
|
let now = crate::platform::unix_now_f();
|
|
let old = self.st().mining.cards.clone();
|
|
let diff = crate::hotplug::diff(&old, &d.cards, &|c| d.listed(c));
|
|
if diff.is_quiet() {
|
|
return;
|
|
}
|
|
for n in &d.dropped {
|
|
self.shared.log(&format!("detection: duplicate OpenCL platform entry {n}"));
|
|
}
|
|
let prefs = self.shared.settings.lock().unwrap().cards.clone();
|
|
let mut new_nvidia = false;
|
|
for i in diff.removed.iter().copied() {
|
|
let name = old[i].name.clone();
|
|
self.drop_worker(i, "the card was removed");
|
|
if let Some(c) = self.st().mining.cards.get_mut(i) {
|
|
crate::hotplug::mark_removed(c, now);
|
|
}
|
|
self.shared.event("warn", &format!("Card removed: {name}; its worker stopped"));
|
|
}
|
|
for (i, problem) in diff.errored.iter() {
|
|
let name = old[*i].name.clone();
|
|
self.drop_worker(*i, &format!("the card reports {problem}"));
|
|
if let Some(c) = self.st().mining.cards.get_mut(*i) {
|
|
crate::detect::mark_unusable(c, problem);
|
|
c.added_at = now;
|
|
}
|
|
self.shared.event("warn", &format!("{name}: not usable ({problem}); its worker stopped. {}", crate::detect::PROBLEM_HINT));
|
|
}
|
|
for (i, fresh) in diff.moved.iter() {
|
|
if let Some(c) = self.st().mining.cards.get_mut(*i) {
|
|
self.shared.log(&format!("{}: device {} is now {} (the running worker keeps its device; the next start uses the new one)", c.name, c.device, fresh.device));
|
|
c.device = fresh.device.clone();
|
|
c.key = fresh.key.clone();
|
|
c.bus = fresh.bus.clone();
|
|
c.name = fresh.name.clone();
|
|
c.platform = fresh.platform.clone();
|
|
if fresh.vram_mb > 0 {
|
|
c.vram_mb = fresh.vram_mb;
|
|
}
|
|
if !fresh.detail.is_empty() {
|
|
c.detail = fresh.detail.clone();
|
|
}
|
|
}
|
|
}
|
|
let mut placed: Vec<(usize, CardState)> = Vec::new();
|
|
for (i, fresh) in diff.recovered.into_iter().chain(diff.revived.into_iter()) {
|
|
placed.push((i, fresh));
|
|
}
|
|
let mut next = self.st().mining.cards.len();
|
|
for fresh in diff.added.into_iter() {
|
|
placed.push((next, fresh));
|
|
next += 1;
|
|
}
|
|
for (i, mut fresh) in placed {
|
|
let pref = crate::hotplug::pref_for(&prefs, &fresh).cloned();
|
|
crate::hotplug::settle_new(&mut fresh, i, pref.as_ref(), now);
|
|
let (kind, text) = crate::hotplug::added_words(&fresh);
|
|
new_nvidia |= fresh.vendor == "nvidia" && fresh.enabled;
|
|
let mut st = self.st();
|
|
if i < st.mining.cards.len() {
|
|
st.mining.cards[i] = fresh;
|
|
} else {
|
|
st.mining.cards.push(fresh);
|
|
}
|
|
drop(st);
|
|
self.shared.event(kind, &text);
|
|
}
|
|
let line = crate::hotplug::cards_line(&self.st().mining.cards);
|
|
self.shared.log(&line);
|
|
self.last_cards_line = Instant::now();
|
|
if self.running {
|
|
self.plan_miners();
|
|
if new_nvidia {
|
|
self.apply_power_limits("new card");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Stops the worker on card `idx` (quit, then up to 8 s) and drops its slot; the sweep on it, if any, is aborted.
|
|
fn drop_worker(&mut self, idx: usize, why: &str) {
|
|
if self.sweep.as_ref().map(|r| r.card == idx).unwrap_or(false) {
|
|
self.sweep_abort(why);
|
|
}
|
|
let Some(pos) = self.miners.iter().position(|m| m.card == idx) else { return };
|
|
let label = self.miners[pos].label.clone();
|
|
if let Some(mut p) = self.miners[pos].proc.take() {
|
|
self.shared.log(&format!("stopping miner {label} ({why})"));
|
|
p.write_stdin("quit\n");
|
|
p.stop(8);
|
|
}
|
|
self.miners.remove(pos);
|
|
self.stability.remove(&idx);
|
|
}
|
|
|
|
fn start(&mut self) {
|
|
if self.running {
|
|
return;
|
|
}
|
|
self.running = true;
|
|
{
|
|
let mut s = self.shared.settings.lock().unwrap();
|
|
s.setup_done = true;
|
|
s.save(&self.shared.settings_path);
|
|
}
|
|
{
|
|
let mut st = self.st();
|
|
st.setup_done = true;
|
|
st.phase = "dashboard".into();
|
|
st.mining.state = if st.mining.paused { "paused".into() } else { "waiting".into() };
|
|
}
|
|
if self.keep_awake.is_none() {
|
|
self.keep_awake = Some(crate::platform::KeepAwake::start());
|
|
self.shared.log("keeping the machine awake while the app runs");
|
|
}
|
|
let _ = std::fs::create_dir_all(&self.shared.runtime.node_dir);
|
|
let _ = std::fs::create_dir_all(&self.shared.runtime.log_dir);
|
|
self.apply_power_limits("start");
|
|
if port_open(self.shared.runtime.rpc_port) {
|
|
self.node_external = true;
|
|
self.shared.event("info", &format!("a node already answers on 127.0.0.1:{}; using it (it is not stopped by this app)", self.shared.runtime.rpc_port));
|
|
let mut st = self.st();
|
|
st.node.state = "syncing".into();
|
|
st.node.message = "external node".into();
|
|
st.proving.verifier_reason = "external node: the app did not start it, so it set no verifier".into();
|
|
st.proving.verifier_note = crate::verifier::note("unknown", "", "", true);
|
|
} else {
|
|
self.start_node();
|
|
}
|
|
self.plan_miners();
|
|
}
|
|
|
|
fn node_args(&self) -> Vec<String> {
|
|
let r = &self.shared.runtime;
|
|
let mut a = vec![
|
|
format!("--{}", r.network),
|
|
format!("--appdir={}", r.node_dir.display()),
|
|
format!("--rpclisten=127.0.0.1:{}", r.rpc_port),
|
|
format!("--evm-rpclisten=127.0.0.1:{}", r.evm_port()),
|
|
format!("--listen=0.0.0.0:{}", r.p2p_port),
|
|
];
|
|
if let Some(n) = r.devnet_suffix {
|
|
a.push(format!("--devnet-suffix={n}"));
|
|
}
|
|
for p in &r.peers {
|
|
a.push(format!("--addpeer={p}"));
|
|
}
|
|
if let Some(path) = self.node_override_file() {
|
|
a.push(format!("--override-params-file={}", path.display()));
|
|
}
|
|
a.extend(["--nodnsseed", "--disable-upnp", "--nologfiles", "--yes"].iter().map(|s| s.to_string()));
|
|
if r.unsynced_mining {
|
|
a.push("--enable-unsynced-mining".into());
|
|
}
|
|
a
|
|
}
|
|
|
|
/// The packager's consensus parameters (igneum-app.json `node_override_params`, for example the difficulty v2
|
|
/// activation height) written to <app data>/override-params.json for --override-params-file. None when the
|
|
/// package pins nothing, so the node runs on the network's defaults as before.
|
|
fn node_override_file(&self) -> Option<std::path::PathBuf> {
|
|
if self.node_override_unusable {
|
|
return None;
|
|
}
|
|
// the signed OTA manifest's consensus override wins over the packager's pin (src/ota.rs write_override)
|
|
if let Some(p) = self.ota.override_path() {
|
|
return Some(p);
|
|
}
|
|
let v = self.shared.packaged.node_override_params.as_ref()?;
|
|
if v.as_object().map(|o| o.is_empty()).unwrap_or(true) {
|
|
return None;
|
|
}
|
|
let path = self.shared.runtime.app_dir.join("override-params.json");
|
|
let text = serde_json::to_string_pretty(v).ok()?;
|
|
if std::fs::read_to_string(&path).ok().as_deref() != Some(text.as_str()) {
|
|
if let Some(d) = path.parent() {
|
|
let _ = std::fs::create_dir_all(d);
|
|
}
|
|
if let Err(e) = std::fs::write(&path, &text) {
|
|
self.shared.log(&format!("could not write {}: {e}; the node starts without the override file", path.display()));
|
|
return None;
|
|
}
|
|
self.shared.log(&format!("node override params written to {}: {}", path.display(), text.replace('\n', " ")));
|
|
}
|
|
Some(path)
|
|
}
|
|
|
|
fn start_node(&mut self) {
|
|
self.node_starts += 1;
|
|
let seg = if self.node_starts > 1 { format!("-r{}", self.node_starts) } else { String::new() };
|
|
let log = self.shared.runtime.log_dir.join(format!("node-{}{seg}.log", self.stamp));
|
|
let args = self.node_args();
|
|
let verifier = self.node_verifier();
|
|
// the switches the Node page names: from the override file this start uses (none = the network's defaults)
|
|
let switches = self.node_override_file().and_then(|p| std::fs::read_to_string(p).ok()).and_then(|t| serde_json::from_str::<Value>(&t).ok()).map(|v| switches_of(&v)).unwrap_or_default();
|
|
{
|
|
let mut st = self.st();
|
|
st.node.consensus_switches = switches;
|
|
st.node.consensus_digest = String::new();
|
|
}
|
|
match procs::spawn(Source::Node, &self.bins.node, &args, None, &log, &self.lines_tx, &verifier.env) {
|
|
Ok(p) => {
|
|
self.shared.log(&format!("igneumd started (pid {}): {}", p.pid(), p.cmdline));
|
|
self.shared.log(&format!("node proof verifier: {} ({})", verifier.mode, verifier.detail));
|
|
let mut st = self.st();
|
|
st.node.pid = p.pid();
|
|
st.node.state = "starting".into();
|
|
st.node.starts = self.node_starts;
|
|
st.node.message = "opening the database".into();
|
|
st.node.restart_in_s = 0;
|
|
drop(st);
|
|
self.node = Some(p);
|
|
self.node_log = Some(log);
|
|
self.node_started_at = Instant::now();
|
|
self.node_restart_at = None;
|
|
self.sync_prev = None;
|
|
self.sync_stable_since = None;
|
|
self.node_last_reading = None;
|
|
self.shared.event(if self.node_starts == 1 { "ok" } else { "info" }, if self.node_starts == 1 { "node started" } else { "node restarted" });
|
|
}
|
|
Err(e) => {
|
|
self.shared.event("error", &format!("igneumd could not start: {e}"));
|
|
let mut st = self.st();
|
|
st.node.state = "failed".into();
|
|
st.node.message = e.to_string();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The proof verifier for this node start (spec 7.7 item 4; src/verifier.rs), decided once and kept across
|
|
/// restarts until a RestartNode command asks again. The Windows probe runs WSL, so the result is cached.
|
|
fn node_verifier(&mut self) -> crate::verifier::Verifier {
|
|
if self.verifier.is_none() {
|
|
let trust = self.shared.settings.lock().unwrap().proof_verify_trust;
|
|
let v = crate::verifier::resolve(&self.bins.dir, trust);
|
|
if v.mode == "trust" {
|
|
self.shared.event("info", "devnet only: the node trusts proof records without verifying them (Settings)");
|
|
}
|
|
self.verifier = Some(v);
|
|
}
|
|
let v = self.verifier.clone().unwrap();
|
|
let mut st = self.st();
|
|
st.proving.verifier_set = v.set_text();
|
|
st.proving.verifier_reason = if v.mode == "command" { String::new() } else { v.detail.clone() };
|
|
let (mode, set, reason) = (st.proving.verifier_mode.clone(), st.proving.verifier_set.clone(), st.proving.verifier_reason.clone());
|
|
st.proving.verifier_note = crate::verifier::note(&mode, &set, &reason, false);
|
|
v
|
|
}
|
|
|
|
fn stop_node(&mut self) {
|
|
if let Some(mut n) = self.node.take() {
|
|
self.shared.log("stopping the node");
|
|
let code = n.stop(30);
|
|
self.shared.log(&format!("node stopped (exit {code:?})"));
|
|
}
|
|
self.st().node.state = "stopped".into();
|
|
}
|
|
|
|
fn start_watch(&mut self) {
|
|
let args = vec!["watch".to_string(), "1000000000".to_string(), self.shared.runtime.rpc_url()];
|
|
let log = self.shared.runtime.log_dir.join(format!("watch-{}.log", self.stamp));
|
|
if let Ok(p) = procs::spawn(Source::Watch, &self.bins.miner, &args, None, &log, &self.lines_tx, &[]) {
|
|
self.watch = Some(p);
|
|
}
|
|
self.watch_retry_at = Instant::now() + Duration::from_secs(4);
|
|
}
|
|
|
|
// ---- miners ------------------------------------------------------------------------------------------------
|
|
|
|
/// One slot per enabled card. Called once the cards are known; re-planned after a settings change.
|
|
fn plan_miners(&mut self) {
|
|
let cards = self.st().mining.cards.clone();
|
|
let id8 = self.shared.runtime.id8();
|
|
for c in cards.iter() {
|
|
if !c.enabled || !c.present() || self.miners.iter().any(|m| m.card == c.index) {
|
|
continue;
|
|
}
|
|
let prefix = match c.vendor.as_str() {
|
|
"apple" => "mac".to_string(),
|
|
v => v.to_string(),
|
|
};
|
|
// <platform>-<machine id8>-<card n>: the miner derives the vote keys from this label (and -1..N per identity)
|
|
let label = format!("{prefix}-{id8}-{}", c.index + 1);
|
|
let worker = match c.worker.as_str() {
|
|
"Metal" => self.bins.metal.clone(),
|
|
"CUDA" => self.bins.cuda.clone(),
|
|
_ => self.bins.opencl.clone(),
|
|
};
|
|
let needs_rebuild = worker.is_none();
|
|
self.miners.push(MinerSlot {
|
|
card: c.index,
|
|
label,
|
|
proc: None,
|
|
worker: worker.unwrap_or_default(),
|
|
restart_at: Some(Instant::now()),
|
|
starts: 0,
|
|
restarts: 0,
|
|
building: false,
|
|
needs_rebuild,
|
|
prepared: false,
|
|
last_status: None,
|
|
error_at: None,
|
|
watch: crate::watchdog::CardWatch::new(),
|
|
pack_rebuilds: crate::watchdog::PackRebuilds::new(),
|
|
});
|
|
}
|
|
if self.miners.is_empty() {
|
|
let mut st = self.st();
|
|
if !cards.is_empty() || self.detected {
|
|
st.mining.state = "idle".into();
|
|
}
|
|
}
|
|
}
|
|
|
|
fn miner_args(&self, slot: &MinerSlot, card: &CardState) -> Vec<String> {
|
|
let s = self.shared.settings.lock().unwrap();
|
|
let r = &self.shared.runtime;
|
|
let mut a = vec![
|
|
"mine".to_string(),
|
|
r.rpc_url(),
|
|
"1".into(),
|
|
"100000000".into(),
|
|
slot.label.clone(),
|
|
"--worker".into(),
|
|
slot.worker.display().to_string(),
|
|
"--status-secs".into(),
|
|
r.status_secs.to_string(),
|
|
"--exit-on-seed-change".into(),
|
|
"--evm-address".into(),
|
|
s.address.clone(),
|
|
];
|
|
if card.identities > 1 {
|
|
a.push("--identities".into());
|
|
a.push(card.identities.to_string());
|
|
}
|
|
if !s.vote {
|
|
a.push("--no-vote".into());
|
|
}
|
|
if !s.dev_fee {
|
|
// the miner software's dev fee (default 1% of templates); the switch in Settings
|
|
a.push("--dev-fee".into());
|
|
a.push("0".into());
|
|
}
|
|
a.push("--payout-label".into());
|
|
a.push(slot.label.clone());
|
|
if r.network != "devnet" {
|
|
a.push("--network".into());
|
|
a.push(r.network.clone());
|
|
}
|
|
// The miner runs with the app data folder as its cwd: the pack paths stay relative (the miner splits
|
|
// --worker-args on spaces, and %LOCALAPPDATA% may carry a space in the user name). Every worker gets
|
|
// --prepare-packs (5 October 2026, program class v3: the Metal worker too compiles v3 only from a prepared
|
|
// pack, `prepare <e> <d> <dir> class=v3 era=<hex>`; before this the flag went to the CUDA and OpenCL
|
|
// workers only and a Mac would answer `need` lines at the first v3 epoch and stop mining, the 18:23Z class).
|
|
a.push("--prepare-packs".into());
|
|
a.push(prepare_packs_arg());
|
|
if card.worker != "Metal" {
|
|
// The prebuilt workers take the exported pack with --pack and build the next program from
|
|
// --prepare-packs; a worker built from source has the program compiled in and exits 42 at the boundary.
|
|
if card.worker == "OpenCL" {
|
|
a.push("--job-nonces".into());
|
|
a.push("2097152".into());
|
|
}
|
|
let mut wargs = String::new();
|
|
if !card.device.is_empty() {
|
|
wargs.push_str(&format!("--device {}", card.device));
|
|
}
|
|
if slot.worker.file_name().map(|f| f.to_string_lossy().starts_with("igneum-worker-")).unwrap_or(false) {
|
|
if !wargs.is_empty() {
|
|
wargs.push(' ');
|
|
}
|
|
wargs.push_str("--pack packs\\devnet");
|
|
}
|
|
if !wargs.is_empty() {
|
|
a.push("--worker-args".into());
|
|
a.push(wargs);
|
|
}
|
|
}
|
|
a
|
|
}
|
|
|
|
fn start_miner(&mut self, i: usize) {
|
|
let card_idx = self.miners[i].card;
|
|
let Some(card) = self.st().mining.cards.get(card_idx).cloned() else { return };
|
|
if self.shared.settings.lock().unwrap().address.is_empty() {
|
|
self.shared.event("error", "no payout address; open settings and set one");
|
|
self.miners[i].restart_at = Some(Instant::now() + Duration::from_secs(60));
|
|
return;
|
|
}
|
|
let args = self.miner_args(&self.miners[i], &card);
|
|
self.miners[i].starts += 1;
|
|
let seg = if self.miners[i].starts > 1 { format!("-r{}", self.miners[i].starts) } else { String::new() };
|
|
let log = self.shared.runtime.log_dir.join(format!("miner-{}-{}{seg}.log", self.miners[i].label, self.stamp));
|
|
// every worker runs with the app data folder as its cwd, so the relative pack paths resolve (the Metal worker
|
|
// too, since it takes --prepare-packs now)
|
|
let cwd = Some(self.shared.runtime.app_dir.clone());
|
|
self.miners[i].prepared = false;
|
|
// the fleet's per-card kernel tuning (from the signed manifest) reaches the GPU worker through the miner's environment
|
|
let envs: Vec<(String, String)> = self.ota.tuning_path().map(|p| vec![("IGNEUM_TUNING_FILE".to_string(), p.display().to_string())]).unwrap_or_default();
|
|
match procs::spawn(Source::Miner(card_idx), &self.bins.miner, &args, cwd.as_deref(), &log, &self.lines_tx, &envs) {
|
|
Ok(p) => {
|
|
self.shared.log(&format!("miner {} started (pid {}): {}", self.miners[i].label, p.pid(), p.cmdline));
|
|
let mut st = self.st();
|
|
if let Some(c) = st.mining.cards.get_mut(card_idx) {
|
|
c.pid = p.pid();
|
|
c.state = "starting".into();
|
|
c.ready = false;
|
|
c.prepared = false;
|
|
c.restart_in_s = 0;
|
|
c.message = "worker loading the program".into();
|
|
c.ids.clear();
|
|
}
|
|
st.mining.state = if st.mining.state == "paused" { st.mining.state.clone() } else { "mining".into() };
|
|
drop(st);
|
|
self.miners[i].proc = Some(p);
|
|
self.miners[i].restart_at = None;
|
|
self.miners[i].last_status = None;
|
|
let t = self.secs(Instant::now());
|
|
self.miners[i].watch.event(t, crate::watchdog::Event::Started);
|
|
}
|
|
Err(e) => {
|
|
self.shared.event("error", &format!("the miner could not start: {e}"));
|
|
self.miners[i].restart_at = Some(Instant::now() + Duration::from_secs(30));
|
|
}
|
|
}
|
|
}
|
|
|
|
fn stop_miners(&mut self, why: &str) {
|
|
let any = self.miners.iter().any(|m| m.proc.is_some());
|
|
if any {
|
|
self.shared.log(&format!("stopping the miners ({why})"));
|
|
}
|
|
let t = self.secs(Instant::now());
|
|
for m in self.miners.iter_mut() {
|
|
if let Some(mut p) = m.proc.take() {
|
|
p.write_stdin("quit\n");
|
|
p.stop(8);
|
|
}
|
|
m.restart_at = None;
|
|
m.watch.event(t, crate::watchdog::Event::Stopped);
|
|
}
|
|
let mut st = self.st();
|
|
let paused = st.mining.paused;
|
|
for c in st.mining.cards.iter_mut() {
|
|
if c.enabled && c.present() && c.state != "faulted" {
|
|
c.state = if paused { "off".into() } else { "waiting".into() };
|
|
c.hash_now = 0.0;
|
|
c.pid = 0;
|
|
}
|
|
}
|
|
st.mining.hash_total = 0.0;
|
|
st.mining.state = if st.mining.paused { "paused".into() } else if self.running { "waiting".into() } else { "stopped".into() };
|
|
}
|
|
|
|
/// Ahead-of-time workers (Windows): export this hour's program pack from the node for the prebuilt worker, or,
|
|
/// with no prebuilt worker, build one from source (today's launcher path; it then has no prepare support and is
|
|
/// rebuilt at every hour boundary). Runs off the supervisor thread; WorkerBuilt starts the miner.
|
|
fn prepare_worker(&mut self, i: usize) {
|
|
let card_idx = self.miners[i].card;
|
|
self.miners[i].building = true;
|
|
let build = self.miners[i].needs_rebuild;
|
|
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
|
c.state = "starting".into();
|
|
c.message = if build { "building the GPU worker from source (about a minute)".into() } else { "exporting this hour's program".into() };
|
|
}
|
|
let shared = self.shared.clone();
|
|
let bins = self.bins.clone();
|
|
let vendor = self.st().mining.cards.get(card_idx).map(|c| c.vendor.clone()).unwrap_or_default();
|
|
let worker = self.miners[i].worker.clone();
|
|
std::thread::spawn(move || {
|
|
let r = if build { build_worker_from_source(&shared, &bins, &vendor) } else { export_pack(&shared, &bins).map(|_| worker) };
|
|
shared.send(Cmd::WorkerBuilt(card_idx, r));
|
|
});
|
|
}
|
|
|
|
/// The user changed which cards mine or how many identities each runs: only the affected workers restart.
|
|
fn apply_cards(&mut self, choices: Vec<CardChoice>) {
|
|
let mut changed = Vec::new();
|
|
let mut power_changed = false;
|
|
let mut pin_aborts = false;
|
|
{
|
|
let mut settings = self.shared.settings.lock().unwrap();
|
|
let mut st = self.st();
|
|
for ch in &choices {
|
|
let Some(c) = st.mining.cards.iter_mut().find(|c| c.key == ch.key) else { continue };
|
|
if !c.present() {
|
|
continue; // a removed or faulty card has no switch; its saved choice waits for it
|
|
}
|
|
let identities = ch.identities.clamp(1, 64);
|
|
let enabled = ch.enabled && c.kind != "unknown";
|
|
let power_pct = ch.power_pct.map(|p| p.clamp(crate::sweep::MIN_PCT, 100)).unwrap_or(c.power_pct);
|
|
if c.vendor == "nvidia" && power_pct != c.power_pct {
|
|
c.power_pct = power_pct;
|
|
c.power_applied = false;
|
|
power_changed = true;
|
|
// a cap set by hand is pinned: the sweep records MH per watt but leaves it (unpin on the tile)
|
|
c.pinned = true;
|
|
if self.sweep.as_ref().map(|r| r.card == c.index).unwrap_or(false) {
|
|
pin_aborts = true;
|
|
}
|
|
}
|
|
let prev = settings.cards.get(&ch.key).cloned().unwrap_or_default();
|
|
settings.cards.insert(ch.key.clone(), crate::config::CardPref { enabled, identities, power_pct: if c.vendor == "nvidia" { power_pct } else { 0 }, pinned: c.pinned, ..prev });
|
|
if c.enabled != enabled || c.identities != identities {
|
|
c.enabled = enabled;
|
|
c.identities = identities;
|
|
c.reason = if !enabled && c.kind == "integrated" { crate::detect::INTEGRATED_REASON.into() } else { String::new() };
|
|
if !enabled {
|
|
c.state = "off".into();
|
|
c.hash_now = 0.0;
|
|
c.message = String::new();
|
|
}
|
|
changed.push((c.index, c.name.clone(), enabled, identities));
|
|
}
|
|
}
|
|
settings.save(&self.shared.settings_path);
|
|
}
|
|
for (idx, name, enabled, identities) in changed {
|
|
if let Some(pos) = self.miners.iter().position(|m| m.card == idx) {
|
|
if let Some(mut p) = self.miners[pos].proc.take() {
|
|
p.write_stdin("quit\n");
|
|
p.stop(8);
|
|
}
|
|
if enabled {
|
|
self.miners[pos].restart_at = Some(Instant::now());
|
|
self.miners[pos].prepared = false;
|
|
self.miners[pos].watch.event(0.0, crate::watchdog::Event::Reset);
|
|
self.shared.event("info", &format!("{name}: {identities} identit{}; its worker restarts", if identities == 1 { "y" } else { "ies" }));
|
|
} else {
|
|
self.miners.remove(pos);
|
|
self.shared.event("info", &format!("{name}: switched off"));
|
|
}
|
|
} else if enabled {
|
|
self.shared.event("ok", &format!("{name}: switched on, {identities} identit{}", if identities == 1 { "y" } else { "ies" }));
|
|
}
|
|
}
|
|
if pin_aborts {
|
|
self.sweep_abort("the cap was set by hand");
|
|
}
|
|
if self.running {
|
|
self.plan_miners();
|
|
if power_changed {
|
|
self.apply_power_limits("settings");
|
|
}
|
|
}
|
|
}
|
|
|
|
/// NVIDIA cards: cap the power at power_pct of the default limit through nvidia-smi -pl, all cards in one
|
|
/// elevated step (one administrator prompt). The limit before the app touched it is restored on quit.
|
|
fn apply_power_limits(&mut self, why: &str) {
|
|
if self.power_busy {
|
|
return;
|
|
}
|
|
if self.shared.runtime.sweep_only {
|
|
self.shared.log(&format!("power cap ({why}): not touched under --sweep; the tune sets every limit itself"));
|
|
return;
|
|
}
|
|
if self.sweep.is_some() || self.sweep_pending.is_some() {
|
|
// the sweep owns the caps until it ends; it applies the chosen one itself
|
|
self.shared.log(&format!("power cap ({why}): deferred, a sweep is running"));
|
|
return;
|
|
}
|
|
let allowed = self.elevation_allowed();
|
|
let (cmds, what, held) = power_cap_plan(&mut self.st().mining.cards, allowed, &crate::platform::tool("nvidia-smi").display().to_string());
|
|
if held > 0 {
|
|
self.shared.log(&format!("power cap ({why}): not asked, Power control is off in Settings ({held} card(s) would need it)"));
|
|
}
|
|
if cmds.is_empty() {
|
|
return;
|
|
}
|
|
self.power_busy = true;
|
|
self.power_restore_pending = true;
|
|
self.shared.log(&format!("power cap ({why}): {}", cmds.join(" & ")));
|
|
let what = what.join(", ");
|
|
// once, ever (src/powertask.rs): a registered task sets the caps with no prompt; the readback judges it
|
|
if cfg!(windows) && self.power_task_registered() {
|
|
let pairs: Vec<(String, u64)> = self.st().mining.cards.iter().filter(|c| c.vendor == "nvidia" && c.enabled && c.present() && c.power_default_w > 0.0 && !c.power_applied).map(|c| (c.device.clone(), requested_watts(c).round() as u64)).collect();
|
|
let dir = crate::powertask::helper_dir();
|
|
let shared = self.shared.clone();
|
|
self.shared.log("power cap: through the Igneum Power Helper task (no prompt)");
|
|
std::thread::spawn(move || {
|
|
let r = crate::powertask::start().and_then(|_| {
|
|
let _ = std::fs::create_dir_all(&dir);
|
|
let mut seq = crate::platform::unix_now() % 1_000_000;
|
|
let mut text = String::new();
|
|
for (dev, w) in &pairs {
|
|
seq += 1;
|
|
text.push_str(&format!("{seq} dev {dev}\n"));
|
|
seq += 1;
|
|
text.push_str(&format!("{seq} pl {w}\n"));
|
|
}
|
|
std::fs::write(dir.join("cmd.txt"), text).map_err(|e| e.to_string())
|
|
});
|
|
std::thread::sleep(Duration::from_secs(6));
|
|
let back: std::collections::HashMap<String, f64> = crate::detect::nvidia_power_limits().into_iter().map(|(k, v)| (k, v.1)).collect();
|
|
shared.send(Cmd::PowerApplied(what, r, back));
|
|
});
|
|
return;
|
|
}
|
|
// the first approval registers the task in the same elevated step as the caps (Windows), so no later step
|
|
// needs a prompt: the registration script is written next to the command file
|
|
let line = if cfg!(windows) {
|
|
let dir = self.sweep_dir();
|
|
let _ = std::fs::create_dir_all(&dir);
|
|
let script = dir.join("register-power-task.ps1");
|
|
let installed: Vec<PathBuf> = crate::powertask::install_candidates();
|
|
match std::env::current_exe().map(|exe| std::fs::write(&script, [b"\xEF\xBB\xBF".as_slice(), crate::powertask::register_script(&crate::powertask::task_exe(&exe, &installed)).as_bytes()].concat())) {
|
|
Ok(Ok(())) => format!("{} & \"{}\" -NoProfile -ExecutionPolicy Bypass -File \"{}\"", cmds.join(" & "), crate::platform::tool("powershell").display(), script.display()),
|
|
_ => cmds.join(" & "),
|
|
}
|
|
} else {
|
|
cmds.join(" & ")
|
|
};
|
|
let want: std::collections::HashMap<String, f64> = self.st().mining.cards.iter().filter(|c| c.vendor == "nvidia" && c.enabled && c.present() && c.power_default_w > 0.0).map(|c| (c.device.clone(), requested_watts(c))).collect();
|
|
if self.wrapper && cfg!(windows) {
|
|
// the window host has a UI context: it shows the administrator prompt and reports back on stdin
|
|
self.power_via_host = Some((line.clone(), what.clone(), want, Instant::now()));
|
|
println!("ELEVATE {line}");
|
|
let _ = std::io::stdout().flush();
|
|
return;
|
|
}
|
|
let shared = self.shared.clone();
|
|
std::thread::spawn(move || {
|
|
let r = crate::platform::run_elevated(&line);
|
|
std::thread::sleep(Duration::from_millis(800));
|
|
let back: std::collections::HashMap<String, f64> = crate::detect::nvidia_power_limits().into_iter().map(|(k, v)| (k, v.1)).collect();
|
|
shared.send(Cmd::PowerApplied(what, r, back));
|
|
});
|
|
}
|
|
|
|
/// After an elevated step (ours or the host's): read the limits back and judge.
|
|
fn finish_power(&mut self, what: String, r: Result<(), String>, _want: std::collections::HashMap<String, f64>) {
|
|
let shared = self.shared.clone();
|
|
std::thread::spawn(move || {
|
|
std::thread::sleep(Duration::from_millis(800));
|
|
let back: std::collections::HashMap<String, f64> = crate::detect::nvidia_power_limits().into_iter().map(|(k, v)| (k, v.1)).collect();
|
|
shared.send(Cmd::PowerApplied(what, r, back));
|
|
});
|
|
}
|
|
|
|
/// On quit: the limits the cards had before the app (one more prompt; nvidia-smi limits do not survive a reboot anyway).
|
|
fn restore_power_limits(&mut self) {
|
|
if !self.power_restore_pending {
|
|
return;
|
|
}
|
|
let st = self.st();
|
|
let cmds: Vec<String> = st
|
|
.mining
|
|
.cards
|
|
.iter()
|
|
.filter(|c| c.vendor == "nvidia" && c.present() && c.power_applied && c.power_before_w > 0.0)
|
|
.map(|c| format!("\"{}\" -i {} -pl {}", crate::platform::tool("nvidia-smi").display(), c.device, c.power_before_w.round() as u64))
|
|
.collect();
|
|
drop(st);
|
|
if cmds.is_empty() {
|
|
return;
|
|
}
|
|
// no administrator prompt on quit (5 October 2026: the app never asks on its own); the limits reset at the
|
|
// next reboot, and the next start with Power control on sets them again
|
|
self.shared.log(&format!("GPU power limits left as set (they reset at the next reboot; no prompt on quit): {}", cmds.join(" & ")));
|
|
}
|
|
|
|
/// Is the Igneum Power Helper task registered (src/powertask.rs)? Asked once per run and after every elevated step.
|
|
fn power_task_registered(&mut self) -> bool {
|
|
if self.power_task.is_none() {
|
|
self.power_task = Some(crate::powertask::registered());
|
|
}
|
|
self.power_task.unwrap_or(false)
|
|
}
|
|
|
|
/// Power control (config.rs power_control): may the engine ask for administrator rights for the cap or the sweep?
|
|
/// The elevated PC sweep job (--sweep) sets caps directly and counts as allowed.
|
|
fn elevation_allowed(&self) -> bool {
|
|
elevation_allowed(self.shared.settings.lock().unwrap().power_control, self.shared.runtime.sweep_only)
|
|
}
|
|
|
|
/// Power control off, with the reason beside the switch and in the feed; a running or queued sweep ends.
|
|
fn power_control_off(&mut self, note: &str) {
|
|
{
|
|
let mut s = self.shared.settings.lock().unwrap();
|
|
s.power_control = false;
|
|
s.sweep = false;
|
|
}
|
|
self.shared.save_settings();
|
|
{
|
|
let mut st = self.st();
|
|
st.settings.power_control = false;
|
|
st.settings.sweep = false;
|
|
st.settings.power_note = note.to_string();
|
|
}
|
|
self.sweep_queue.clear();
|
|
if self.sweep.is_some() || self.sweep_pending.is_some() {
|
|
self.sweep_abort("power control is off");
|
|
}
|
|
if cfg!(windows) && self.power_task_registered() {
|
|
// the kill switch: the task unregisters itself (elevated) and exits; nothing is left behind
|
|
let hdir = crate::powertask::helper_dir();
|
|
let _ = std::fs::create_dir_all(&hdir);
|
|
let _ = std::fs::write(hdir.join("cmd.txt"), "remove\n");
|
|
match crate::powertask::start() {
|
|
Ok(()) => self.shared.log("power control off: the Igneum Power Helper task removes itself"),
|
|
Err(e) => self.shared.log(&format!("power control off: the task could not be started to remove itself ({e}); remove it in Task Scheduler")),
|
|
}
|
|
self.power_task = None;
|
|
}
|
|
self.shared.event(if note.starts_with("power control off:") { "error" } else { "info" }, note);
|
|
}
|
|
|
|
/// nvidia-smi -l 5: power draw, GPU and memory temperature, the limit in force, every 5 s, as a child whose
|
|
/// lines come through the same channel as the miners'.
|
|
fn tick_telemetry(&mut self, now: Instant) {
|
|
self.tick_amd_telemetry(now);
|
|
if cfg!(target_os = "macos") || self.st().mining.cards.iter().all(|c| c.vendor != "nvidia") {
|
|
return;
|
|
}
|
|
if let Some(t) = self.telemetry.as_mut() {
|
|
if !t.alive() {
|
|
self.telemetry = None;
|
|
self.telemetry_retry_at = now + Duration::from_secs(30);
|
|
}
|
|
}
|
|
if self.telemetry.is_none() && now >= self.telemetry_retry_at {
|
|
let args: Vec<String> = ["--query-gpu=index,power.draw,temperature.gpu,temperature.memory,power.limit,clocks.gr,clocks.mem", "--format=csv,noheader,nounits", "-l", "5"].iter().map(|s| s.to_string()).collect();
|
|
let log = self.shared.runtime.log_dir.join(format!("gpu-{}.log", self.stamp));
|
|
match procs::spawn(Source::Telemetry, &crate::platform::tool("nvidia-smi"), &args, None, &log, &self.lines_tx, &[]) {
|
|
Ok(p) => self.telemetry = Some(p),
|
|
Err(_) => self.telemetry_retry_at = now + Duration::from_secs(300),
|
|
}
|
|
}
|
|
if now.duration_since(self.last_stability) >= Duration::from_secs(300) {
|
|
self.last_stability = now;
|
|
self.stability_line();
|
|
}
|
|
if let Some((line, what, want, since)) = self.power_via_host.clone() {
|
|
if now.duration_since(since) > Duration::from_secs(150) {
|
|
// no second prompt through PowerShell (5 October 2026): an unanswered prompt is a refusal
|
|
self.power_via_host = None;
|
|
self.shared.log(&format!("the window host did not answer the elevated step in 150 s: {line}"));
|
|
self.finish_power(what, Err("the administrator prompt was not answered in 150 s".into()), want);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// AMD cards: igneum-gpu-telemetry -l 5 (ADLX on Windows, the amdgpu sysfs on Linux; proto-opencl/gpu-telemetry.c),
|
|
/// restarted 30 s after it ends, 300 s after it could not start. Nothing on macOS or without an AMD card.
|
|
fn tick_amd_telemetry(&mut self, now: Instant) {
|
|
if cfg!(target_os = "macos") || self.st().mining.cards.iter().all(|c| c.vendor != "amd") {
|
|
return;
|
|
}
|
|
let Some(exe) = self.bins.telemetry.clone() else { return };
|
|
if let Some(t) = self.amd_telemetry.as_mut() {
|
|
if !t.alive() {
|
|
self.amd_telemetry = None;
|
|
self.amd_telemetry_retry_at = now + Duration::from_secs(30);
|
|
}
|
|
}
|
|
if self.amd_telemetry.is_none() && now >= self.amd_telemetry_retry_at {
|
|
let args: Vec<String> = vec!["-l".into(), "5".into()];
|
|
let log = self.shared.runtime.log_dir.join(format!("gpu-amd-{}.log", self.stamp));
|
|
match procs::spawn(Source::AmdTelemetry, &exe, &args, None, &log, &self.lines_tx, &[]) {
|
|
Ok(p) => self.amd_telemetry = Some(p),
|
|
Err(_) => self.amd_telemetry_retry_at = now + Duration::from_secs(300),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One `amd ...` line of igneum-gpu-telemetry to the matching card. The helper lists cards in ADLX (or sysfs)
|
|
/// order with their kind; the app's AMD cards come from the OpenCL worker's list, which has no bus. They are
|
|
/// matched by kind (integrated, discrete) and ordinal within the kind, which is exact for one card of each kind
|
|
/// (PC 1: the gfx1036 and the 9070 XT) and approximate for two discrete AMD cards of one model.
|
|
fn amd_telemetry_line(&mut self, text: &str) {
|
|
let Some(s) = parse_amd_telemetry(text) else { return };
|
|
let mut st = self.st();
|
|
let mut nth = 0usize;
|
|
let mut target: Option<usize> = None;
|
|
for c in st.mining.cards.iter() {
|
|
if c.vendor != "amd" || c.kind != s.kind {
|
|
continue;
|
|
}
|
|
if nth == s.ordinal_in_kind {
|
|
target = Some(c.index);
|
|
break;
|
|
}
|
|
nth += 1;
|
|
}
|
|
let Some(idx) = target else { return };
|
|
let Some(c) = st.mining.cards.iter_mut().find(|c| c.index == idx) else { return };
|
|
if s.watts > 0.0 {
|
|
c.power_w = s.watts;
|
|
}
|
|
if s.temp_c > 0.0 {
|
|
c.temp_gpu = s.temp_c;
|
|
}
|
|
c.fan_pct = s.fan_pct.max(0.0);
|
|
c.fan_rpm = s.fan_rpm.max(0.0);
|
|
c.mclk_mhz = s.mclk_mhz.max(0.0);
|
|
c.gclk_mhz = s.gclk_mhz.max(0.0);
|
|
c.util_pct = s.util_pct.max(0.0);
|
|
c.amd_ordinal = s.ordinal as i64;
|
|
if s.gmax_mhz > 0.0 {
|
|
c.clock_cap_mhz = s.gmax_mhz as u32;
|
|
}
|
|
c.telemetry_at = crate::platform::unix_now_f();
|
|
let (idx, draw, gclk, mclk, temp) = (c.index, s.watts, s.gclk_mhz, s.mclk_mhz, s.temp_c);
|
|
drop(st);
|
|
if let Some(run) = self.sweep.as_mut() {
|
|
if run.card == idx {
|
|
run.sample_telemetry(draw.max(0.0), gclk.max(0.0), mclk.max(0.0), temp.max(0.0));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// "index, draw, gpu temp, mem temp, limit" every 5 s.
|
|
fn telemetry_line(&mut self, text: &str) {
|
|
let p: Vec<&str> = text.split(',').map(|s| s.trim()).collect();
|
|
if p.len() < 5 {
|
|
return;
|
|
}
|
|
let f = |s: &str| s.parse::<f64>().unwrap_or(0.0);
|
|
let (draw, tgpu, tmem, limit) = (f(p[1]), f(p[2]), f(p[3]), f(p[4]));
|
|
let (gclk, mclk) = (p.get(5).map(|v| f(v)).unwrap_or(0.0), p.get(6).map(|v| f(v)).unwrap_or(0.0));
|
|
let sweeping = self.sweep.is_some() || self.sweep_pending.is_some();
|
|
let mut st = self.st();
|
|
let Some(c) = st.mining.cards.iter_mut().find(|c| c.vendor == "nvidia" && c.device == p[0]) else { return };
|
|
c.power_w = draw;
|
|
c.temp_gpu = tgpu;
|
|
c.temp_mem = tmem;
|
|
if gclk > 0.0 {
|
|
c.gclk_mhz = gclk;
|
|
}
|
|
if mclk > 0.0 {
|
|
c.mclk_mhz = mclk;
|
|
}
|
|
if limit > 0.0 {
|
|
c.power_limit_w = limit;
|
|
// the readback is the truth: a cap the elevated step set after the judgement (or one set by hand
|
|
// outside the app) that matches what was asked counts as applied
|
|
if !sweeping && !c.power_applied && c.enabled && c.power_default_w > 0.0 && (limit - requested_watts(c)).abs() < 1.0 {
|
|
c.power_applied = true;
|
|
c.power_note = format!("power cap: {} W in force (read back)", limit as u64);
|
|
}
|
|
}
|
|
c.telemetry_at = crate::platform::unix_now_f();
|
|
let idx = c.index;
|
|
let mining = c.state == "mining";
|
|
drop(st);
|
|
if let Some(run) = self.sweep.as_mut() {
|
|
if run.card == idx {
|
|
run.sample_telemetry(draw, gclk, mclk, tgpu);
|
|
}
|
|
}
|
|
if mining {
|
|
let s = self.stability.entry(idx).or_default();
|
|
s.draws.push(draw);
|
|
if s.draws.len() > 20_000 {
|
|
s.draws.remove(0);
|
|
}
|
|
s.max_tgpu = s.max_tgpu.max(tgpu);
|
|
s.max_tmem = s.max_tmem.max(tmem);
|
|
}
|
|
}
|
|
|
|
/// One line per card: power draw p95 and the hottest memory and GPU readings this session, for the log drawer,
|
|
/// so a crash can be read back against the numbers.
|
|
fn stability_line(&mut self) {
|
|
let cards = self.st().mining.cards.clone();
|
|
for (idx, s) in self.stability.iter() {
|
|
if s.draws.is_empty() {
|
|
continue;
|
|
}
|
|
let Some(c) = cards.iter().find(|c| c.index == *idx) else { continue };
|
|
let mut v = s.draws.clone();
|
|
v.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
|
let p95 = v[((v.len() as f64 * 0.95) as usize).min(v.len() - 1)];
|
|
self.shared.log(&format!(
|
|
"stability: {}: draw p95 {:.0} W, max {:.0} W, limit {:.0} W ({}), max GPU {:.0} C, max memory {:.0} C, {} samples",
|
|
c.name,
|
|
p95,
|
|
v[v.len() - 1],
|
|
c.power_limit_w,
|
|
if c.power_applied { format!("cap in force, {}% of default", c.power_pct) } else { "cap NOT applied".to_string() },
|
|
s.max_tgpu,
|
|
s.max_tmem,
|
|
v.len()
|
|
));
|
|
}
|
|
}
|
|
|
|
// ---- Ember Tune (src/ember.rs): every card tuned for MH per watt ---------------------------------------------
|
|
|
|
/// A TUNE line: the app log, and stdout under --sweep (the PC job reads it there).
|
|
fn sweep_say(&self, line: &str) {
|
|
self.shared.log(line);
|
|
if self.shared.runtime.sweep_only {
|
|
println!("{line}");
|
|
let _ = std::io::stdout().flush();
|
|
}
|
|
}
|
|
|
|
fn sweep_dir(&self) -> PathBuf {
|
|
self.shared.runtime.app_dir.join("sweep")
|
|
}
|
|
|
|
/// Where this engine's helper reads its commands: the task's own folder when the task is the helper, else this
|
|
/// engine's sweep folder (the prompted helper script was handed that path on its command line).
|
|
fn helper_cmd_dir(&self) -> PathBuf {
|
|
if self.sweep_helper_is_task {
|
|
crate::powertask::helper_dir()
|
|
} else {
|
|
self.sweep_dir()
|
|
}
|
|
}
|
|
|
|
/// The manifest's tuning object (<app data>/tuning.json, written by the updater from the signed manifest), or
|
|
/// None: the Ember settings (kill switch, thresholds) and the fleet priors live under it.
|
|
fn tuning_object(&self) -> Option<Value> {
|
|
let p = self.ota.tuning_path()?;
|
|
std::fs::read_to_string(p).ok().and_then(|t| serde_json::from_str(&t).ok())
|
|
}
|
|
|
|
/// Every 10 s: drive the running tune, else start one that is due. One card at a time. Never while a remote
|
|
/// job holds the GPU, never under a pause, never inside the last 10 minutes before the hour boundary, never
|
|
/// while the manifest's kill switch is off. A card is due once after install, then every period, and again
|
|
/// after a driver or program-class change; a pinned card is skipped; "Tune now" queues one regardless.
|
|
fn tick_sweep(&mut self, now: Instant) {
|
|
if self.sweep.is_some() {
|
|
self.sweep_drive(now);
|
|
return;
|
|
}
|
|
if self.sweep_pending.is_some() || now < self.sweep_next_check {
|
|
return;
|
|
}
|
|
self.sweep_next_check = now + Duration::from_secs(10);
|
|
if self.quitting || !self.running || self.power_busy || self.job_hold || self.jobs.holds_miners() {
|
|
return;
|
|
}
|
|
let tuning = self.tuning_object();
|
|
let ember = crate::ember::settings_of(tuning.as_ref());
|
|
let (auto_on, paused, power_control) = {
|
|
let s = self.shared.settings.lock().unwrap();
|
|
let st = self.st();
|
|
(s.sweep, st.mining.paused, elevation_allowed(s.power_control, self.shared.runtime.sweep_only))
|
|
};
|
|
{
|
|
let mut st = self.st();
|
|
st.settings.tuning_off = !ember.enabled;
|
|
st.settings.tuning_note = if ember.enabled { String::new() } else { "tuning paused fleet-wide by the signed manifest".into() };
|
|
st.settings.tune_period_s = ember.period_s;
|
|
}
|
|
if paused || !ember.enabled {
|
|
return;
|
|
}
|
|
let unix = crate::platform::unix_now();
|
|
let (eta_known, eta_s) = {
|
|
let st = self.st();
|
|
(st.node.daa > 0 && st.program.boundary_daa > 0, st.program.eta_s)
|
|
};
|
|
let boundary_ok = !eta_known || eta_s > crate::ember::NEEDS_S;
|
|
let prefs = self.shared.settings.lock().unwrap().cards.clone();
|
|
let cards = self.st().mining.cards.clone();
|
|
let mut pick: Option<(usize, bool)> = None;
|
|
for c in cards.iter() {
|
|
if !(c.enabled && matches!(c.vendor.as_str(), "nvidia" | "amd" | "apple")) {
|
|
continue;
|
|
}
|
|
let forced = self.sweep_queue.contains(&c.key);
|
|
let due = match prefs.get(&c.key) {
|
|
None => true,
|
|
Some(p) => p.sweep_at == 0 || unix.saturating_sub(p.sweep_at) >= ember.period_s || (!c.driver.is_empty() && !p.sweep_driver.is_empty() && crate::ember::driver_major(&c.driver) != crate::ember::driver_major(&p.sweep_driver)) || (!c.program_class.is_empty() && !p.sweep_class.is_empty() && c.program_class != p.sweep_class),
|
|
};
|
|
let auto = auto_on || !c.tune_control;
|
|
// a measure-only card (Apple, NVIDIA without Power control) measures its baseline once a period even
|
|
// with the switch off: the row should say what the card does
|
|
if !forced && !(auto && !c.pinned && due) {
|
|
continue;
|
|
}
|
|
if c.tune_control && c.vendor == "nvidia" && !power_control && !forced {
|
|
// measure-only until Power control is on; the baseline plan covers it below
|
|
}
|
|
if self.sweep_retry.get(&c.index).map(|t| now < *t).unwrap_or(false) {
|
|
continue;
|
|
}
|
|
let stable = self.miners.iter().find(|m| m.card == c.index).map(|m| m.proc.as_ref().map(|p| p.started.elapsed() >= Duration::from_secs(crate::ember::STABLE_S)).unwrap_or(false) && m.last_status.is_some()).unwrap_or(false);
|
|
let wait = if c.state != "mining" {
|
|
Some(format!("tuning: waits for the card to mine ({})", c.state))
|
|
} else if !stable {
|
|
Some(format!("tuning: waits for {} s of steady mining", crate::ember::STABLE_S))
|
|
} else if !boundary_ok {
|
|
Some(format!("tuning: starts after the hour boundary ({eta_s} s)"))
|
|
} else {
|
|
None
|
|
};
|
|
match wait {
|
|
Some(w) => {
|
|
if forced || c.sweep_pct == 0 {
|
|
if let Some(cc) = self.st().mining.cards.get_mut(c.index) {
|
|
cc.sweep_note = w;
|
|
}
|
|
}
|
|
}
|
|
None => {
|
|
pick = Some((c.index, forced));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
if let Some((idx, forced)) = pick {
|
|
self.sweep_begin(idx, forced);
|
|
} else if self.shared.runtime.sweep_only && self.sweep_queue.is_empty() && self.sweep_pending.is_none() {
|
|
self.shared.send(Cmd::Quit("the --sweep run (every card done)"));
|
|
}
|
|
}
|
|
|
|
/// Step 1 of a tune: the probe. NVIDIA: the vendor's maximum core clock and driver (nvidia-smi query), and
|
|
/// whether this process may set limits (one `-pl` at the current limit: "All done" = direct; else the elevated
|
|
/// helper, only with Power control on; else measure only). AMD: the helper's `--tune` line (the clock and power
|
|
/// ranges in force; no elevation). Apple: measure only.
|
|
fn sweep_begin(&mut self, idx: usize, forced: bool) {
|
|
let Some(c) = self.st().mining.cards.get(idx).cloned() else { return };
|
|
self.sweep_pending = Some((idx, forced));
|
|
self.sweep_queue.retain(|k| k != &c.key);
|
|
*self.sweep_attempts.entry(idx).or_default() += 1;
|
|
if let Some(cc) = self.st().mining.cards.get_mut(idx) {
|
|
cc.sweep_state = "running".into();
|
|
cc.sweep_note = "tuning: reading what the card allows".into();
|
|
}
|
|
let shared = self.shared.clone();
|
|
let allowed = self.elevation_allowed();
|
|
match c.vendor.as_str() {
|
|
"nvidia" => {
|
|
let smi = crate::platform::tool("nvidia-smi");
|
|
let device = c.device.clone();
|
|
let current = if c.power_limit_w > 0.0 { c.power_limit_w } else { c.power_default_w }.round() as u64;
|
|
let known_direct = self.sweep_direct;
|
|
std::thread::spawn(move || {
|
|
let q = crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "--query-gpu=clocks.max.gr,driver_version,clocks.mem,clocks.max.mem", "--format=csv,noheader,nounits"]), None, Duration::from_secs(10)).unwrap_or_default();
|
|
let p: Vec<&str> = q.trim().split(',').map(|s| s.trim()).collect();
|
|
let clock_max = p.first().and_then(|s| s.parse::<f64>().ok()).unwrap_or(0.0) as u32;
|
|
let driver = p.get(1).map(|s| s.to_string()).unwrap_or_default();
|
|
// Ember 2: the memory clock now (the default under load) and the vendor's maximum
|
|
let mem_default = p.get(2).and_then(|s| s.parse::<f64>().ok()).unwrap_or(0.0) as u32;
|
|
let mem_max = p.get(3).and_then(|s| s.parse::<f64>().ok()).unwrap_or(0.0) as u32;
|
|
let direct = match known_direct {
|
|
Some(d) => Some(d),
|
|
None if allowed || current > 0 => crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "-pl", ¤t.to_string()]), None, Duration::from_secs(20)).map(|out| out.contains("All done")),
|
|
None => Some(false),
|
|
};
|
|
shared.send(Cmd::TuneProbe(idx, Ok(TuneProbe { clock_max_mhz: clock_max, clock_min_mhz: 0, driver, direct: direct.unwrap_or(false), amd_ordinal: -1, mem_default_mhz: mem_default, mem_max_mhz: mem_max, ..Default::default() })));
|
|
});
|
|
}
|
|
"amd" => {
|
|
let Some(exe) = self.bins.telemetry.clone() else {
|
|
self.sweep_pending = None;
|
|
self.shared.send(Cmd::TuneProbe(idx, Ok(TuneProbe { clock_max_mhz: 0, clock_min_mhz: 0, driver: c.driver.clone(), direct: false, amd_ordinal: -1, ..Default::default() })));
|
|
return;
|
|
};
|
|
let ordinal = c.amd_ordinal;
|
|
let driver = c.driver.clone();
|
|
std::thread::spawn(move || {
|
|
let out = crate::detect::run_timeout(std::process::Command::new(&exe).arg("--tune"), None, Duration::from_secs(20)).unwrap_or_default();
|
|
let t = out.lines().filter_map(parse_amd_tune).find(|t| ordinal < 0 || t.ordinal as i64 == ordinal);
|
|
shared.send(Cmd::TuneProbe(idx, Ok(match t {
|
|
// PC 1's 9070 XT (ember-tune-pc1-1, 22:30 UTC): `gmax 0 gmax_range -500 1000`, an OFFSET from
|
|
// the stock clock, not MHz; a range with a negative floor is an offset range and the clock
|
|
// knob stays closed until the stock clock is known (the power limit is the AMD lever), and
|
|
// `plimit_range -30 10` bounds the power ladder (the percent scale rides power_* below)
|
|
Some(t) if t.ok => TuneProbe { clock_max_mhz: if t.gmax_min >= 0.0 && t.gmax_max > 0.0 { t.gmax_max as u32 } else { 0 }, clock_min_mhz: if t.gmax_min > 0.0 { t.gmax_min as u32 } else { 0 }, driver, direct: true, amd_ordinal: t.ordinal as i64, plimit_min: t.plimit_min, plimit_max: t.plimit_max, mem_default_mhz: 0, mem_max_mhz: 0 },
|
|
Some(t) => TuneProbe { clock_max_mhz: 0, clock_min_mhz: 0, driver, direct: false, amd_ordinal: t.ordinal as i64, ..Default::default() },
|
|
None => TuneProbe { clock_max_mhz: 0, clock_min_mhz: 0, driver, direct: false, amd_ordinal: -1, ..Default::default() },
|
|
})));
|
|
});
|
|
}
|
|
_ => {
|
|
self.shared.send(Cmd::TuneProbe(idx, Ok(TuneProbe { clock_max_mhz: 0, clock_min_mhz: 0, driver: c.driver.clone(), direct: false, amd_ordinal: -1, ..Default::default() })));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Step 2: the plan from what the probe found (full, confirm from the fleet prior, or baseline), the helper
|
|
/// when NVIDIA needs it, then the run.
|
|
fn sweep_probe_known(&mut self, idx: usize, r: Result<TuneProbe, String>) {
|
|
let Some((pidx, forced)) = self.sweep_pending else { return };
|
|
if pidx != idx {
|
|
return;
|
|
}
|
|
let probe = match r {
|
|
Ok(p) => p,
|
|
Err(e) => {
|
|
self.sweep_abort(&format!("cannot read the card's limits: {e}"));
|
|
return;
|
|
}
|
|
};
|
|
let Some(c) = self.st().mining.cards.get(idx).cloned() else {
|
|
self.sweep_pending = None;
|
|
return;
|
|
};
|
|
// NVIDIA control: this process is elevated (direct), or Power control is on so the one-prompt helper may run.
|
|
// The --sweep job alone never counts: it must not raise a prompt on a PC with nobody there (5 October 2026).
|
|
// a registered Igneum Power Helper task (src/powertask.rs) is permission too: it sets limits with no prompt,
|
|
// so an unelevated measurement engine (Power control forced off) still controls NVIDIA through it
|
|
let power_control = probe.direct || self.shared.settings.lock().unwrap().power_control || (cfg!(windows) && self.power_task_registered());
|
|
// AMD's power limit is a percent offset from the default (ADLX): the plan's watts scale becomes a percent
|
|
// scale (default 100, floor 100 + plimit_min, ceiling 100 + plimit_max), tune_apply sends pct - 100
|
|
let limits = if c.vendor == "amd" && probe.direct && probe.plimit_max >= probe.plimit_min && probe.plimit_min > -100.0 {
|
|
crate::ember::Limits { power_default_w: 100.0, power_min_w: 100.0 + probe.plimit_min, power_max_w: 100.0 + probe.plimit_max, clock_max_mhz: probe.clock_max_mhz, clock_min_mhz: probe.clock_min_mhz, mem_default_mhz: 0, mem_max_mhz: 0 }
|
|
} else {
|
|
crate::ember::Limits { power_default_w: c.power_default_w, power_min_w: c.power_min_w, power_max_w: c.power_max_w, clock_max_mhz: probe.clock_max_mhz, clock_min_mhz: probe.clock_min_mhz, mem_default_mhz: probe.mem_default_mhz, mem_max_mhz: probe.mem_max_mhz }
|
|
};
|
|
let control = match c.vendor.as_str() {
|
|
"nvidia" => crate::ember::control_reason("nvidia", &limits, &c.device, power_control, false),
|
|
"amd" => crate::ember::control_reason("amd", &limits, &c.device, power_control, probe.direct && probe.amd_ordinal >= 0),
|
|
v => crate::ember::control_reason(v, &limits, &c.device, power_control, false),
|
|
};
|
|
if c.vendor == "nvidia" {
|
|
self.sweep_direct = Some(probe.direct);
|
|
// the approved step that made this engine elevated registers the Igneum Power Helper here too (run 5,
|
|
// 6 October 2026: a --sweep engine skipped the cap path where the registration lived, so the one
|
|
// approved click registered nothing); no prompt: this process already holds the rights
|
|
if probe.direct && cfg!(windows) && !self.power_task_registered() {
|
|
let dir = self.sweep_dir();
|
|
let _ = std::fs::create_dir_all(&dir);
|
|
let script = dir.join("register-power-task.ps1");
|
|
let installed: Vec<PathBuf> = crate::powertask::install_candidates();
|
|
if let Ok(exe) = std::env::current_exe() {
|
|
let target = crate::powertask::task_exe(&exe, &installed);
|
|
if std::fs::write(&script, [b"\xEF\xBB\xBF".as_slice(), crate::powertask::register_script(&target).as_bytes()].concat()).is_ok() {
|
|
let mut p = std::process::Command::new(crate::platform::tool("powershell"));
|
|
p.args(["-NoProfile", "-ExecutionPolicy", "Bypass", "-File", &script.display().to_string()]);
|
|
crate::platform::quiet(&mut p);
|
|
let ok = p.status().map(|s| s.success()).unwrap_or(false);
|
|
self.power_task = None;
|
|
self.sweep_say(&format!("TUNE helper registered={} action={}", ok, target.display()));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
{
|
|
let mut st = self.st();
|
|
if let Some(cc) = st.mining.cards.get_mut(idx) {
|
|
cc.clock_max_mhz = probe.clock_max_mhz;
|
|
cc.clock_min_mhz = probe.clock_min_mhz;
|
|
if !probe.driver.is_empty() {
|
|
cc.driver = probe.driver.clone();
|
|
}
|
|
if probe.amd_ordinal >= 0 {
|
|
cc.amd_ordinal = probe.amd_ordinal;
|
|
}
|
|
cc.tune_control = control.is_none();
|
|
}
|
|
}
|
|
let tuning = self.tuning_object();
|
|
let ember = crate::ember::settings_of(tuning.as_ref());
|
|
let before = crate::ember::Point { clock_mhz: c.clock_cap_mhz, power_pct: if c.power_pct == 0 { 80 } else { c.power_pct }, mem_mhz: c.mem_cap_mhz };
|
|
let before_w = if c.power_limit_w > 0.0 { c.power_limit_w } else { requested_watts(&c) };
|
|
let key = crate::ember::prior_key(&c.name, &probe.driver, &c.program_class);
|
|
let prior = crate::ember::prior_of(tuning.as_ref(), &key, ember.min_samples);
|
|
let full_due = self.tune_full_due.remove(&idx);
|
|
let (goal, climb_on) = {
|
|
let s = self.shared.settings.lock().unwrap();
|
|
(crate::ember::Goal::parse(&s.tune_goal), s.tune_climb)
|
|
};
|
|
let plan = if let Some(why) = control.as_ref() {
|
|
if let Some(cc) = self.st().mining.cards.get_mut(idx) {
|
|
cc.sweep_note = why.clone();
|
|
}
|
|
crate::ember::Plan::baseline(&limits, before, ember.tolerance_pct)
|
|
} else if climb_on || prior.as_ref().map(|p| p.point.mem_mhz > 0).unwrap_or(false) {
|
|
// Ember 2: the hill-climb from the fleet prior (or the card's point) toward the goal
|
|
let start = prior.as_ref().map(|p| p.point).unwrap_or(before);
|
|
crate::ember::Plan::climb(&limits, start, goal, ember.tolerance_pct)
|
|
} else if let (Some(p), false, 0) = (prior.as_ref(), full_due, c.sweep_pct) {
|
|
crate::ember::Plan::confirm(&limits, p.point, before, ember.tolerance_pct)
|
|
} else {
|
|
crate::ember::Plan::full(&limits, before, ember.tolerance_pct)
|
|
};
|
|
if plan.is_empty() {
|
|
self.sweep_abort("no steps: the card reported neither a default power limit nor a maximum clock");
|
|
return;
|
|
}
|
|
if c.vendor == "nvidia" && control.is_none() && !probe.direct && !self.sweep_helper {
|
|
if let Err(e) = self.sweep_helper_start(&c) {
|
|
self.sweep_abort(&format!("the elevated helper could not start: {e}"));
|
|
return;
|
|
}
|
|
}
|
|
let label = self.miners.iter().find(|m| m.card == idx).map(|m| m.label.clone()).unwrap_or_else(|| format!("card-{idx}"));
|
|
let now = Instant::now();
|
|
let kind = plan.kind;
|
|
let steps = plan.len();
|
|
let run = crate::ember::Run::new(idx, &c.device, &label, plan, before_w, forced, crate::ember::Timing::from_env(), now);
|
|
self.sweep_pending = None;
|
|
self.sweep_say(&format!(
|
|
"TUNE start card={label} name={} plan={} steps={} clock_max={} clock_min={} default={:.0} min={:.0} max={:.0} before_clock={} before_pct={} before={:.0} mode={} driver={} class={} prior={}",
|
|
c.name.replace(' ', "_"),
|
|
kind.name(),
|
|
steps,
|
|
limits.clock_max_mhz,
|
|
limits.clock_floor(),
|
|
limits.power_default_w,
|
|
limits.power_min_w,
|
|
limits.power_max_w,
|
|
before.clock_mhz,
|
|
before.power_pct,
|
|
before_w,
|
|
if control.is_some() { "measure" } else if c.vendor == "amd" { "adlx" } else if probe.direct { "direct" } else { "helper" },
|
|
probe.driver.replace(' ', "_"),
|
|
if c.program_class.is_empty() { "v2" } else { &c.program_class },
|
|
prior.as_ref().map(|p| format!("{}mhz/{}pct/{}samples", p.point.clock_mhz, p.point.power_pct, p.samples)).unwrap_or_else(|| "none".into())
|
|
));
|
|
let what = match kind {
|
|
crate::ember::PlanKind::Full => format!("{steps} steps over the power limit and the core clock, {} s each on the live program", (run.timing.settle + run.timing.hold).as_secs()),
|
|
crate::ember::PlanKind::Confirm => format!("the fleet prior ({} samples) and one neighbour, {} s each", prior.as_ref().map(|p| p.samples).unwrap_or(0), (run.timing.settle + run.timing.hold).as_secs()),
|
|
crate::ember::PlanKind::Baseline => format!("measuring the card as it runs ({})", control.clone().unwrap_or_default()),
|
|
crate::ember::PlanKind::Climb => format!("the hill-climb: memory up, core down, up to {steps} probes of {} s", (run.timing.settle + run.timing.hold).as_secs()),
|
|
};
|
|
self.shared.event("info", &format!("{}: tuning started: {what}", c.name));
|
|
self.sweep = Some(run);
|
|
self.sweep_drive(now);
|
|
}
|
|
|
|
/// The elevated helper (src/sweep.rs helper_script_*): one administrator prompt; it polls <app>/sweep/cmd.txt.
|
|
fn sweep_helper_start(&mut self, c: &CardState) -> Result<(), String> {
|
|
let task = cfg!(windows) && self.power_task_registered();
|
|
if !task && !self.shared.settings.lock().unwrap().power_control {
|
|
return Err("Power control is off in Settings".into());
|
|
}
|
|
let dir = self.sweep_dir();
|
|
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
|
let _ = std::fs::write(dir.join("cmd.txt"), "");
|
|
let restore = (if c.power_limit_w > 0.0 { c.power_limit_w } else { c.power_default_w }).round() as u64;
|
|
let smi = crate::platform::tool("nvidia-smi").display().to_string();
|
|
let line = if cfg!(windows) {
|
|
let script = dir.join("helper.ps1");
|
|
std::fs::write(&script, [b"\xEF\xBB\xBF".as_slice(), crate::sweep::helper_script_windows().replace('\n', "\r\n").as_bytes()].concat()).map_err(|e| e.to_string())?;
|
|
format!("\"{}\" -NoProfile -ExecutionPolicy Bypass -File \"{}\" \"{}\" \"{}\" {} {}", crate::platform::tool("powershell").display(), script.display(), dir.display(), smi, c.device, restore)
|
|
} else {
|
|
let script = dir.join("helper.sh");
|
|
std::fs::write(&script, crate::sweep::helper_script_unix()).map_err(|e| e.to_string())?;
|
|
format!("sh \"{}\" \"{}\" \"{}\" {} {}", script.display(), dir.display(), smi, c.device, restore)
|
|
};
|
|
if cfg!(windows) && self.power_task_registered() {
|
|
// once, ever: the registered task is the helper; it reads ITS command file (powertask::helper_dir, never
|
|
// under a scratch IGNEUM_APP_DATA), no prompt
|
|
let hdir = crate::powertask::helper_dir();
|
|
let _ = std::fs::create_dir_all(&hdir);
|
|
let _ = std::fs::write(hdir.join("cmd.txt"), format!("{} dev {}\n", crate::platform::unix_now() % 1_000_000, c.device));
|
|
crate::powertask::start()?;
|
|
self.shared.log("tune helper: the Igneum Power Helper task (no prompt)");
|
|
self.sweep_helper = true;
|
|
self.sweep_helper_is_task = true;
|
|
return Ok(());
|
|
}
|
|
self.shared.log(&format!("tune helper (administrator prompt): {line}"));
|
|
self.sweep_helper = true;
|
|
self.sweep_helper_is_task = false;
|
|
let shared = self.shared.clone();
|
|
std::thread::spawn(move || {
|
|
let r = crate::platform::run_elevated(&line);
|
|
shared.send(Cmd::SweepHelperDone(r));
|
|
});
|
|
Ok(())
|
|
}
|
|
|
|
fn sweep_helper_quit(&mut self) {
|
|
if self.sweep_helper {
|
|
let _ = std::fs::write(self.helper_cmd_dir().join("cmd.txt"), "quit\n");
|
|
if self.sweep_helper_is_task {
|
|
// the task exits on quit and reports nothing back; the next tune starts it again
|
|
self.sweep_helper = false;
|
|
self.sweep_helper_is_task = false;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Sets a point on the card. NVIDIA: `-pl <W>` and `-lgc 0,<MHz>` (`-rgc` for unlocked), directly when this
|
|
/// process is elevated, else through the helper's command file (`<seq> pl <W>`, `<seq> lgc <MHz>`, `<seq> rgc`).
|
|
/// AMD: igneum-gpu-telemetry `--card N --set-plimit <offset>` and `--set-gmax <MHz>` (`--reset` for the default
|
|
/// point). The result comes back as Cmd::TuneSet(seq, ..): the acknowledgement the run waits for.
|
|
fn tune_apply(&mut self, idx: usize, device: &str, step: &crate::ember::Step) {
|
|
let seq = self.sweep.as_ref().map(|r| r.seq).unwrap_or(0);
|
|
let card = self.st().mining.cards.get(idx).cloned();
|
|
let Some(c) = card else { return };
|
|
let w = step.watts.round() as u64;
|
|
let clock = step.point.clock_mhz;
|
|
let mem = step.point.mem_mhz;
|
|
let shared = self.shared.clone();
|
|
self.tune_acked = None;
|
|
match c.vendor.as_str() {
|
|
"nvidia" if self.sweep_direct == Some(true) => {
|
|
let smi = crate::platform::tool("nvidia-smi");
|
|
let device = device.to_string();
|
|
let power_only = c.power_default_w > 0.0;
|
|
std::thread::spawn(move || {
|
|
let mut ok = true;
|
|
let mut text = String::new();
|
|
if power_only && w > 0 {
|
|
let out = crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "-pl", &w.to_string()]), None, Duration::from_secs(20)).unwrap_or_else(|| "nvidia-smi did not answer".into());
|
|
ok &= out.contains("All done") || out.contains("Power limit for GPU");
|
|
text.push_str(out.trim());
|
|
}
|
|
let out = if clock > 0 {
|
|
crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "-lgc", &format!("0,{clock}")]), None, Duration::from_secs(20)).unwrap_or_else(|| "nvidia-smi did not answer".into())
|
|
} else {
|
|
crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "-rgc"]), None, Duration::from_secs(20)).unwrap_or_else(|| "nvidia-smi did not answer".into())
|
|
};
|
|
ok &= out.contains("All done") || out.to_ascii_lowercase().contains("clocks set") || out.to_ascii_lowercase().contains("reset");
|
|
text.push(' ');
|
|
text.push_str(out.trim());
|
|
// Ember 2: the memory clock, locked to one value (`-lmc m,m`) or reset (`-rmc`)
|
|
let out = if mem > 0 {
|
|
crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "-lmc", &format!("{mem},{mem}")]), None, Duration::from_secs(20)).unwrap_or_else(|| "nvidia-smi did not answer".into())
|
|
} else {
|
|
crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "-rmc"]), None, Duration::from_secs(20)).unwrap_or_else(|| "nvidia-smi did not answer".into())
|
|
};
|
|
ok &= out.contains("All done") || out.to_ascii_lowercase().contains("clocks set") || out.to_ascii_lowercase().contains("reset") || out.to_ascii_lowercase().contains("not supported");
|
|
text.push(' ');
|
|
text.push_str(out.trim());
|
|
shared.send(Cmd::TuneSet(seq, if ok { Ok(text) } else { Err(text) }));
|
|
});
|
|
}
|
|
"nvidia" => {
|
|
if !self.sweep_helper {
|
|
// measure only: nothing is set; the run notices the missing acknowledgement and measures
|
|
return;
|
|
}
|
|
let dev = device.to_string();
|
|
let cmd = format!("{seq}0 dev {dev}\n{seq}1 pl {w}\n{seq}2 {}\n{seq}3 {}\n", if clock > 0 { format!("lgc {clock}") } else { "rgc".to_string() }, if mem > 0 { format!("lmc {mem}") } else { "rmc".to_string() });
|
|
let _ = std::fs::write(self.helper_cmd_dir().join("cmd.txt"), cmd);
|
|
// the helper polls twice a second and nvidia-smi answers within a second or two
|
|
std::thread::spawn(move || {
|
|
std::thread::sleep(Duration::from_secs(4));
|
|
shared.send(Cmd::TuneSet(seq, Ok("helper".into())));
|
|
});
|
|
}
|
|
"amd" => {
|
|
let Some(exe) = self.bins.telemetry.clone() else { return };
|
|
if c.amd_ordinal < 0 {
|
|
return;
|
|
}
|
|
let n = c.amd_ordinal.to_string();
|
|
// the step's limit on the AMD scale is a percent (the probe's Limits); the offset is that minus 100
|
|
let offset = if c.power_default_w <= 0.0 { step.watts.round() as i64 - 100 } else { step.point.power_pct as i64 - 100 };
|
|
let unlocked = clock == 0 && offset == 0;
|
|
let gmax = if clock > 0 { clock } else { c.clock_max_mhz };
|
|
std::thread::spawn(move || {
|
|
let run = |args: &[&str]| crate::detect::run_timeout(std::process::Command::new(&exe).args(args), None, Duration::from_secs(20)).unwrap_or_else(|| "igneum-gpu-telemetry did not answer".into());
|
|
let mut text = String::new();
|
|
let mut ok = true;
|
|
if unlocked {
|
|
let out = run(&["--card", &n, "--reset"]);
|
|
ok &= out.lines().any(|l| l.starts_with("tune ") && l.contains(" ok"));
|
|
text.push_str(out.trim());
|
|
} else {
|
|
if gmax > 0 {
|
|
let out = run(&["--card", &n, "--set-gmax", &gmax.to_string()]);
|
|
ok &= out.lines().any(|l| l.starts_with("tune ") && l.contains(" ok"));
|
|
text.push_str(out.trim());
|
|
}
|
|
let out = run(&["--card", &n, "--set-plimit", &offset.to_string()]);
|
|
ok &= out.lines().any(|l| l.starts_with("tune ") && l.contains(" ok"));
|
|
text.push(' ');
|
|
text.push_str(out.trim());
|
|
}
|
|
shared.send(Cmd::TuneSet(seq, if ok { Ok(text) } else { Err(text) }));
|
|
});
|
|
}
|
|
_ => {}
|
|
}
|
|
self.shared.log(&format!("tune: {} MHz, {}% ({w} W), memory {} requested on device {device} (request {seq})", if clock > 0 { clock.to_string() } else { "unlocked".into() }, step.point.power_pct, if mem > 0 { format!("{mem} MHz") } else { "default".into() }));
|
|
}
|
|
|
|
/// A measurement engine's progress for one card (POST /api/tune-progress, forwarded by the job playbook from the
|
|
/// engine's `TUNE progress` lines): the installed app shows the step, the live rate and draw and the time left
|
|
/// instead of a bare "off" while a job holds its miners. `done` ends it: the row reads the tuned line.
|
|
fn tune_progress(&mut self, v: &Value) {
|
|
let key = v.get("key").and_then(|k| k.as_str()).unwrap_or("");
|
|
let mut st = self.st();
|
|
let Some(c) = st.mining.cards.iter_mut().find(|c| c.key == key || (!key.is_empty() && c.name.replace(' ', "_") == key)) else { return };
|
|
let n = |k: &str| v.get(k).and_then(|x| x.as_f64()).unwrap_or(0.0);
|
|
if v.get("done").and_then(|d| d.as_bool()).unwrap_or(false) {
|
|
c.sweep_state = "idle".into();
|
|
c.state = if self.job_hold { "held".into() } else { c.state.clone() };
|
|
c.tune_step = 0;
|
|
c.tune_steps = 0;
|
|
c.tune_eta_s = 0;
|
|
if n("mhs") > 0.0 && n("watts") > 0.0 {
|
|
c.sweep_mhs = n("mhs");
|
|
c.sweep_watts = n("watts");
|
|
c.sweep_eff = n("mhs") / n("watts");
|
|
c.sweep_pct = n("power_pct") as u32;
|
|
c.tune_clock_mhz = n("clock_mhz") as u32;
|
|
c.tune_source = v.get("plan").and_then(|p| p.as_str()).unwrap_or("full").into();
|
|
c.tune_line = crate::ember::tuned_line(n("mhs"), n("watts"), n("mhs") / n("watts"));
|
|
c.sweep_at = crate::platform::unix_now_f();
|
|
}
|
|
c.sweep_note = v.get("note").and_then(|x| x.as_str()).unwrap_or("").to_string();
|
|
return;
|
|
}
|
|
c.state = "tuning".into();
|
|
c.sweep_state = "running".into();
|
|
c.sweep_note = v.get("note").and_then(|x| x.as_str()).unwrap_or("tuning").to_string();
|
|
c.tune_step = n("step") as u32;
|
|
c.tune_steps = n("of") as u32;
|
|
c.tune_eta_s = n("eta_s") as i64;
|
|
c.tune_plan = v.get("plan").and_then(|p| p.as_str()).unwrap_or("").into();
|
|
if n("mhs") > 0.0 {
|
|
c.hash_now = n("mhs");
|
|
}
|
|
if n("watts") > 0.0 {
|
|
c.power_w = n("watts");
|
|
}
|
|
c.message = String::new();
|
|
}
|
|
|
|
/// Faults and conditions first, then the state machine.
|
|
fn sweep_drive(&mut self, now: Instant) {
|
|
let Some((idx, device, started, seq)) = self.sweep.as_ref().map(|r| (r.card, r.device.clone(), r.started, r.seq)) else { return };
|
|
let card = self.st().mining.cards.get(idx).cloned();
|
|
let Some(c) = card else {
|
|
self.sweep_abort("the card disappeared");
|
|
return;
|
|
};
|
|
let slot_error = self.miners.iter().find(|m| m.card == idx).and_then(|m| m.error_at).map(|t| t > started).unwrap_or(false);
|
|
let fault = if self.quitting {
|
|
Some("the app is quitting".to_string())
|
|
} else if self.job_hold || self.jobs.holds_miners() {
|
|
Some("a remote job took the GPU".into())
|
|
} else if self.st().mining.paused {
|
|
Some("mining paused".into())
|
|
} else if c.state != "mining" && c.state != "tuning" {
|
|
Some(format!("the card left mining ({})", if c.state == "starting" || c.state == "restarting" { "worker restart, likely the hour boundary" } else { &c.state }))
|
|
} else if slot_error {
|
|
Some("the worker reported an error".into())
|
|
} else if c.temp_gpu >= 90.0 {
|
|
Some(format!("GPU at {:.0} C", c.temp_gpu))
|
|
} else {
|
|
None
|
|
};
|
|
if let Some(why) = fault {
|
|
self.sweep_abort(&why);
|
|
return;
|
|
}
|
|
let acked = self.tune_acked == Some(seq);
|
|
let rb = crate::ember::Readback { limit_w: if c.vendor == "nvidia" { c.power_limit_w } else { 0.0 }, acked };
|
|
// the clock readback during a hold: a core clock well over the cap means the cap did not take
|
|
if let Some(run) = self.sweep.as_mut() {
|
|
if matches!(run.phase, crate::ember::Phase::Holding { .. }) {
|
|
if let Some(step) = run.current.as_ref() {
|
|
if step.point.clock_mhz > 0 && c.gclk_mhz > step.point.clock_mhz as f64 * 1.05 && c.vendor == "nvidia" && c.tune_control {
|
|
run.mark_unapplied();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
let outs = self.sweep.as_mut().map(|r| r.tick(now, rb)).unwrap_or_default();
|
|
let label = self.sweep.as_ref().map(|r| r.label.clone()).unwrap_or_default();
|
|
for o in outs {
|
|
match o {
|
|
crate::ember::Out::Apply(step) => self.tune_apply(idx, &device, &step),
|
|
crate::ember::Out::Row(row) => {
|
|
let i = self.sweep.as_ref().map(|r| r.rows.len().saturating_sub(1)).unwrap_or(0);
|
|
self.sweep_say(&row.line(&label, i));
|
|
if let Some(cc) = self.st().mining.cards.get_mut(idx) {
|
|
if row.usable() {
|
|
cc.sweep_note = format!("tuning: step {} done · {:.3} MH/W", i + 1, row.eff);
|
|
}
|
|
}
|
|
}
|
|
crate::ember::Out::Finished(row) => {
|
|
self.sweep_finish(row);
|
|
return;
|
|
}
|
|
crate::ember::Out::Failed(e) => {
|
|
self.sweep_abort(&e);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
if let Some((words, step, of, eta, plan)) = self.sweep.as_ref().map(|r| (r.words(now), r.rows.len() as u32 + 1, r.plan.len() as u32, r.eta_s(now), r.plan.kind.name())) {
|
|
let (key, mhs, watts) = self.st().mining.cards.get(idx).map(|c| (c.key.clone(), c.hash_now, c.power_w)).unwrap_or_default();
|
|
if let Some(cc) = self.st().mining.cards.get_mut(idx) {
|
|
cc.sweep_note = words.clone();
|
|
cc.state = if cc.state == "mining" { "tuning".into() } else { cc.state.clone() };
|
|
cc.tune_step = step;
|
|
cc.tune_steps = of;
|
|
cc.tune_eta_s = eta;
|
|
cc.tune_plan = plan.into();
|
|
}
|
|
if self.shared.runtime.sweep_only {
|
|
// the job playbook forwards this to the installed app's /api/tune-progress
|
|
println!("TUNE progress key={} step={step} of={of} eta_s={eta} mhs={mhs:.2} watts={watts:.1} plan={plan} note={}", key.replace(' ', "_"), words.replace(' ', "_"));
|
|
let _ = std::io::stdout().flush();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The chosen point is in force: record it on the card and in the settings, write the fleet record, say so,
|
|
/// let the helper go. A confirm check that found a better neighbour queues the full plan.
|
|
fn sweep_finish(&mut self, row: crate::ember::Row) {
|
|
let Some(run) = self.sweep.take() else { return };
|
|
let idx = run.card;
|
|
let unix = crate::platform::unix_now();
|
|
let kind = run.plan.kind;
|
|
let full_due = run.full_due();
|
|
let (name, pinned, control, vendor, driver, class, key) = {
|
|
let mut settings = self.shared.settings.lock().unwrap();
|
|
let mut st = self.st();
|
|
let Some(c) = st.mining.cards.get_mut(idx) else { return };
|
|
c.sweep_pct = row.point.power_pct;
|
|
c.sweep_eff = row.eff;
|
|
c.sweep_watts = row.watts;
|
|
c.sweep_mhs = row.mhs;
|
|
c.sweep_at = unix as f64;
|
|
c.sweep_state = "idle".into();
|
|
if c.state == "tuning" {
|
|
c.state = "mining".into();
|
|
}
|
|
c.tune_step = 0;
|
|
c.tune_steps = 0;
|
|
c.tune_eta_s = 0;
|
|
c.tune_clock_mhz = row.point.clock_mhz;
|
|
c.tune_mem_mhz = row.point.mem_mhz;
|
|
c.tune_source = kind.name().into();
|
|
c.tune_line = crate::ember::result_line(kind, row.mhs, row.watts, row.eff);
|
|
c.tune_curve = run.rows.iter().map(|r| r.json()).collect();
|
|
let control = c.tune_control;
|
|
if kind == crate::ember::PlanKind::Baseline {
|
|
c.sweep_note = if control { String::new() } else { c.sweep_note.clone() };
|
|
} else if c.pinned {
|
|
c.sweep_note = format!("tuning: best {} · {:.3} MH/W; your setting stays pinned", point_words(&row.point), row.eff);
|
|
} else {
|
|
c.power_pct = row.point.power_pct;
|
|
c.clock_cap_mhz = row.point.clock_mhz;
|
|
c.mem_cap_mhz = row.point.mem_mhz;
|
|
if c.vendor == "nvidia" {
|
|
c.power_limit_w = row.limit;
|
|
c.power_applied = true;
|
|
c.power_note = format!("power cap: {} W held by the tune (best MH per watt)", row.limit as u64);
|
|
}
|
|
c.sweep_note = if full_due { "tuning: a neighbour beat the fleet prior; the full tune runs next".into() } else { String::new() };
|
|
}
|
|
let e = settings.cards.entry(c.key.clone()).or_insert_with(|| crate::config::CardPref { enabled: c.enabled, identities: c.identities, ..Default::default() });
|
|
e.sweep_at = unix;
|
|
e.sweep_pct = row.point.power_pct;
|
|
e.sweep_eff = row.eff;
|
|
e.sweep_watts = row.watts;
|
|
e.sweep_mhs = row.mhs;
|
|
e.sweep_clock_mhz = row.point.clock_mhz;
|
|
e.sweep_mem_mhz = row.point.mem_mhz;
|
|
e.sweep_driver = c.driver.clone();
|
|
e.sweep_class = c.program_class.clone();
|
|
e.sweep_source = kind.name().into();
|
|
if !c.pinned && kind != crate::ember::PlanKind::Baseline {
|
|
e.power_pct = row.point.power_pct;
|
|
}
|
|
(c.name.clone(), c.pinned, control, c.vendor.clone(), c.driver.clone(), c.program_class.clone(), c.key.clone())
|
|
};
|
|
let _ = key;
|
|
self.shared.save_settings();
|
|
// 6 October 2026, run 6: the 5090's chosen clock sat on the ladder's floor (1,854 MHz = the old 60%), so a
|
|
// chosen point at the floor says so: it is the lowest step measured, not the optimum (Ember 2's climb walks on)
|
|
let floor = row.point.clock_mhz > 0 && row.point.clock_mhz <= run.plan.limits.clock_floor();
|
|
self.sweep_say(&format!("TUNE chosen card={} clock={} cap={} mem={} limit={:.0} watts={:.1} mhs={:.2} eff={:.4} plan={}{}{}", run.label, row.point.clock_mhz, row.point.power_pct, row.point.mem_mhz, row.limit, row.watts, row.mhs, row.eff, kind.name(), if pinned { " pinned=1" } else { "" }, if floor { " floor=1 note=floor,_not_optimum" } else { "" }));
|
|
let before = run.rows.first().filter(|_| kind == crate::ember::PlanKind::Full).cloned();
|
|
let record = crate::ember::record_json(crate::platform::unix_now_f(), &crate::config::fingerprint8(&self.shared.runtime.machine_id), VERSION, crate::manifest::platform_name(), &name, &vendor, &driver, &class, kind, &run.rows, Some(&row), before.as_ref(), floor);
|
|
self.sweep_say(&format!("TUNE {record}"));
|
|
let line = crate::ember::result_line(kind, row.mhs, row.watts, row.eff);
|
|
match kind {
|
|
crate::ember::PlanKind::Baseline => self.shared.event("info", &format!("{name}: {line}{}", if control { String::new() } else { " (measure only; no control on this card)".into() })),
|
|
_ if pinned => self.shared.event("ok", &format!("{name}: best point {} ({line}); your setting stays pinned", point_words(&row.point))),
|
|
_ => self.shared.event("ok", &format!("{name}: {line}, {} held", point_words(&row.point))),
|
|
}
|
|
if pinned && kind != crate::ember::PlanKind::Baseline {
|
|
let p = self.st().mining.cards.get(idx).map(|c| crate::ember::Point { clock_mhz: c.clock_cap_mhz, power_pct: c.power_pct, mem_mhz: c.mem_cap_mhz }).unwrap_or(run.before);
|
|
let w = self.st().mining.cards.get(idx).map(requested_watts).unwrap_or(run.before_w);
|
|
self.tune_apply(idx, &run.device, &crate::ember::Step { point: p, watts: w, kind: crate::ember::Kind::Confirm });
|
|
}
|
|
self.sweep_helper_quit();
|
|
if full_due && !pinned {
|
|
self.tune_full_due.insert(idx);
|
|
let k = self.st().mining.cards.get(idx).map(|c| c.key.clone());
|
|
if let Some(k) = k {
|
|
self.sweep_queue.push(k);
|
|
}
|
|
self.sweep_retry.insert(idx, Instant::now() + Duration::from_secs(60));
|
|
}
|
|
self.upload_logs(false);
|
|
if self.shared.runtime.sweep_only && self.sweep_queue.is_empty() {
|
|
self.shared.send(Cmd::Quit("the --sweep run (every card done)"));
|
|
}
|
|
}
|
|
|
|
/// Stops a tune: the point goes back to what it was, the log says why, the card waits an hour before the
|
|
/// scheduler tries again (a forced tune under --sweep retries twice).
|
|
fn sweep_abort(&mut self, why: &str) {
|
|
let pending = self.sweep_pending.take();
|
|
let run = self.sweep.take();
|
|
let (idx, label, device, before, before_w, forced, touched) = match (&run, pending) {
|
|
(Some(r), _) => (r.card, r.label.clone(), r.device.clone(), r.before, r.before_w, r.forced, r.seq > 0 && r.plan.kind != crate::ember::PlanKind::Baseline),
|
|
(None, Some((idx, forced))) => {
|
|
let c = self.st().mining.cards.get(idx).cloned();
|
|
let (label, device, w, p) = c.map(|c| (format!("card-{idx}"), c.device.clone(), if c.power_limit_w > 0.0 { c.power_limit_w } else { requested_watts(&c) }, crate::ember::Point { clock_mhz: c.clock_cap_mhz, power_pct: c.power_pct, mem_mhz: c.mem_cap_mhz })).unwrap_or_default();
|
|
(idx, label, device, p, w, forced, false)
|
|
}
|
|
_ => return,
|
|
};
|
|
self.sweep_say(&format!("TUNE aborted card={label} reason={}", why.replace(' ', "_")));
|
|
let name = {
|
|
let mut st = self.st();
|
|
match st.mining.cards.get_mut(idx) {
|
|
Some(c) => {
|
|
c.sweep_state = "idle".into();
|
|
if c.state == "tuning" {
|
|
c.state = "mining".into();
|
|
}
|
|
c.tune_step = 0;
|
|
c.tune_steps = 0;
|
|
c.tune_eta_s = 0;
|
|
c.sweep_note = format!("tuning stopped: {why}");
|
|
if touched {
|
|
c.power_pct = before.power_pct;
|
|
c.clock_cap_mhz = before.clock_mhz;
|
|
c.power_applied = false;
|
|
c.power_note = format!("cap restored to {} W after the tune stopped", before_w as u64);
|
|
}
|
|
c.name.clone()
|
|
}
|
|
None => label.clone(),
|
|
}
|
|
};
|
|
if touched && !device.is_empty() {
|
|
self.sweep = None;
|
|
self.tune_restore(idx, &device, before, before_w);
|
|
}
|
|
self.sweep_helper_quit();
|
|
self.shared.event(if touched { "error" } else { "info" }, &format!("{name}: tuning stopped ({why}){}", if touched { format!("; back to {} and {} W", point_words(&before), before_w as u64) } else { String::new() }));
|
|
let attempts = self.sweep_attempts.get(&idx).copied().unwrap_or(0);
|
|
if forced && self.shared.runtime.sweep_only && attempts < 3 && !self.quitting {
|
|
let key = self.st().mining.cards.get(idx).map(|c| c.key.clone());
|
|
if let Some(key) = key {
|
|
self.sweep_queue.push(key);
|
|
}
|
|
self.sweep_retry.insert(idx, Instant::now() + Duration::from_secs(30));
|
|
} else {
|
|
self.sweep_retry.insert(idx, Instant::now() + Duration::from_secs(3600));
|
|
if self.shared.runtime.sweep_only && self.sweep_queue.is_empty() {
|
|
self.shared.send(Cmd::Quit("the --sweep run (every card done)"));
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Puts a card back on a point outside a run (the abort path): the same commands as a step, with no run to
|
|
/// acknowledge them.
|
|
fn tune_restore(&mut self, idx: usize, device: &str, point: crate::ember::Point, watts: f64) {
|
|
let saved = self.sweep.take();
|
|
self.tune_apply(idx, device, &crate::ember::Step { point, watts, kind: crate::ember::Kind::Confirm });
|
|
self.sweep = saved;
|
|
}
|
|
|
|
// ---- the tick ----------------------------------------------------------------------------------------------
|
|
|
|
fn tick(&mut self) {
|
|
let now = Instant::now();
|
|
if now.duration_since(self.last_awake) >= Duration::from_secs(60) {
|
|
self.last_awake = now;
|
|
crate::platform::keep_awake_tick();
|
|
}
|
|
if self.running {
|
|
self.tick_node(now);
|
|
self.tick_watch(now);
|
|
self.tick_miners(now);
|
|
if let Some(at) = self.resume_check_at {
|
|
if now >= at {
|
|
self.resume_check_at = None;
|
|
let (cards, paused) = { let st = self.st(); (st.mining.cards.clone(), st.mining.paused) };
|
|
if !paused {
|
|
for line in resume_check(&cards) {
|
|
self.shared.log(&format!("resume: {line}"));
|
|
self.shared.event("error", &format!("resume: {line}"));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
self.tick_telemetry(now);
|
|
self.tick_sweep(now);
|
|
}
|
|
self.derive(now);
|
|
if now.duration_since(self.last_status) >= Duration::from_secs(self.shared.runtime.status_secs as u64) {
|
|
self.last_status = now;
|
|
if self.running {
|
|
self.status_line();
|
|
}
|
|
}
|
|
if now.duration_since(self.last_upload) >= Duration::from_secs(60) {
|
|
self.last_upload = now;
|
|
self.upload_logs(false);
|
|
}
|
|
self.tick_update();
|
|
self.tick_jobs();
|
|
if now >= self.clock_next_https {
|
|
self.command(Cmd::ClockCheck);
|
|
}
|
|
if self.clock_node_at.map(|t| now.duration_since(t) > Duration::from_secs(90)).unwrap_or(false) {
|
|
// the node stopped complaining: the clock (or the peers) changed
|
|
self.clock_node_at = None;
|
|
self.resolve_clock();
|
|
}
|
|
if self.wrapper && now.duration_since(self.last_state_print) >= Duration::from_secs(3) {
|
|
self.last_state_print = now;
|
|
println!("STATE {}", self.shared.wrapper_state());
|
|
let _ = std::io::stdout().flush();
|
|
}
|
|
if now.duration_since(self.last_settings_save) >= Duration::from_secs(120) {
|
|
self.last_settings_save = now;
|
|
self.shared.save_settings();
|
|
}
|
|
// hot-plug (src/hotplug.rs): enumerate again every POLL_S, and keep the card list in the log for the console
|
|
if self.detected && !self.detect_busy && !self.quitting && now >= self.detect_next {
|
|
self.detect_next = now + Duration::from_secs(crate::hotplug::POLL_S);
|
|
self.shared.send(Cmd::Detect);
|
|
}
|
|
if self.detected && now.duration_since(self.last_cards_line) >= Duration::from_secs(crate::hotplug::CARDS_LINE_S) {
|
|
self.last_cards_line = now;
|
|
let line = crate::hotplug::cards_line(&self.st().mining.cards);
|
|
self.shared.log(&line);
|
|
}
|
|
}
|
|
|
|
/// The over-the-air updater (src/ota.rs): checks, downloads and stages on its own threads; this tick hands it what
|
|
/// a safe moment needs and applies when it says so.
|
|
fn tick_update(&mut self) {
|
|
let ctx = {
|
|
let st = self.st();
|
|
crate::ota::Ctx {
|
|
node_synced: st.node.synced && st.clock.severity != "block",
|
|
boundary_eta_s: if st.node.daa > 0 && st.program.boundary_daa > 0 { Some(st.program.eta_s) } else { None },
|
|
// a remote job in progress counts as busy: no update applies under it (src/jobrun.rs)
|
|
miner_busy: self.miners.iter().any(|m| m.building) || st.mining.cards.iter().any(|c| c.enabled && c.state == "starting") || self.jobs.active(),
|
|
daa: st.node.daa,
|
|
}
|
|
};
|
|
// C35 (PC 1, 5 October 2026, 22:31 UTC): a second engine started by a measurement job found itself under the
|
|
// manifest's min_supported_version ("urgent" beats auto_update = false), ran the per-user installer, and the
|
|
// installer's PrepareToInstall quit the INSTALLED app through its api/quit. A second engine never updates.
|
|
if !self.no_ota {
|
|
if let Some(crate::ota::Action::Apply) = self.ota.tick(&self.shared, &ctx) {
|
|
self.apply_update();
|
|
}
|
|
}
|
|
if let Some(p) = self.ota.take_override_change() {
|
|
self.shared.log(&format!("consensus override changed ({}); the node restarts with it at a safe moment", p.display()));
|
|
self.node_override_restart = true;
|
|
}
|
|
if let Some(p) = self.ota.take_tuning_change() {
|
|
// no restart: a worker started before this reads the file at its next prepare only if it was started with
|
|
// the path, so a miner without it is restarted at the next hour boundary by the usual path (exit 42 or
|
|
// restart); a running worker that has the path picks the change up at the next prepare by itself
|
|
self.shared.log(&format!("kernel tuning changed ({}); the workers read it at their next hourly prepare", p.display()));
|
|
}
|
|
if self.node_override_restart && self.node.is_some() && !self.node_external {
|
|
// between hourly boundaries unless the switch is close
|
|
let (eta, daa) = { let st = self.st(); (st.program.eta_s, st.node.daa) };
|
|
let urgent = crate::manifest::fork_is_close(Some(self.ota.override_daa()).filter(|h| *h > 0), daa);
|
|
let safe = eta > crate::manifest::BOUNDARY_GUARD_S && eta < 3600 - crate::manifest::BOUNDARY_GUARD_S;
|
|
if urgent || safe || self.st().node.daa == 0 {
|
|
self.node_override_restart = false;
|
|
self.st().node.override_restart_wait = String::new();
|
|
self.shared.event("info", "restarting the node with the new consensus parameters");
|
|
self.stop_miners("consensus parameters changed");
|
|
self.stop_node();
|
|
self.start_node();
|
|
for m in self.miners.iter_mut() {
|
|
m.restart_at = Some(Instant::now() + Duration::from_secs(5));
|
|
}
|
|
} else {
|
|
self.st().node.override_restart_wait = format!("node restarts with the new consensus parameters after the hour boundary ({} s)", eta);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The remote-job runner (src/jobrun.rs): polls and runs on its own threads; this tick starts queued jobs and
|
|
/// does what a job asks of the engine (miners stopped and held, a restart, the updater).
|
|
fn tick_jobs(&mut self) {
|
|
if self.quitting {
|
|
return;
|
|
}
|
|
if let Some(a) = self.jobs.tick(&self.shared) {
|
|
self.job_action(a);
|
|
}
|
|
if self.job_hold && !self.jobs.holds_miners() {
|
|
self.job_hold = false;
|
|
for c in self.st().mining.cards.iter_mut().filter(|c| c.state == "held" || c.state == "tuning") {
|
|
c.state = "off".into();
|
|
c.sweep_state = if c.sweep_state == "running" { "idle".into() } else { c.sweep_state.clone() };
|
|
c.tune_step = 0;
|
|
c.tune_steps = 0;
|
|
c.tune_eta_s = 0;
|
|
}
|
|
if self.running {
|
|
self.shared.event("info", "job finished; the miners restart");
|
|
for m in self.miners.iter_mut() {
|
|
m.restart_at = Some(Instant::now());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn job_action(&mut self, a: crate::jobrun::Action) {
|
|
use crate::jobrun::Action;
|
|
match a {
|
|
Action::StopMiners(why) => {
|
|
self.job_hold = true;
|
|
self.stop_miners(&why);
|
|
for c in self.st().mining.cards.iter_mut().filter(|c| c.enabled) {
|
|
// never a bare "off" at 0 MH/s: the row says why (a job holds the card)
|
|
c.state = "held".into();
|
|
c.message = format!("held for a remote job: {}", short(&why, 80));
|
|
}
|
|
self.jobs.miners_stopped(&self.shared);
|
|
}
|
|
Action::CardsOff(keys) => {
|
|
// `--cards-off` (6 October 2026): the runner switches the job's cards through the app's own card path
|
|
// and keeps the exact choices that put them back; the script never touches /api/cards
|
|
let live: Vec<crate::jobrun::LiveCard> = self.st().mining.cards.iter().map(|c| (c.key.clone(), c.enabled, c.identities, c.power_pct, c.present())).collect();
|
|
let (off, restore) = crate::jobrun::cards_off_choices(&keys, &live);
|
|
if off.is_empty() {
|
|
self.shared.event("info", &format!("remote job asked for cards off ({}) but no present card matched; nothing switched", keys.join(",")));
|
|
} else {
|
|
self.shared.event("info", &format!("remote job: cards off for the job: {}", off.iter().map(|c| c.key.as_str()).collect::<Vec<_>>().join(", ")));
|
|
self.apply_cards(off);
|
|
}
|
|
if let Some(next) = self.jobs.cards_off_done(&self.shared, restore) {
|
|
self.job_action(next);
|
|
}
|
|
}
|
|
Action::RestoreCards(choices) => {
|
|
self.shared.event("info", &format!("remote job: cards restored: {}", choices.iter().map(|c| format!("{} enabled={} identities={}", c.key, c.enabled, c.identities)).collect::<Vec<_>>().join("; ")));
|
|
self.apply_cards(choices);
|
|
}
|
|
Action::RestartMiners => {
|
|
self.stop_miners("remote job: restart miners");
|
|
for m in self.miners.iter_mut() {
|
|
m.restart_at = Some(Instant::now());
|
|
}
|
|
}
|
|
Action::RestartNode => {
|
|
if self.node_external {
|
|
self.shared.event("info", "remote job asked for a node restart, but the node is external; nothing done");
|
|
} else {
|
|
self.restart_node("restart asked by a remote job", Duration::from_secs(3));
|
|
}
|
|
}
|
|
Action::RestartApp => {
|
|
self.shared.event("info", "remote job: the app restarts (miners stop, then the node, then it opens again)");
|
|
self.quitting = true;
|
|
self.st().quitting = true;
|
|
}
|
|
Action::UpdateNow => {
|
|
self.shared.event("info", "remote job: update check now; a newer version installs at once");
|
|
self.ota.install_now(&self.shared);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Hands over to the update helper, then leaves through the quit path (miners first, then the node, the last
|
|
/// log upload, EXIT for the window). The helper waits for this process to end before it swaps the app.
|
|
fn apply_update(&mut self) {
|
|
if self.quitting {
|
|
return;
|
|
}
|
|
let v = self.ota.version();
|
|
match self.ota.launch_apply(&self.shared, self.host_pid()) {
|
|
Ok(crate::ota::Launch::QuitNow) => {
|
|
{
|
|
let mut st = self.st();
|
|
st.update.applying = true;
|
|
st.update.status = "applying".into();
|
|
st.update.wait = String::new();
|
|
st.quitting = true;
|
|
}
|
|
self.shared.event("info", &format!("installing Igneum Miner {v}: the miners stop, then the node, then the app opens again"));
|
|
self.quitting = true;
|
|
}
|
|
Ok(crate::ota::Launch::InstallerRunning) => {
|
|
// Windows: the installer stops this engine itself (api/quit) once it may run; until then we mine
|
|
{
|
|
let mut st = self.st();
|
|
st.update.applying = true;
|
|
st.update.status = "applying".into();
|
|
st.update.wait = "the installer is starting; if Windows asks for permission the miners keep running until it is given".into();
|
|
}
|
|
self.shared.event("info", &format!("installing Igneum Miner {v}: the installer runs first; the miners keep running until it is allowed to, then it stops them, then the node, and the app opens again"));
|
|
}
|
|
Err(e) => {
|
|
self.shared.event("error", &format!("the update could not start: {e}"));
|
|
let mut st = self.st();
|
|
st.update.error = e;
|
|
st.update.status = "error".into();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The window host's pid when the engine runs under one (macOS: the helper asks it to quit).
|
|
fn host_pid(&self) -> u32 {
|
|
#[cfg(unix)]
|
|
{
|
|
if self.wrapper { unsafe { libc::getppid() as u32 } } else { 0 }
|
|
}
|
|
#[cfg(not(unix))]
|
|
{
|
|
0
|
|
}
|
|
}
|
|
|
|
fn tick_node(&mut self, now: Instant) {
|
|
if self.node_external {
|
|
if !port_open(self.shared.runtime.rpc_port) && now.duration_since(self.last_sync_check) >= Duration::from_secs(5) {
|
|
self.last_sync_check = now;
|
|
let mut st = self.st();
|
|
st.node.state = "stopped".into();
|
|
st.node.message = "the external node went away".into();
|
|
}
|
|
return;
|
|
}
|
|
if let Some(n) = self.node.as_mut() {
|
|
if !n.alive() {
|
|
let code = n.exit_code.unwrap_or(-1);
|
|
let ran = n.started.elapsed().as_secs();
|
|
let tail = tail_of(&n.log_path, 5);
|
|
self.node = None;
|
|
// an igneumd that does not know a field in the override file dies at once: run it without the file
|
|
// rather than loop (the app update that carries the newer node fixes it)
|
|
if ran < 10 && !self.node_override_unusable && tail.iter().any(|l| l.contains("override params file")) {
|
|
self.node_override_unusable = true;
|
|
self.shared.event("error", "this node build does not understand the consensus parameters in the signed manifest; starting it without them (an app update carries the newer node)");
|
|
self.st().node.override_restart_wait = "override not applied: the bundled node is too old for it".into();
|
|
self.start_node();
|
|
return;
|
|
}
|
|
if ran < 5 && self.node_starts == 1 {
|
|
// out at once: a port in use or an older database; say so and try once more, later
|
|
self.shared.event("error", &format!("igneumd exited at once (code {code}). {}", tail.last().cloned().unwrap_or_default()));
|
|
} else {
|
|
self.shared.event("error", &format!("igneumd exited with code {code} after {ran} s; restarting"));
|
|
}
|
|
for l in &tail {
|
|
self.shared.log(&format!(" node: {l}"));
|
|
}
|
|
self.node_restarts += 1;
|
|
let delay = jitter_secs(crate::platform::unix_now());
|
|
self.node_restart_at = Some(now + Duration::from_secs(delay));
|
|
let mut st = self.st();
|
|
st.node.state = "restarting".into();
|
|
st.node.synced = false;
|
|
st.node.restarts = self.node_restarts;
|
|
st.node.message = format!("restart in {delay} s");
|
|
drop(st);
|
|
let any = self.miners.iter().any(|m| m.proc.is_some());
|
|
if any {
|
|
self.shared.log("stopping the miners until the node is back and synced (their connection died with it)");
|
|
self.stop_miners("node restart");
|
|
}
|
|
}
|
|
}
|
|
if let Some(at) = self.node_restart_at {
|
|
if now >= at {
|
|
self.start_node();
|
|
} else {
|
|
self.st().node.restart_in_s = (at - now).as_secs();
|
|
}
|
|
}
|
|
}
|
|
|
|
fn tick_watch(&mut self, now: Instant) {
|
|
let node_up = self.node_external || self.node.is_some();
|
|
if let Some(w) = self.watch.as_mut() {
|
|
if !w.alive() {
|
|
self.watch = None;
|
|
self.watch_retry_at = now + Duration::from_secs(3);
|
|
}
|
|
}
|
|
if self.watch.is_none() && node_up && now >= self.watch_retry_at {
|
|
self.start_watch();
|
|
}
|
|
// no reading in a while: the node is away
|
|
let accepted_recent = self.last_accepted.map(|t| now.duration_since(t) <= Duration::from_secs(60)).unwrap_or(false);
|
|
if let Some(t) = self.node_last_reading {
|
|
if now.duration_since(t) > Duration::from_secs(45) && !accepted_recent {
|
|
let mut st = self.st();
|
|
if st.node.state == "synced" {
|
|
st.node.state = "syncing".into();
|
|
st.node.synced = false;
|
|
st.node.message = "no reading from the node for 45 s".into();
|
|
}
|
|
}
|
|
} else if node_up && !self.node_external && self.node_started_at.elapsed() > Duration::from_secs(60) {
|
|
let mut st = self.st();
|
|
if st.node.state == "starting" {
|
|
st.node.message = "no answer from the RPC yet (still opening its database?)".into();
|
|
}
|
|
}
|
|
// the node watchdog: our node silent for 120 s (no reading, nothing accepted) is restarted in-process
|
|
if self.node.is_some() && !self.node_external && self.node_restart_at.is_none() {
|
|
let silent_s = self.node_last_reading.map(|t| now.duration_since(t)).unwrap_or_else(|| self.node_started_at.elapsed()).as_secs_f64();
|
|
if let Some(delay) = self.node_watch.tick(self.secs(now), true, silent_s, accepted_recent) {
|
|
let n = self.node_watch.restarts_in_window();
|
|
self.shared.event("error", &format!("no reading from the node for {} s; the watchdog restarts it in {delay} s (restart {n} in the last 10 minutes)", silent_s as u64));
|
|
self.restart_node("no reading from the node for 120 s", Duration::from_secs(delay));
|
|
}
|
|
}
|
|
}
|
|
|
|
fn tick_miners(&mut self, now: Instant) {
|
|
// a clock over the consensus bound: the node cannot sync and a miner would only submit rejected blocks
|
|
let synced = self.st().node.synced && self.st().clock.severity != "block";
|
|
let paused = self.st().mining.paused;
|
|
for i in 0..self.miners.len() {
|
|
let card_idx = self.miners[i].card;
|
|
if let Some(p) = self.miners[i].proc.as_mut() {
|
|
if !p.alive() {
|
|
let code = p.exit_code.unwrap_or(-1);
|
|
let tail = tail_of(&p.log_path, 3);
|
|
let long_tail = if code == crate::watchdog::PACK_OUT_OF_DATE_CODE { tail_of(&p.log_path, 40) } else { Vec::new() };
|
|
self.miners[i].proc = None;
|
|
let t = self.secs(now);
|
|
let verdict = self.miners[i].watch.event(t, crate::watchdog::Event::Exited(code));
|
|
if code == 42 {
|
|
self.shared.event("build", "the hourly program changed; the worker restarts on the new one");
|
|
if self.bins_worker_missing(card_idx) {
|
|
self.miners[i].needs_rebuild = true;
|
|
}
|
|
self.miners[i].restart_at = Some(now);
|
|
} else if code == crate::watchdog::PACK_OUT_OF_DATE_CODE {
|
|
// The worker refused its program pack and the miner could not rebuild it (or hit its own
|
|
// cap): export the pack from the node again before the next start, not a blind restart;
|
|
// at most PACK_REBUILD_CAP times per epoch, then the card shows the reason
|
|
let why = long_tail.iter().rev().find_map(|l| crate::watchdog::pack_refusal(l)).unwrap_or_else(|| "the worker refused its program pack".into());
|
|
let epoch = std::fs::read_to_string(self.shared.runtime.app_dir.join("packs").join("devnet").join("seeds.txt")).ok().and_then(|s| crate::watchdog::pack_epoch_of(&s)).unwrap_or_default();
|
|
let name = self.st().mining.cards.get(card_idx).map(|c| c.name.clone()).unwrap_or_else(|| self.miners[i].label.clone());
|
|
match self.miners[i].pack_rebuilds.decide(&epoch, &why) {
|
|
crate::watchdog::PackAction::Rebuild { n, cap } => {
|
|
self.shared.event("build", "program pack out of date, rebuilding");
|
|
self.shared.log(&format!("{name}: program pack out of date, rebuilding (export {n} of {cap} for epoch {epoch}): {why}"));
|
|
self.miners[i].prepared = false; // prepare_worker exports the pack again before the start
|
|
self.miners[i].restart_at = Some(now);
|
|
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
|
c.state = "restarting".into();
|
|
c.hash_now = 0.0;
|
|
c.message = "program pack out of date, rebuilding".into();
|
|
}
|
|
}
|
|
crate::watchdog::PackAction::GiveUp { n: _, reason } => {
|
|
self.shared.event("error", &format!("{name}: {reason}"));
|
|
self.shared.log(&format!("{name}: {reason}; next try in 10 minutes or at the next hour"));
|
|
self.miners[i].prepared = false;
|
|
self.miners[i].restart_at = Some(now + Duration::from_secs(600));
|
|
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
|
c.state = "failed".into();
|
|
c.hash_now = 0.0;
|
|
c.message = reason;
|
|
}
|
|
}
|
|
}
|
|
} else if verdict != crate::watchdog::Action::None {
|
|
// exit 43: the miner gave up on its worker; once more, then the card is faulted
|
|
self.watchdog_verdict(i, verdict, &tail);
|
|
} else {
|
|
self.miners[i].restarts += 1;
|
|
let delay = jitter_secs(crate::platform::unix_now() + i as u64);
|
|
self.miners[i].restart_at = Some(now + Duration::from_secs(delay));
|
|
self.shared.event("error", &format!("miner {} exited with code {code}; restarting in {delay} s", self.miners[i].label));
|
|
for l in &tail {
|
|
self.shared.log(&format!(" miner: {l}"));
|
|
}
|
|
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
|
c.state = "restarting".into();
|
|
c.restarts = self.miners[i].restarts;
|
|
c.hash_now = 0.0;
|
|
c.message = tail.last().cloned().unwrap_or_default();
|
|
}
|
|
}
|
|
} else {
|
|
// the watchdog: no status for 90 s, or a zero rate for 60 s while synced, is one restart, then faulted
|
|
let t = self.secs(now);
|
|
let verdict = self.miners[i].watch.tick(t, synced);
|
|
if verdict != crate::watchdog::Action::None {
|
|
if let Some(mut p) = self.miners[i].proc.take() {
|
|
p.write_stdin("quit\n");
|
|
p.stop(3);
|
|
}
|
|
self.watchdog_verdict(i, verdict, &[]);
|
|
continue;
|
|
}
|
|
if let Some(t) = self.miners[i].last_status {
|
|
if now.duration_since(t) > Duration::from_secs(self.shared.runtime.status_secs as u64 * 4) {
|
|
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
|
if c.state == "mining" {
|
|
c.message = "no status from the miner for a while".into();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
if self.miners[i].watch.faulted().is_some() {
|
|
// faulted by the watchdog: not restarted until the user changes the card's settings or resumes
|
|
continue;
|
|
}
|
|
if self.job_hold {
|
|
// a remote job has the GPU; the miners wait until it lets go (src/jobrun.rs)
|
|
continue;
|
|
}
|
|
if paused || !synced {
|
|
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
|
if !paused && c.state != "failed" {
|
|
c.state = "waiting".into();
|
|
c.message = "waiting for the node to sync".into();
|
|
}
|
|
}
|
|
continue;
|
|
}
|
|
if self.miners[i].building {
|
|
continue;
|
|
}
|
|
if let Some(at) = self.miners[i].restart_at {
|
|
if now >= at {
|
|
let aot = self.st().mining.cards.get(card_idx).map(|c| c.worker != "Metal").unwrap_or(false);
|
|
if (aot || self.miners[i].needs_rebuild) && !self.miners[i].prepared {
|
|
self.prepare_worker(i);
|
|
} else {
|
|
self.start_miner(i);
|
|
}
|
|
} else if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
|
c.restart_in_s = (at - now).as_secs();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The strip, the log and the card for a program pack the worker refused or the miner found stale: the miner
|
|
/// rebuilds the pack and restarts the worker itself (or exits 44 for the app to export it). One strip line per
|
|
/// 30 s: the miner prints the refusal on stderr and its own line on stdout.
|
|
fn pack_notice(&mut self, i: usize, card: usize, card_name: &str, why: &str) {
|
|
let now = Instant::now();
|
|
self.shared.log(&format!("{card_name}: program pack out of date, rebuilding: {why}"));
|
|
if now.duration_since(self.last_error_event) >= Duration::from_secs(30) {
|
|
self.last_error_event = now;
|
|
self.shared.event("build", "program pack out of date, rebuilding");
|
|
}
|
|
self.miners[i].error_at = Some(now);
|
|
let t = self.secs(now);
|
|
self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart("program pack out of date, rebuilding"));
|
|
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
|
c.hash_now = 0.0;
|
|
c.message = "program pack out of date, rebuilding".into();
|
|
}
|
|
}
|
|
|
|
/// Applies a watchdog verdict to slot `i` (its process already stopped or gone): a restart now with the reason on
|
|
/// the card, or the card marked faulted with the reason in the UI and the log while the other cards keep mining.
|
|
fn watchdog_verdict(&mut self, i: usize, verdict: crate::watchdog::Action, tail: &[String]) {
|
|
use crate::watchdog::Action;
|
|
let card_idx = self.miners[i].card;
|
|
let name = self.st().mining.cards.get(card_idx).map(|c| c.name.clone()).unwrap_or_else(|| self.miners[i].label.clone());
|
|
for l in tail {
|
|
self.shared.log(&format!(" miner: {l}"));
|
|
}
|
|
match verdict {
|
|
Action::Restart(reason) => {
|
|
self.miners[i].restarts += 1;
|
|
self.miners[i].restart_at = Some(Instant::now() + Duration::from_secs(1));
|
|
self.shared.event("error", &format!("{name}: {reason}; the watchdog restarts the miner (restart {} of 1 before the card is marked faulted)", self.miners[i].watch.watchdog_restarts()));
|
|
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
|
c.state = "restarting".into();
|
|
c.restarts = self.miners[i].restarts;
|
|
c.hash_now = 0.0;
|
|
c.pid = 0;
|
|
c.message = format!("watchdog: {reason}");
|
|
}
|
|
}
|
|
Action::Fault(reason) => {
|
|
self.miners[i].restart_at = None;
|
|
self.shared.event("error", &format!("{name}: {reason}; the card is marked faulted and its miner is not restarted again (the other cards keep mining; change the card's settings or resume mining to try again)"));
|
|
if let Some(c) = self.st().mining.cards.get_mut(card_idx) {
|
|
c.state = "faulted".into();
|
|
c.hash_now = 0.0;
|
|
c.pid = 0;
|
|
c.restart_in_s = 0;
|
|
c.message = format!("faulted: {reason}");
|
|
}
|
|
}
|
|
Action::None => {}
|
|
}
|
|
}
|
|
|
|
/// Stops the miners and the node and schedules the node's start after `delay`; the miners follow once it is
|
|
/// synced. The remote-job restart kind and the node watchdog share it.
|
|
fn restart_node(&mut self, why: &str, delay: Duration) {
|
|
self.stop_miners(why);
|
|
self.stop_node();
|
|
self.node_restart_at = Some(Instant::now() + delay);
|
|
for m in self.miners.iter_mut() {
|
|
m.restart_at = Some(Instant::now());
|
|
}
|
|
let mut st = self.st();
|
|
st.node.state = "restarting".into();
|
|
st.node.synced = false;
|
|
st.node.message = format!("{why}; restart in {} s", delay.as_secs());
|
|
}
|
|
|
|
fn bins_worker_missing(&self, card_idx: usize) -> bool {
|
|
let st = self.st();
|
|
match st.mining.cards.get(card_idx).map(|c| c.worker.as_str()) {
|
|
Some("Metal") => self.bins.metal.is_none(),
|
|
Some("CUDA") => self.bins.cuda.is_none(),
|
|
Some(_) => self.bins.opencl.is_none(),
|
|
None => false,
|
|
}
|
|
}
|
|
|
|
/// Derived fields: ages, the hash total, the program countdown, the finality age, the mining state word.
|
|
fn derive(&mut self, now: Instant) {
|
|
let unix = crate::platform::unix_now_f();
|
|
let mut st = self.st();
|
|
st.node.last_reading_age_s = self.node_last_reading.map(|t| now.duration_since(t).as_secs_f64()).unwrap_or(-1.0);
|
|
let mut total = 0.0;
|
|
for c in st.mining.cards.iter_mut() {
|
|
if c.state == "mining" {
|
|
total += c.hash_now;
|
|
}
|
|
// live hash per watt (the number next to the draw on the tile); needs a draw reading under a minute old
|
|
c.eff_mhw = if c.state == "mining" && c.power_w > 1.0 && c.hash_now > 0.0 && unix - c.telemetry_at < 60.0 { c.hash_now / c.power_w } else { 0.0 };
|
|
}
|
|
st.mining.hash_total = total;
|
|
crate::hotplug::age(&mut st.mining.cards, unix);
|
|
let cut = unix - 3600.0;
|
|
st.mining.found.retain(|t| *t > cut);
|
|
if st.node.daa > 0 {
|
|
let computed = (st.node.daa / POW_EPOCH_BLOCKS + 1) * POW_EPOCH_BLOCKS;
|
|
let boundary = if st.program.boundary_daa > st.node.daa { st.program.boundary_daa } else { computed };
|
|
st.program.boundary_daa = boundary;
|
|
st.program.eta_s = boundary as i64 - st.node.daa as i64;
|
|
st.program.epoch_index = st.node.daa / POW_EPOCH_BLOCKS;
|
|
st.program.message = if st.program.prepared {
|
|
"next program compiled and ready".into()
|
|
} else if st.program.prepare_sent {
|
|
"next program compiling".into()
|
|
} else {
|
|
"next program not known yet".into()
|
|
};
|
|
}
|
|
if st.finality.last_lock > 0 {
|
|
st.finality.age_s = unix - st.finality.last_lock_at;
|
|
st.finality.message = String::new();
|
|
}
|
|
if self.running {
|
|
let mining_now = st.mining.cards.iter().any(|c| c.state == "mining");
|
|
let any_slot = !self.miners.is_empty();
|
|
st.mining.state = if st.mining.paused {
|
|
"paused".into()
|
|
} else if mining_now {
|
|
"mining".into()
|
|
} else if !any_slot {
|
|
if self.detected { "idle".into() } else { "waiting".into() }
|
|
} else {
|
|
"waiting".into()
|
|
};
|
|
}
|
|
}
|
|
|
|
// ---- lines -------------------------------------------------------------------------------------------------
|
|
|
|
fn line(&mut self, l: Line) {
|
|
let tag = l.src.tag();
|
|
self.shared.rings.lock().unwrap().log(&tag, l.stderr, &l.text);
|
|
match l.src {
|
|
Source::Watch => {
|
|
if l.text.contains(" node1 blocks=") {
|
|
self.watch_line(&l.text);
|
|
}
|
|
}
|
|
Source::Miner(card) => self.miner_line(card, l.stderr, &l.text),
|
|
Source::Node => self.node_line(&l.text),
|
|
Source::Telemetry => {
|
|
if !l.stderr {
|
|
self.telemetry_line(&l.text);
|
|
}
|
|
}
|
|
Source::AmdTelemetry => {
|
|
if !l.stderr {
|
|
self.amd_telemetry_line(&l.text);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// igneumd's own lines. One matters for the user: relayed blocks refused as "too far into the future", which is
|
|
/// this machine's clock running behind the network (PC 2, 4 October 2026: 60 s slow after a power cut, 0 blocks).
|
|
fn node_line(&mut self, text: &str) {
|
|
if let Some(d) = digest_from_line(text) {
|
|
let mut st = self.st();
|
|
if st.node.consensus_digest != d {
|
|
st.node.consensus_digest = d;
|
|
}
|
|
return;
|
|
}
|
|
if !text.contains("too far into the future") {
|
|
return;
|
|
}
|
|
if let Some(behind) = behind_from_warning(text) {
|
|
self.clock_node_behind = behind;
|
|
}
|
|
let first = self.clock_node_at.is_none();
|
|
self.clock_node_at = Some(Instant::now());
|
|
if first {
|
|
self.shared.log(&format!("node: relayed blocks refused as too far in the future: this clock is at least {:.0} s behind the network", self.clock_node_behind));
|
|
}
|
|
self.resolve_clock();
|
|
}
|
|
|
|
/// The worst of the three clock sources decides. skew = local minus network; behind = negative.
|
|
fn resolve_clock(&mut self) {
|
|
let now = Instant::now();
|
|
let mut skew: Option<f64> = None;
|
|
let mut source = "";
|
|
// 1. the node's own refusal: a lower bound on how far behind we are
|
|
if let Some(t) = self.clock_node_at {
|
|
if now.duration_since(t) <= Duration::from_secs(90) {
|
|
skew = Some(-self.clock_node_behind.max(10.0));
|
|
source = "node";
|
|
}
|
|
}
|
|
// 2. the median of local-minus-block-time over the last samples: behind only (a stalled chain reads as ahead)
|
|
if self.clock_samples.len() >= 3 {
|
|
let mut v = self.clock_samples.clone();
|
|
v.sort_by(|a, b| a.partial_cmp(b).unwrap());
|
|
let median = v[v.len() / 2];
|
|
if median < -5.0 && skew.map(|s| median < s).unwrap_or(true) {
|
|
skew = Some(median);
|
|
source = "blocks";
|
|
}
|
|
}
|
|
// 3. the HTTPS Date header: either direction, 1 s resolution plus the request latency
|
|
if let Some((d, t)) = self.clock_https {
|
|
if now.duration_since(t) <= Duration::from_secs(1200) && d.abs() > 5.0 && skew.map(|s| d.abs() > s.abs()).unwrap_or(true) {
|
|
skew = Some(d);
|
|
source = "https";
|
|
}
|
|
}
|
|
let mut st = self.st();
|
|
let was = st.clock.severity.clone();
|
|
st.clock.checked_at = crate::platform::unix_now_f();
|
|
match skew {
|
|
Some(d) if d.abs() > 10.0 => {
|
|
st.clock.severity = "block".into();
|
|
st.clock.skew_s = d;
|
|
st.clock.source = source.into();
|
|
let n = d.abs().round() as i64;
|
|
st.clock.message = if d < 0.0 {
|
|
format!("Your clock is about {n} seconds behind the network{}; mining cannot start until it is fixed.", if source == "node" { " (at least)" } else { "" })
|
|
} else {
|
|
format!("Your clock is about {n} seconds ahead of the network; mining cannot start until it is fixed.")
|
|
};
|
|
}
|
|
Some(d) => {
|
|
st.clock.severity = "warn".into();
|
|
st.clock.skew_s = d;
|
|
st.clock.source = source.into();
|
|
let n = d.abs().round() as i64;
|
|
st.clock.message = format!("Your clock is about {n} seconds {} the network. Over 10 s the node refuses blocks.", if d < 0.0 { "behind" } else { "ahead of" });
|
|
}
|
|
None => {
|
|
st.clock.severity = "none".into();
|
|
st.clock.skew_s = 0.0;
|
|
st.clock.source = String::new();
|
|
st.clock.message = String::new();
|
|
}
|
|
}
|
|
let is = st.clock.severity.clone();
|
|
let msg = st.clock.message.clone();
|
|
drop(st);
|
|
if is != was {
|
|
match is.as_str() {
|
|
"block" => {
|
|
self.shared.event("error", &msg);
|
|
if self.miners.iter().any(|m| m.proc.is_some()) {
|
|
self.stop_miners("clock over the consensus bound");
|
|
for m in self.miners.iter_mut() {
|
|
m.restart_at = Some(Instant::now());
|
|
}
|
|
}
|
|
}
|
|
"warn" => self.shared.event("info", &msg),
|
|
_ => self.shared.event("ok", "clock agrees with the network again"),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn watch_line(&mut self, text: &str) {
|
|
let now = Instant::now();
|
|
self.node_last_reading = Some(now);
|
|
let blocks = kv_u64(text, "blocks").unwrap_or(0);
|
|
let headers = kv_u64(text, "headers").unwrap_or(0);
|
|
let daa = kv_u64(text, "daa").unwrap_or(0);
|
|
let peers = kv_u64(text, "peers").unwrap_or(0);
|
|
let tips = kv_u64(text, "tips").unwrap_or(0);
|
|
let blue = kv_u64(text, "blue").unwrap_or(0);
|
|
let difficulty = kv_f64(text, "difficulty").unwrap_or(0.0);
|
|
let flag = kv(text, "synced").map(|s| s == "true");
|
|
let was_synced = self.st().node.synced;
|
|
// Derived from every reading, never a one-shot transition (PC 2, 4 October 2026: the node caught up after a
|
|
// clock fix and mined at 118 MH/s while the card still said "syncing", because getInfo's flag stayed false).
|
|
let caught_up_moving = self.sync_prev.map(|p| blocks > p).unwrap_or(false);
|
|
if peers > 0 && headers <= blocks + 2 && blocks > 1 {
|
|
self.sync_stable_since.get_or_insert(now);
|
|
} else {
|
|
self.sync_stable_since = None;
|
|
}
|
|
let stable_s = self.sync_stable_since.map(|t| now.duration_since(t).as_secs_f64()).unwrap_or(0.0);
|
|
let accepted_recent = self.last_accepted.map(|t| now.duration_since(t) <= Duration::from_secs(60)).unwrap_or(false);
|
|
let private = self.shared.runtime.unsynced_mining && self.shared.runtime.peers.is_empty();
|
|
let synced = sync_decision(&Reading { blocks, headers, peers, flag }, caught_up_moving, stable_s, accepted_recent, private);
|
|
self.sync_prev = Some(blocks);
|
|
// the first clock source: local time against the latest block the peers produced, once blocks arrive
|
|
if blocks > 0 && (peers > 0 || self.shared.runtime.peers.is_empty()) && now.duration_since(self.clock_last_sample) >= Duration::from_secs(9) {
|
|
self.clock_last_sample = now;
|
|
let port = self.shared.runtime.evm_port();
|
|
let shared = self.shared.clone();
|
|
std::thread::spawn(move || {
|
|
if let Some(t) = crate::update::latest_block_time(port) {
|
|
shared.send(Cmd::ClockSample(crate::platform::unix_now_f() - t));
|
|
}
|
|
});
|
|
}
|
|
{
|
|
let mut st = self.st();
|
|
st.node.blocks = blocks;
|
|
st.node.headers = headers;
|
|
st.node.daa = daa;
|
|
st.node.peers = peers;
|
|
st.node.tips = tips;
|
|
st.node.blue = blue;
|
|
st.node.difficulty = difficulty;
|
|
st.node.synced = synced;
|
|
st.node.state = if synced { "synced".into() } else { "syncing".into() };
|
|
st.node.message = if synced {
|
|
String::new()
|
|
} else if peers == 0 {
|
|
"waiting for a peer".into()
|
|
} else if headers > blocks + 2 {
|
|
format!("{blocks} of {headers} blocks")
|
|
} else {
|
|
"caught up, waiting for the next block".into()
|
|
};
|
|
}
|
|
if synced && !was_synced {
|
|
self.shared.event("ok", &format!("node synced: {blocks} blocks, {peers} peer(s)"));
|
|
for m in self.miners.iter_mut() {
|
|
if m.proc.is_none() && m.restart_at.is_none() && m.watch.faulted().is_none() {
|
|
m.restart_at = Some(now);
|
|
}
|
|
}
|
|
}
|
|
if !synced && !self.no_peer_warned && peers == 0 && self.node_started_at.elapsed() >= Duration::from_secs(90) && !self.shared.runtime.peers.is_empty() {
|
|
self.no_peer_warned = true;
|
|
self.shared.event("error", &format!("no peer after 90 s: is the seed node reachable ({})? The node keeps retrying", self.shared.runtime.peers.join(", ")));
|
|
}
|
|
}
|
|
|
|
fn miner_line(&mut self, card: usize, stderr: bool, text: &str) {
|
|
let Some(i) = self.miners.iter().position(|m| m.card == card) else { return };
|
|
let label = self.miners[i].label.clone();
|
|
let card_name = self.st().mining.cards.get(card).map(|c| c.name.clone()).unwrap_or_else(|| label.clone());
|
|
if stderr {
|
|
if text.contains(" rejected nonce=") {
|
|
if let Some(run) = self.sweep.as_mut() {
|
|
if run.card == card {
|
|
run.sample_fault();
|
|
}
|
|
}
|
|
let mut st = self.st();
|
|
st.mining.rejected_session += 1;
|
|
if let Some(c) = st.mining.cards.get_mut(card) {
|
|
c.rejected += 1;
|
|
}
|
|
} else if text.starts_with("template error") || text.contains(" template error") {
|
|
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
|
c.message = "the node is not answering; the miner retries".into();
|
|
}
|
|
} else if let Some(why) = crate::watchdog::pack_refusal(text) {
|
|
// the worker refused its program pack; the miner rebuilds the pack and restarts the worker itself
|
|
// (or exits 44 for us to export it): the strip and the card name the condition in plain words
|
|
self.pack_notice(i, card, &card_name, &why);
|
|
} else if text.contains("WORKER MISMATCH") || text.contains("worker error") || text.contains("worker exited") || text.contains("worker killed by a guard") || text.contains("panicked") || text.contains("CUDA error") || text.contains("submit error") {
|
|
let now = Instant::now();
|
|
if now.duration_since(self.last_error_event) >= Duration::from_secs(30) {
|
|
self.last_error_event = now;
|
|
self.shared.event("error", &format!("{card_name}: {}", short(text, 160)));
|
|
}
|
|
self.miners[i].error_at = Some(now);
|
|
if text.contains("worker exited") || text.contains("worker killed by a guard") {
|
|
// the miner restarts its worker itself: the watchdog waits for `ready` instead of restarting the miner too
|
|
let t = self.secs(now);
|
|
let reason = short(text.split_once(' ').map(|(_, r)| r).unwrap_or(text), 160);
|
|
self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart(&reason));
|
|
}
|
|
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
|
// the restart note never hides the fault reason the card already shows
|
|
if !(c.message.starts_with("worker fault: ") && (text.contains("worker killed by a guard") || text.contains("worker exited"))) {
|
|
c.message = short(text, 160);
|
|
}
|
|
}
|
|
}
|
|
return;
|
|
}
|
|
if text.contains("PACK OUT OF DATE") {
|
|
// the miner found its pack stale or refused (stdout): it rebuilds the pack before the restart
|
|
let why = crate::watchdog::pack_refusal(text).unwrap_or_else(|| short(text, 160));
|
|
self.pack_notice(i, card, &card_name, &why);
|
|
return;
|
|
}
|
|
if text.contains(" WORKER FAULT ") {
|
|
// the miner's guards (interval, job time, cpu re-check, stall) killed the worker; it restarts it itself
|
|
let reason = crate::watchdog::fault_reason(text).unwrap_or_else(|| short(text, 160));
|
|
let t = self.secs(Instant::now());
|
|
self.miners[i].watch.event(t, crate::watchdog::Event::WorkerRestart(&reason));
|
|
self.shared.event("error", &format!("{card_name}: worker fault: {}; the miner restarts the worker", short(&reason, 200)));
|
|
if let Some(c) = self.st().mining.cards.get_mut(card) {
|
|
c.faults += 1;
|
|
c.hash_now = 0.0;
|
|
c.message = format!("worker fault: {}", short(&reason, 160));
|
|
}
|
|
return;
|
|
}
|
|
if text.contains(" ACCEPTED block") {
|
|
self.last_accepted = Some(Instant::now());
|
|
{
|
|
// the node accepted our block: it is at the tip, whatever its flag says
|
|
let mut st = self.st();
|
|
if !st.node.synced {
|
|
st.node.synced = true;
|
|
st.node.state = "synced".into();
|
|
st.node.message = String::new();
|
|
}
|
|
}
|
|
let unix = crate::platform::unix_now_f();
|
|
let total = {
|
|
let mut s = self.shared.settings.lock().unwrap();
|
|
s.accepted_total += 1;
|
|
s.accepted_total
|
|
};
|
|
let mut st = self.st();
|
|
st.mining.accepted_session += 1;
|
|
st.mining.accepted_total = total;
|
|
st.mining.found.push(unix);
|
|
if let Some(c) = st.mining.cards.get_mut(card) {
|
|
c.accepted += 1;
|
|
}
|
|
drop(st);
|
|
self.shared.event("block", &format!("block accepted by the node ({card_name})"));
|
|
} else if let Some(s) = crate::watchdog::parse_status(text) {
|
|
let now = Instant::now();
|
|
self.miners[i].last_status = Some(now);
|
|
let t = self.secs(now);
|
|
self.miners[i].watch.event(t, crate::watchdog::Event::Status(&s));
|
|
let avg = kv_f64(text, "hash").unwrap_or(0.0);
|
|
// the v4 miner's interval rate: "now=12.34 MH/s wall (...)"; older miners: the average
|
|
let now_rate = if s.hash_now > 0.0 { s.hash_now } else { avg };
|
|
if let Some(run) = self.sweep.as_mut() {
|
|
if run.card == card {
|
|
run.sample_rate(now_rate);
|
|
}
|
|
}
|
|
let more_mismatched = self.st().mining.cards.get(card).map(|c| s.mismatched > c.mismatched).unwrap_or(false);
|
|
if more_mismatched {
|
|
if let Some(run) = self.sweep.as_mut() {
|
|
if run.card == card {
|
|
run.sample_fault();
|
|
}
|
|
}
|
|
}
|
|
let mut st = self.st();
|
|
if let Some(c) = st.mining.cards.get_mut(card) {
|
|
c.hash_avg = avg;
|
|
// the v4 miner's interval rate: "now=12.34 MH/s wall (...)"; older miners: the average
|
|
c.hash_now = s.hash_now;
|
|
c.template_age_s = kv_f64(text, "template_age").unwrap_or(0.0);
|
|
c.synced = s.synced;
|
|
c.mismatched = s.mismatched;
|
|
c.faults = c.faults.max(s.faults);
|
|
c.last_status_age_s = 0.0;
|
|
if c.state == "starting" || c.state == "ready" {
|
|
c.state = "mining".into();
|
|
}
|
|
if c.state == "mining" && s.hash_now > 0.0 {
|
|
c.message = if s.mismatched > 0 { format!("{} share(s) failed the CPU re-check this run", s.mismatched) } else { String::new() };
|
|
}
|
|
}
|
|
} else if text.contains(" worker: race ") {
|
|
self.race_line(card, &card_name, text);
|
|
} else if text.contains(" worker: ready ") {
|
|
let prepare = text.contains(" prepare 1");
|
|
let t = self.secs(Instant::now());
|
|
self.miners[i].watch.event(t, crate::watchdog::Event::Ready);
|
|
let mut st = self.st();
|
|
if let Some(c) = st.mining.cards.get_mut(card) {
|
|
c.ready = true;
|
|
c.prepare = prepare;
|
|
c.state = "mining".into();
|
|
c.message = String::new();
|
|
}
|
|
drop(st);
|
|
if self.miners[i].starts == 1 {
|
|
self.shared.event("ok", &format!("{card_name} worker ready{}", if prepare { ", hot swap at the hour boundary" } else { ", restart at the hour boundary" }));
|
|
}
|
|
} else if text.contains(" worker: prepared ") {
|
|
let mut st = self.st();
|
|
st.program.prepared = true;
|
|
if let Some(c) = st.mining.cards.get_mut(card) {
|
|
c.prepared = true;
|
|
}
|
|
drop(st);
|
|
self.shared.event("build", "next hourly program compiled and ready");
|
|
} else if text.contains(" PREPARE sent for ") {
|
|
let mut st = self.st();
|
|
st.program.prepare_sent = true;
|
|
st.program.prepared = false;
|
|
if let Some(b) = text.split("boundary at ").nth(1).and_then(|r| r.split(|c: char| !c.is_ascii_digit()).next()).and_then(|n| n.parse::<u64>().ok()) {
|
|
st.program.boundary_daa = b;
|
|
}
|
|
} else if text.contains(" SEED CHANGE at daa ") {
|
|
let mut st = self.st();
|
|
st.program.prepared = false;
|
|
st.program.prepare_sent = false;
|
|
st.program.boundary_daa = 0;
|
|
for c in st.mining.cards.iter_mut() {
|
|
c.prepared = false;
|
|
}
|
|
drop(st);
|
|
let detail = text.rsplit("): ").next().unwrap_or("").to_string();
|
|
self.shared.event("build", &format!("hourly program changed: {}", if detail.is_empty() { "new program".to_string() } else { detail }));
|
|
} else if text.contains(" vote_key_hash=") {
|
|
if let Some(h) = kv(text, "vote_key_hash") {
|
|
let short_id: String = h.chars().take(8).collect();
|
|
let mut st = self.st();
|
|
if let Some(c) = st.mining.cards.get_mut(card) {
|
|
if !c.ids.contains(&short_id) {
|
|
c.ids.push(short_id);
|
|
}
|
|
}
|
|
}
|
|
} else if text.contains(" LOCK checkpoint ") {
|
|
if let Some(n) = text.split(" LOCK checkpoint ").nth(1).and_then(|r| r.split_whitespace().next()).and_then(|n| n.parse::<u64>().ok()) {
|
|
let mut st = self.st();
|
|
if n > st.finality.last_lock {
|
|
st.finality.last_lock = n;
|
|
st.finality.last_lock_at = crate::platform::unix_now_f();
|
|
st.finality.age_s = 0.0;
|
|
}
|
|
}
|
|
} else if text.contains(" VOTE index=") {
|
|
self.st().finality.votes += 1;
|
|
} else if text.contains("worker could not prepare") || text.contains(" prepare-failed ") {
|
|
self.st().program.prepared = false;
|
|
self.shared.event("error", &format!("{card_name}: the worker could not compile the next program; it compiles at the boundary"));
|
|
} else if text.contains("exiting with code 42") {
|
|
// expected; the exit handler restarts the miner
|
|
}
|
|
}
|
|
|
|
/// A worker's race line (one per hourly prepare, docs/design/miner-tuning.md):
|
|
/// `race <epoch16> device <name> driver <d> arch <a> loads <n> wide <n> variants <k> <name>=<MH/s>/<regs>r/<warps>w ...
|
|
/// winner <name> <MH/s> base <MH/s> gain <pct>% compile <ms> bench <ms> total <ms> ms [| <variant>: <why>]`.
|
|
/// Becomes the fleet record: one `TUNING {json}` line in the app log (uploaded to the intake, aggregated by
|
|
/// tools/tuning.mjs) with the card's power figures, plus the card state and one event.
|
|
fn race_line(&mut self, card: usize, card_name: &str, text: &str) {
|
|
let Some(body) = text.split(" worker: race ").nth(1) else { return };
|
|
let Some(r) = parse_race(body) else { return };
|
|
let (vendor, worker, power_limit_w, power_w, power_pct) = {
|
|
let mut st = self.st();
|
|
match st.mining.cards.get_mut(card) {
|
|
Some(c) => {
|
|
c.variant = r.winner.clone();
|
|
c.race_mhs = r.mhs;
|
|
c.race_gain_pct = r.gain_pct;
|
|
c.race_variants = r.variants.len() as u32;
|
|
if !r.driver.is_empty() && c.driver.is_empty() {
|
|
c.driver = r.driver.clone();
|
|
}
|
|
c.program_class = crate::ember::program_class(r.loads as u32, r.wide as u32);
|
|
(c.vendor.clone(), c.worker.clone(), c.power_limit_w, c.power_w, c.power_pct)
|
|
}
|
|
None => (String::new(), String::new(), 0.0, 0.0, 0),
|
|
}
|
|
};
|
|
let record = serde_json::json!({
|
|
"ts": crate::platform::unix_now_f().round(),
|
|
"machine": self.shared.runtime.id8(),
|
|
"app": VERSION,
|
|
"card": r.device,
|
|
"vendor": vendor,
|
|
"worker": worker,
|
|
"driver": r.driver,
|
|
"arch": r.arch,
|
|
"epoch": r.epoch,
|
|
"loads": r.loads,
|
|
"wide": r.wide,
|
|
"variants": r.variants,
|
|
"winner": r.winner,
|
|
"mhs": r.mhs,
|
|
"base_mhs": r.base_mhs,
|
|
"gain_pct": r.gain_pct,
|
|
"power_limit_w": power_limit_w,
|
|
"power_w": power_w,
|
|
"power_pct": power_pct,
|
|
"mh_per_w": if power_w > 0.0 { (r.mhs / power_w * 1000.0).round() / 1000.0 } else { 0.0 },
|
|
"total_ms": r.total_ms,
|
|
"pinned": r.pinned,
|
|
"tuned": r.tuned,
|
|
"notes": r.notes,
|
|
});
|
|
self.shared.log(&format!("TUNING {record}"));
|
|
if !r.winner.is_empty() {
|
|
self.shared.event("build", &format!("{card_name} kernel race: {} at {:.1} MH/s ({:+.1}% over base, {} variants, {:.0} s)", r.winner, r.mhs, r.gain_pct, r.variants.len(), r.total_ms / 1000.0));
|
|
}
|
|
}
|
|
|
|
// ---- status, uploads, shutdown ---------------------------------------------------------------------------
|
|
|
|
fn status_line(&mut self) {
|
|
let st = self.st();
|
|
let node_txt = match st.node.state.as_str() {
|
|
"synced" | "syncing" => format!("node {} blocks, {} peers, {}", st.node.blocks, st.node.peers, st.node.state),
|
|
s => format!("node {s}"),
|
|
};
|
|
let miner_txt = if st.mining.cards.is_empty() {
|
|
"no GPU miner".to_string()
|
|
} else {
|
|
format!("accepted {} blocks ({} this run, dev fee {}), {:.2} MH/s, {}", st.mining.accepted_total, st.mining.accepted_session, st.mining.fee_session, st.mining.hash_total, st.mining.state)
|
|
};
|
|
let up = fmt_uptime(self.shared.started.elapsed().as_secs());
|
|
drop(st);
|
|
self.shared.log(&format!("status: {miner_txt} | {node_txt} | up {up}"));
|
|
}
|
|
|
|
fn upload_logs(&mut self, wait: bool) {
|
|
let p = &self.shared.packaged;
|
|
if p.log_intake_url.is_empty() || p.log_intake_key.is_empty() || !self.running {
|
|
return;
|
|
}
|
|
if let Some(t) = self.upload_thread.take() {
|
|
if t.is_finished() || wait {
|
|
let _ = t.join();
|
|
} else {
|
|
self.upload_thread = Some(t);
|
|
return;
|
|
}
|
|
}
|
|
let mut files: Vec<(String, PathBuf)> = Vec::new();
|
|
files.push((self.label_base.clone(), self.shared.log_path.clone()));
|
|
if let Some(n) = &self.node_log {
|
|
files.push((format!("nodelog-{}", self.label_base), n.clone()));
|
|
}
|
|
for m in &self.miners {
|
|
if let Some(pr) = &m.proc {
|
|
files.push((format!("miner-{}", m.label), pr.log_path.clone()));
|
|
}
|
|
}
|
|
// machine = "<hostname>-<id8>": readable in tools/logs.mjs and distinct for cloned PCs
|
|
let (url, key, machine, run_id) = (p.log_intake_url.clone(), p.log_intake_key.clone(), format!("{}-{}", self.shared.runtime.host, self.shared.runtime.id8()), format!("{}-{}", self.label_base, self.stamp));
|
|
let header = self.shared.upload_header();
|
|
let t = std::thread::spawn(move || {
|
|
for (label, path) in files {
|
|
crate::update::upload_log(&url, &key, &label, &machine, &run_id, &path, &header);
|
|
}
|
|
});
|
|
if wait {
|
|
let _ = t.join();
|
|
} else {
|
|
self.upload_thread = Some(t);
|
|
}
|
|
}
|
|
|
|
fn shutdown(&mut self) {
|
|
self.shared.log(&format!("quit: stopping the miners, then the node (source: {})", self.quit_source));
|
|
if let Some(a) = self.jobs.abort(&self.shared, "the app is quitting") {
|
|
// the job's cards back before the exit (an app update under a job left the 9070 XT off on 6 October 2026)
|
|
self.job_action(a);
|
|
}
|
|
if self.sweep.is_some() || self.sweep_pending.is_some() {
|
|
self.sweep_abort("the app is quitting");
|
|
}
|
|
self.sweep_helper_quit();
|
|
self.stop_miners("quit");
|
|
self.stability_line();
|
|
if let Some(mut t) = self.telemetry.take() {
|
|
t.stop(2);
|
|
}
|
|
if let Some(mut t) = self.amd_telemetry.take() {
|
|
t.stop(2);
|
|
}
|
|
if self.shared.runtime.sweep_only {
|
|
// the measurement leaves the chosen cap in force for the app that takes the card back
|
|
self.shared.log("--sweep: the chosen caps stay in force (not restored)");
|
|
} else {
|
|
self.restore_power_limits();
|
|
}
|
|
if let Some(mut w) = self.watch.take() {
|
|
w.stop(2);
|
|
}
|
|
self.stop_node();
|
|
if let Some(mut k) = self.keep_awake.take() {
|
|
k.stop();
|
|
}
|
|
let st = self.st();
|
|
let summary = format!(
|
|
"SUMMARY after {}: node started {} time(s), restarts {}, {} accepted blocks this run ({} lifetime); data stays in {}",
|
|
fmt_uptime(self.shared.started.elapsed().as_secs()),
|
|
self.node_starts,
|
|
self.node_restarts,
|
|
st.mining.accepted_session,
|
|
st.mining.accepted_total,
|
|
self.shared.runtime.node_dir.display()
|
|
);
|
|
drop(st);
|
|
self.shared.log(&summary);
|
|
self.shared.save_settings();
|
|
self.upload_logs(true);
|
|
self.shared.log("stopped");
|
|
if self.wrapper {
|
|
println!("STATE {}", json!({ "phase": "quit", "quitting": true }));
|
|
println!("EXIT");
|
|
let _ = std::io::stdout().flush();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The watts a card's cap asks for: power_pct of the default limit, inside the card's min and max.
|
|
/// the founder, 5 October 2026: "if we don't have to ask then don't ask". The NVIDIA power cap and the efficiency sweep need
|
|
/// administrator rights (one UAC prompt on Windows, pkexec on Linux); the engine builds an elevated command only when
|
|
/// Power control is on in Settings, or when it is itself the elevated PC sweep job (--sweep).
|
|
fn elevation_allowed(power_control: bool, sweep_only: bool) -> bool {
|
|
// C35 (5 October 2026, 22:30 UTC): the unattended --sweep job on PC 1 counted as allowed and raised the one
|
|
// administrator prompt nobody was there to answer; an elevated job sets limits directly without asking (the tune
|
|
// probe's `direct`), so the flag adds nothing and Power control alone decides
|
|
let _ = sweep_only;
|
|
power_control
|
|
}
|
|
|
|
/// The notice when the one prompt was refused, cancelled or not answered: Power control goes back off, no retries.
|
|
pub const POWER_CONTROL_REFUSED: &str = "power control off: administrator rights were not given";
|
|
|
|
/// After the elevated step: Some(notice) when the administrator prompt was refused, cancelled or timed out (the
|
|
/// words platform::run_elevated and the window host use), None when rights were given, even if a card then
|
|
/// disagreed with the readback.
|
|
fn power_control_after_prompt(r: &Result<(), String>) -> Option<&'static str> {
|
|
match r {
|
|
Err(e) if e.contains("administrator prompt") || e.contains("refused") || e.contains("cancel") => Some(POWER_CONTROL_REFUSED),
|
|
_ => None,
|
|
}
|
|
}
|
|
|
|
/// The nvidia-smi -pl lines one elevated step runs, the human list of what they set, and how many cards were held
|
|
/// back because Power control is off (their note says so). Nothing is built when not allowed.
|
|
fn power_cap_plan(cards: &mut [CardState], allowed: bool, smi: &str) -> (Vec<String>, Vec<String>, usize) {
|
|
let mut cmds = Vec::new();
|
|
let mut what = Vec::new();
|
|
let mut held = 0;
|
|
for c in cards.iter_mut().filter(|c| c.vendor == "nvidia" && c.enabled && c.present() && c.power_default_w > 0.0) {
|
|
let pct = if c.power_pct == 0 { 80 } else { c.power_pct.clamp(crate::sweep::MIN_PCT, 100) };
|
|
c.power_pct = pct;
|
|
let watts = requested_watts(c);
|
|
if (c.power_limit_w - watts).abs() < 1.0 && c.power_applied {
|
|
continue;
|
|
}
|
|
if !allowed {
|
|
c.power_note = "power cap not set: Power control is off in Settings".into();
|
|
held += 1;
|
|
continue;
|
|
}
|
|
cmds.push(format!("\"{smi}\" -i {} -pl {}", c.device, watts as u64));
|
|
what.push(format!("{} {} W ({}% of {} W)", c.name, watts as u64, pct, c.power_default_w as u64));
|
|
c.power_note = "setting the power cap (administrator prompt)".into();
|
|
}
|
|
(cmds, what, held)
|
|
}
|
|
|
|
/// What a tune's probe found for a card (Cmd::TuneProbe).
|
|
#[derive(Clone, Debug, Default, PartialEq)]
|
|
pub struct TuneProbe {
|
|
pub clock_max_mhz: u32,
|
|
pub clock_min_mhz: u32,
|
|
pub driver: String,
|
|
/// NVIDIA: this process sets limits itself (elevated); AMD: the helper answered its `--tune` line with ok
|
|
pub direct: bool,
|
|
pub amd_ordinal: i64,
|
|
/// AMD: the power offset range in percent from the `tune` line (PC 1's 9070 XT: -30 to 10)
|
|
pub plimit_min: f64,
|
|
pub plimit_max: f64,
|
|
/// Ember 2, NVIDIA: the memory clock under load and the vendor's maximum (nvidia-smi clocks.mem, clocks.max.mem)
|
|
pub mem_default_mhz: u32,
|
|
pub mem_max_mhz: u32,
|
|
}
|
|
|
|
/// One `tune` line of igneum-gpu-telemetry --tune:
|
|
/// `tune <ordinal> name "<name>" gmax <MHz> gmax_range <min> <max> plimit <offset %> plimit_range <min> <max> factory 0|1 ok|<error>`.
|
|
#[derive(Clone, Debug, Default, PartialEq)]
|
|
pub struct AmdTune {
|
|
pub ordinal: usize,
|
|
pub name: String,
|
|
pub gmax: f64,
|
|
pub gmax_min: f64,
|
|
pub gmax_max: f64,
|
|
pub plimit: f64,
|
|
pub plimit_min: f64,
|
|
pub plimit_max: f64,
|
|
pub factory: bool,
|
|
pub ok: bool,
|
|
pub error: String,
|
|
}
|
|
|
|
pub fn parse_amd_tune(line: &str) -> Option<AmdTune> {
|
|
let line = line.trim();
|
|
if !line.starts_with("tune ") {
|
|
return None;
|
|
}
|
|
let (head, rest) = line.split_once(" name \"")?;
|
|
let (name, tail) = rest.split_once('"')?;
|
|
let ordinal: usize = head.split_whitespace().nth(1)?.parse().ok()?;
|
|
let tp: Vec<&str> = tail.split_whitespace().collect();
|
|
let at = |key: &str| tp.iter().position(|p| *p == key);
|
|
let num = |i: usize| tp.get(i).and_then(|v| if *v == "-" { Some(-1.0) } else { v.parse::<f64>().ok() }).unwrap_or(-1.0);
|
|
let g = at("gmax")?;
|
|
let gr = at("gmax_range")?;
|
|
let pl = at("plimit")?;
|
|
let pr = at("plimit_range")?;
|
|
let fi = at("factory")?;
|
|
let verdict = tp.get(fi + 2).copied().unwrap_or("");
|
|
Some(AmdTune { ordinal, name: name.to_string(), gmax: num(g + 1), gmax_min: num(gr + 1), gmax_max: num(gr + 2), plimit: num(pl + 1), plimit_min: num(pr + 1), plimit_max: num(pr + 2), factory: tp.get(fi + 1).copied() == Some("1"), ok: verdict == "ok", error: if verdict == "ok" { String::new() } else { tp[fi + 2..].join(" ") } })
|
|
}
|
|
|
|
/// "2,472 MHz at 100%" or "100%" for the feed.
|
|
fn point_words(p: &crate::ember::Point) -> String {
|
|
if p.clock_mhz > 0 { format!("{} MHz at {}%", p.clock_mhz, p.power_pct) } else { format!("{}% (clock unlocked)", p.power_pct) }
|
|
}
|
|
|
|
fn requested_watts(c: &CardState) -> f64 {
|
|
let pct = if c.power_pct == 0 { 80 } else { c.power_pct.clamp(crate::sweep::MIN_PCT, 100) };
|
|
let mut w = c.power_default_w * pct as f64 / 100.0;
|
|
if c.power_min_w > 0.0 {
|
|
w = w.max(c.power_min_w);
|
|
}
|
|
if c.power_max_w > 0.0 {
|
|
w = w.min(c.power_max_w);
|
|
}
|
|
w.round()
|
|
}
|
|
|
|
/// Per-card session statistics for the stability line.
|
|
#[derive(Default)]
|
|
struct Stability {
|
|
draws: Vec<f64>,
|
|
max_tgpu: f64,
|
|
max_tmem: f64,
|
|
}
|
|
|
|
/// One `igneum-miner watch` reading.
|
|
pub struct Reading {
|
|
pub blocks: u64,
|
|
pub headers: u64,
|
|
pub peers: u64,
|
|
/// getInfo's is_synced, when the line carries one
|
|
pub flag: Option<bool>,
|
|
}
|
|
|
|
/// Is the node synced? Any of: a private test node; our block accepted in the last minute; the node's own flag;
|
|
/// caught up (headers within 2 of blocks, a peer, more than genesis) and the count moving or stable for 60 s.
|
|
/// A false flag never overrides caught-up-and-moving: getInfo said false on PC 2 while it mined (4 October 2026).
|
|
pub fn sync_decision(r: &Reading, moving: bool, stable_s: f64, accepted_recent: bool, private: bool) -> bool {
|
|
if private || accepted_recent || r.flag == Some(true) {
|
|
return true;
|
|
}
|
|
let caught_up = r.peers > 0 && r.headers <= r.blocks + 2 && r.blocks > 1;
|
|
caught_up && (moving || stable_s >= 60.0)
|
|
}
|
|
|
|
/// "...block timestamp is 1791112663000 but maximum timestamp allowed is 1791112600000" -> at least 63 s behind.
|
|
fn behind_from_warning(text: &str) -> Option<f64> {
|
|
let nums: Vec<f64> = text.split(|c: char| !c.is_ascii_digit()).filter(|s| s.len() >= 12).filter_map(|s| s.parse::<f64>().ok()).collect();
|
|
if nums.len() < 2 {
|
|
return None;
|
|
}
|
|
let behind = (nums[nums.len() - 2] - nums[nums.len() - 1]) / 1000.0;
|
|
if behind > 0.0 { Some(behind) } else { None }
|
|
}
|
|
|
|
/// A worker's race line after "race " (docs/design/miner-tuning.md section 3), parsed into the record's fields.
|
|
#[derive(Debug, Default, PartialEq)]
|
|
pub struct RaceParsed {
|
|
pub epoch: String,
|
|
pub device: String,
|
|
pub driver: String,
|
|
pub arch: String,
|
|
pub loads: u64,
|
|
pub wide: u64,
|
|
/// variant name -> MH/s (null when the variant was discarded or not timed)
|
|
pub variants: serde_json::Map<String, Value>,
|
|
pub winner: String,
|
|
pub mhs: f64,
|
|
pub base_mhs: f64,
|
|
pub gain_pct: f64,
|
|
pub total_ms: f64,
|
|
pub pinned: bool,
|
|
pub tuned: bool,
|
|
pub notes: String,
|
|
}
|
|
|
|
pub fn parse_race(body: &str) -> Option<RaceParsed> {
|
|
let (main, notes) = match body.split_once(" | ") {
|
|
Some((m, n)) => (m, n),
|
|
None => (body, ""),
|
|
};
|
|
let t: Vec<&str> = main.split_whitespace().collect();
|
|
let after = |key: &str| t.iter().position(|x| *x == key).and_then(|i| t.get(i + 1)).map(|s| s.to_string());
|
|
let num = |key: &str| after(key).and_then(|v| v.trim_end_matches('%').parse::<f64>().ok());
|
|
let epoch = t.first()?.to_string();
|
|
if epoch.len() < 8 || !epoch.chars().all(|c| c.is_ascii_hexdigit()) {
|
|
return None;
|
|
}
|
|
let mut r = RaceParsed { epoch, device: after("device").unwrap_or_default(), driver: after("driver").unwrap_or_default(), arch: after("arch").unwrap_or_default(), loads: num("loads").unwrap_or(0.0) as u64, wide: num("wide").unwrap_or(0.0) as u64, ..Default::default() };
|
|
let n = num("variants").unwrap_or(0.0) as usize;
|
|
if let Some(i) = t.iter().position(|x| *x == "variants") {
|
|
for tok in t.iter().skip(i + 2).take(n) {
|
|
if let Some((name, rest)) = tok.split_once('=') {
|
|
let mhs = rest.split('/').next().and_then(|m| m.parse::<f64>().ok());
|
|
r.variants.insert(name.to_string(), mhs.map(Value::from).unwrap_or(Value::Null));
|
|
}
|
|
}
|
|
}
|
|
r.winner = after("winner").unwrap_or_default();
|
|
r.mhs = t.iter().position(|x| *x == "winner").and_then(|i| t.get(i + 2)).and_then(|v| v.parse::<f64>().ok()).unwrap_or(0.0);
|
|
r.base_mhs = num("base").unwrap_or(0.0);
|
|
r.gain_pct = num("gain").unwrap_or(0.0);
|
|
r.total_ms = num("total").unwrap_or(0.0);
|
|
r.pinned = main.contains(" pinned by tuning");
|
|
r.tuned = main.contains(" tuned order");
|
|
r.notes = notes.to_string();
|
|
Some(r)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod prepare_packs_tests {
|
|
use super::*;
|
|
|
|
/// Program class v3 (5 October 2026): the Metal worker needs the prepare directory too, in the platform's
|
|
/// separator, relative to the app data folder the miner runs in.
|
|
#[test]
|
|
fn every_worker_gets_the_prepare_directory_in_the_platform_form() {
|
|
let arg = prepare_packs_arg();
|
|
if cfg!(windows) {
|
|
assert_eq!(arg, "packs\\prepare");
|
|
} else {
|
|
assert_eq!(arg, "packs/prepare", "the Mac's Metal worker gets a forward-slash path");
|
|
}
|
|
assert!(!arg.contains(' ') && !arg.starts_with('/'), "relative, no space: the miner splits --worker-args on spaces and the data folder may carry one");
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod resume_tests {
|
|
use super::*;
|
|
|
|
fn card(name: &str, enabled: bool, state: &str, hash: f64) -> CardState {
|
|
CardState { name: name.into(), vendor: "nvidia".into(), kind: "discrete".into(), enabled, state: state.into(), hash_now: hash, ..Default::default() }
|
|
}
|
|
|
|
/// The state machine: paused (every slot stopped, restart_at cleared) -> resumed -> every slot without a live
|
|
/// worker is re-armed, faulted or not.
|
|
#[test]
|
|
fn resume_rearms_every_stopped_slot() {
|
|
let slots = [ResumeSlot { faulted: false, live: false }, ResumeSlot { faulted: true, live: false }, ResumeSlot { faulted: false, live: true }];
|
|
assert_eq!(slots_to_rearm_on_resume(&slots), vec![0, 1], "the healthy stopped slot and the faulted one restart; the live one is left alone");
|
|
}
|
|
|
|
/// The known-failed case (PC 2, 5 October 2026, 21:25:11Z, app 0.3.9: `[ok] mining resumed`, then `0.00 MH/s,
|
|
/// waiting` for 20 minutes): one healthy slot, stopped by the pause, not faulted. The old rule re-armed only
|
|
/// faulted slots and returned nothing for it; the new rule returns it.
|
|
#[test]
|
|
fn the_pc2_resume_of_21_25_11z_restarts_under_the_new_rule_and_not_the_old() {
|
|
let old_rule = |slots: &[ResumeSlot]| -> Vec<usize> { slots.iter().enumerate().filter(|(_, s)| s.faulted).map(|(i, _)| i).collect() };
|
|
let pc2 = [ResumeSlot { faulted: false, live: false }];
|
|
assert!(old_rule(&pc2).is_empty(), "the 0.3.9 rule left the 5090's slot unarmed: this is the defect");
|
|
assert_eq!(slots_to_rearm_on_resume(&pc2), vec![0]);
|
|
}
|
|
|
|
/// The check 90 s after a resume names every enabled card without a hash rate, and nothing else.
|
|
#[test]
|
|
fn resume_check_names_the_cards_not_mining() {
|
|
let cards = [card("NVIDIA GeForce RTX 5090", true, "off", 0.0), card("AMD Radeon(TM) Graphics", true, "mining", 3.4), card("Intel UHD", false, "off", 0.0), card("RTX 3060", true, "faulted", 0.0)];
|
|
let lines = resume_check(&cards);
|
|
assert_eq!(lines.len(), 1, "{lines:?}");
|
|
assert!(lines[0].starts_with("NVIDIA GeForce RTX 5090 is not mining 90 s after resume (state off"), "{}", lines[0]);
|
|
assert!(resume_check(&[card("RTX 5090", true, "mining", 118.9)]).is_empty());
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
#[test]
|
|
fn power_control_off_builds_no_elevated_command() {
|
|
// the decision (the founder, 5 October 2026): off = the app never asks; the elevated PC sweep job is the exception
|
|
assert!(!super::elevation_allowed(false, false));
|
|
assert!(super::elevation_allowed(true, false));
|
|
assert!(!super::elevation_allowed(false, true), "the --sweep job alone never asks (C35)");
|
|
let mut cards = vec![
|
|
super::CardState { vendor: "nvidia".into(), enabled: true, device: "0".into(), name: "RTX 5090".into(), power_default_w: 575.0, power_limit_w: 575.0, power_pct: 80, ..Default::default() },
|
|
super::CardState { vendor: "amd".into(), enabled: true, device: "1".into(), name: "RX 9070 XT".into(), power_default_w: 300.0, power_limit_w: 300.0, power_pct: 80, ..Default::default() },
|
|
];
|
|
let (cmds, what, held) = super::power_cap_plan(&mut cards, false, "nvidia-smi");
|
|
assert!(cmds.is_empty() && what.is_empty(), "{cmds:?}");
|
|
assert_eq!(held, 1);
|
|
assert_eq!(cards[0].power_note, "power cap not set: Power control is off in Settings");
|
|
let (cmds, what, held) = super::power_cap_plan(&mut cards, true, "nvidia-smi");
|
|
assert_eq!(cmds, vec!["\"nvidia-smi\" -i 0 -pl 460".to_string()]);
|
|
assert_eq!(what, vec!["RTX 5090 460 W (80% of 575 W)".to_string()]);
|
|
assert_eq!(held, 0);
|
|
// a cap already in force asks for nothing either way
|
|
cards[0].power_limit_w = 460.0;
|
|
cards[0].power_applied = true;
|
|
assert!(super::power_cap_plan(&mut cards, true, "nvidia-smi").0.is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn a_refused_prompt_switches_power_control_off_with_the_notice() {
|
|
let refused = Err("the administrator prompt was refused, cancelled or timed out (exit 251)".to_string());
|
|
assert_eq!(super::power_control_after_prompt(&refused), Some("power control off: administrator rights were not given"));
|
|
assert_eq!(super::power_control_after_prompt(&Err("the administrator prompt was cancelled".into())), Some(super::POWER_CONTROL_REFUSED));
|
|
assert_eq!(super::power_control_after_prompt(&Err("elevated fail: the administrator prompt was cancelled".into())), Some(super::POWER_CONTROL_REFUSED));
|
|
assert_eq!(super::power_control_after_prompt(&Err("the administrator prompt was not answered in 150 s".into())), Some(super::POWER_CONTROL_REFUSED));
|
|
// rights given: the step ran, whatever the card then said
|
|
assert_eq!(super::power_control_after_prompt(&Ok(())), None);
|
|
assert_eq!(super::power_control_after_prompt(&Err("the elevated step exited with code 2".into())), None);
|
|
}
|
|
|
|
use super::{digest_from_line, parse_race, switches_of, sync_decision, Reading};
|
|
|
|
#[test]
|
|
fn digest_comes_from_the_nodes_own_line_only() {
|
|
let own = "2026-10-05 17:58:13.001+01:00 [INFO ] Consensus params digest: 1f4b44255fcd2ea8f75664ed47200f409186ddd2292960c9e2cf95bbbdc11505 (exchanged in the p2p handshake; a peer with another digest is refused)";
|
|
assert_eq!(digest_from_line(own).as_deref(), Some("1f4b44255fcd2ea8f75664ed47200f409186ddd2292960c9e2cf95bbbdc11505"));
|
|
let peer = "[WARN ] Refusing peer 188.245.5.161:26611: consensus params digest mismatch, local 72532d35 remote a6da35e8 (the peer's override file, environment or build differs)";
|
|
assert_eq!(digest_from_line(peer), None);
|
|
assert_eq!(digest_from_line("Consensus params digest: abc"), None);
|
|
assert_eq!(digest_from_line("[INFO ] Processed 100 blocks"), None);
|
|
}
|
|
|
|
#[test]
|
|
fn switches_are_the_activation_heights_lowest_first_in_plain_words() {
|
|
let v: serde_json::Value = serde_json::from_str(r#"{"difficulty_v2_activation_daa":33000,"fees_v1_activation_daa":210000,"finality_v3_activation_daa":135200,"proving_v0_activation_daa":84100,"max_block_mass":500000}"#).unwrap();
|
|
let s = switches_of(&v);
|
|
assert_eq!(s.iter().map(|x| (x.name.as_str(), x.daa)).collect::<Vec<_>>(), vec![("Difficulty v2", 33000), ("Proving v0", 84100), ("Finality v3", 135200), ("Fees v1", 210000)]);
|
|
assert_eq!(s[3].key, "fees_v1_activation_daa");
|
|
assert!(switches_of(&serde_json::json!({})).is_empty());
|
|
assert!(switches_of(&serde_json::json!(null)).is_empty());
|
|
}
|
|
|
|
#[test]
|
|
fn race_line_parses() {
|
|
let line = "a1b2c3d4e5f60718 device NVIDIA_GeForce_RTX_5090 driver 581.4 arch sm_120 loads 128 wide 0 variants 4 base=115.900/40r/1w u2=117.200/40r/1w ldg=- w4=118.300/40r/4w winner w4 118.300 base 115.900 gain +2.07% compile 1234 bench 9876 total 11110 ms tuned order | ldg: compile: identifier __ldg undefined";
|
|
let r = parse_race(line).unwrap();
|
|
assert_eq!(r.epoch, "a1b2c3d4e5f60718");
|
|
assert_eq!(r.device, "NVIDIA_GeForce_RTX_5090");
|
|
assert_eq!((r.driver.as_str(), r.arch.as_str(), r.loads, r.wide), ("581.4", "sm_120", 128, 0));
|
|
assert_eq!(r.variants.len(), 4);
|
|
assert_eq!(r.variants["w4"], 118.3);
|
|
assert!(r.variants["ldg"].is_null());
|
|
assert_eq!((r.winner.as_str(), r.mhs, r.base_mhs, r.gain_pct, r.total_ms), ("w4", 118.3, 115.9, 2.07, 11110.0));
|
|
assert!(r.tuned && !r.pinned);
|
|
assert_eq!(r.notes, "ldg: compile: identifier __ldg undefined");
|
|
// the Metal shape, pinned, no notes
|
|
let r = parse_race("0000000000000000 device Apple_M5_Max driver macos-26.0.1 arch metal loads 128 wide 0 variants 2 base=-/1024t/32w u2=9.800/1024t/32w winner u2 9.800 base 0.000 gain +0.00% compile 300 bench 10 total 310 ms pinned by tuning").unwrap();
|
|
assert!(r.pinned && r.winner == "u2" && r.variants["base"].is_null());
|
|
assert!(parse_race("not a race line").is_none());
|
|
}
|
|
|
|
fn r(blocks: u64, headers: u64, peers: u64, flag: Option<bool>) -> Reading {
|
|
Reading { blocks, headers, peers, flag }
|
|
}
|
|
|
|
/// PC 2, 4 October 2026: syncing, then every relayed block refused (clock), then the catch-up burst, then steady.
|
|
#[test]
|
|
fn sync_state_machine() {
|
|
// syncing: headers run ahead of blocks
|
|
assert!(!sync_decision(&r(0, 500, 1, Some(false)), false, 0.0, false, false));
|
|
// blocks refused for a slow clock: headers climb, blocks do not, nothing accepted
|
|
assert!(!sync_decision(&r(0, 600, 1, Some(false)), false, 0.0, false, false));
|
|
assert!(!sync_decision(&r(0, 700, 1, Some(false)), false, 120.0, false, false));
|
|
// the clock is fixed: the catch-up burst, blocks moving but still behind the headers
|
|
assert!(!sync_decision(&r(46, 700, 1, Some(false)), true, 0.0, false, false));
|
|
assert!(!sync_decision(&r(400, 700, 1, Some(false)), true, 0.0, false, false));
|
|
// caught up and moving: synced, whatever getInfo's flag says
|
|
assert!(sync_decision(&r(699, 700, 1, Some(false)), true, 0.0, false, false));
|
|
assert!(sync_decision(&r(700, 700, 1, None), true, 0.0, false, false));
|
|
// steady: one reading with the same count (a quiet 10 s) is still synced once stable for a minute
|
|
assert!(!sync_decision(&r(700, 700, 1, Some(false)), false, 10.0, false, false));
|
|
assert!(sync_decision(&r(700, 700, 1, Some(false)), false, 60.0, false, false));
|
|
// our block accepted in the last minute: proof of sync even with a stale reading
|
|
assert!(sync_decision(&r(0, 700, 1, Some(false)), false, 0.0, true, false));
|
|
// the node's own flag
|
|
assert!(sync_decision(&r(650, 700, 1, Some(true)), false, 0.0, false, false));
|
|
// no peer: never synced from the numbers alone
|
|
assert!(!sync_decision(&r(700, 700, 0, None), true, 100.0, false, false));
|
|
// a private test node
|
|
assert!(sync_decision(&r(0, 0, 0, None), false, 0.0, false, true));
|
|
}
|
|
|
|
#[test]
|
|
fn node_warning() {
|
|
let l = "2026-10-04 11:02:11.123+00:00 [WARN ] HandleRelayInvsFlow flow error: the block timestamp is too far into the future: block timestamp is 1791112663000 but maximum timestamp allowed is 1791112600000";
|
|
assert_eq!(super::behind_from_warning(l), Some(63.0));
|
|
assert_eq!(super::behind_from_warning("no numbers here"), None);
|
|
}
|
|
}
|
|
|
|
fn short(s: &str, n: usize) -> String {
|
|
if s.chars().count() <= n { s.to_string() } else { format!("{}...", s.chars().take(n).collect::<String>()) }
|
|
}
|
|
|
|
fn tail_of(path: &std::path::Path, n: usize) -> Vec<String> {
|
|
let Ok(text) = std::fs::read_to_string(path) else { return vec![] };
|
|
let lines: Vec<&str> = text.lines().filter(|l| !l.trim().is_empty()).collect();
|
|
lines.iter().rev().take(n).rev().map(|s| s.to_string()).collect()
|
|
}
|
|
|
|
/// Days since 1970-01-01 to a civil date (Howard Hinnant's algorithm).
|
|
fn civil_from_days(z: i64) -> (i64, u32, u32) {
|
|
let z = z + 719_468;
|
|
let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
|
|
let doe = z - era * 146_097;
|
|
let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
|
|
let y = yoe + era * 400;
|
|
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
|
let mp = (5 * doy + 2) / 153;
|
|
let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
|
|
let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
|
|
(if m <= 2 { y + 1 } else { y }, m, d)
|
|
}
|
|
|
|
/// One pack export at a time (5 October 2026). prepare_worker runs on a thread per card, so two cards starting
|
|
/// together ran two `igneum-miner export-pack` processes into the same folder; across an epoch change they
|
|
/// interleaved and PC 1's packs\devnet was left with one epoch's program.h and the other's seeds.txt, which every
|
|
/// OpenCL worker start then refused ("the epoch seed bytes do not give the pack's IGNEUM_SEEDW_INIT") until the next
|
|
/// export. The second export of a pair rewrites the same pack, which is harmless.
|
|
static EXPORT_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(());
|
|
|
|
/// Exports this hour's program pack from the node to <app data>\packs\devnet (the prebuilt workers read it with --pack).
|
|
#[allow(unused_variables)]
|
|
fn export_pack(shared: &Arc<Shared>, bins: &Bins) -> Result<(), String> {
|
|
let _one_at_a_time = EXPORT_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
|
let pack = shared.runtime.app_dir.join("packs").join("devnet");
|
|
let _ = std::fs::create_dir_all(&pack);
|
|
let out = crate::detect::run_timeout(std::process::Command::new(&bins.miner).args(["export-pack", &shared.runtime.rpc_url(), &pack.display().to_string()]), None, Duration::from_secs(120)).unwrap_or_default();
|
|
shared.log(&format!("export-pack: {}", out.lines().last().unwrap_or("no output")));
|
|
if pack.join("seeds.txt").exists() { Ok(()) } else { Err("export-pack wrote no seeds.txt (is the node reachable?)".into()) }
|
|
}
|
|
|
|
/// Today's Windows path when no prebuilt worker ships: export the program pack, run proto-cuda\build.bat (or
|
|
/// proto-opencl\build.bat) in the MSVC environment, and use the exe it writes. Untested from the Mac.
|
|
#[allow(unused_variables)]
|
|
fn build_worker_from_source(shared: &Arc<Shared>, bins: &Bins, vendor: &str) -> Result<PathBuf, String> {
|
|
#[cfg(not(windows))]
|
|
{
|
|
Err("no GPU worker is installed".into())
|
|
}
|
|
#[cfg(windows)]
|
|
{
|
|
use std::process::Command;
|
|
let _one_at_a_time = EXPORT_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
|
let pack = shared.runtime.app_dir.join("packs").join("devnet");
|
|
let _ = std::fs::create_dir_all(&pack);
|
|
let out = crate::detect::run_timeout(Command::new(&bins.miner).args(["export-pack", &shared.runtime.rpc_url(), &pack.display().to_string()]), None, Duration::from_secs(120)).unwrap_or_default();
|
|
shared.log(&format!("export-pack: {}", out.lines().last().unwrap_or("")));
|
|
if !pack.join("seeds.txt").exists() {
|
|
return Err("export-pack wrote no seeds.txt (is the node reachable?)".into());
|
|
}
|
|
let (sub, exe, cmd) = if vendor == "nvidia" {
|
|
("proto-cuda", "igneum-bench-cuda-devnet.exe", "build.bat devnet sm_120")
|
|
} else {
|
|
("proto-opencl", "igneum-bench-cl-devnet.exe", "build.bat devnet")
|
|
};
|
|
// the sources ship under Program Files (read-only, R4.3.3); the build runs under the app data folder
|
|
let src = bins.dir.join(sub);
|
|
if !src.join("build.bat").exists() {
|
|
return Err(format!("{sub}\\build.bat is not installed; the prebuilt worker is missing too"));
|
|
}
|
|
let dir = shared.runtime.app_dir.join("build").join(sub);
|
|
let _ = std::fs::create_dir_all(&dir);
|
|
if let Ok(rd) = std::fs::read_dir(&src) {
|
|
for e in rd.flatten() {
|
|
if e.path().is_file() {
|
|
let _ = std::fs::copy(e.path(), dir.join(e.file_name()));
|
|
}
|
|
}
|
|
}
|
|
// the pack where build.bat expects it
|
|
let dest = dir.join("packs").join("devnet");
|
|
let _ = std::fs::create_dir_all(&dest);
|
|
if let Ok(rd) = std::fs::read_dir(&pack) {
|
|
for e in rd.flatten() {
|
|
let _ = std::fs::copy(e.path(), dest.join(e.file_name()));
|
|
}
|
|
}
|
|
let mut vcvars = None;
|
|
for pf in ["C:\\Program Files\\Microsoft Visual Studio", "C:\\Program Files (x86)\\Microsoft Visual Studio"] {
|
|
if let Ok(years) = std::fs::read_dir(pf) {
|
|
for y in years.flatten() {
|
|
if let Ok(eds) = std::fs::read_dir(y.path()) {
|
|
for e in eds.flatten() {
|
|
let p = e.path().join("VC").join("Auxiliary").join("Build").join("vcvarsall.bat");
|
|
if p.exists() {
|
|
vcvars = Some(p);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
let vc = vcvars.ok_or("Visual Studio with the MSVC v143 x64 component is not installed (vcvarsall.bat not found)")?;
|
|
let line = format!("\"{}\" x64 -vcvars_ver=14.30 >nul 2>&1 && cd /d \"{}\" && {}", vc.display(), dir.display(), cmd);
|
|
shared.log(&format!("building the {sub} worker: cmd /s /c \"{line}\""));
|
|
let out = crate::detect::run_timeout(Command::new(crate::platform::tool("cmd")).args(["/s", "/c", &format!("\"{line}\"")]), None, Duration::from_secs(900)).unwrap_or_default();
|
|
for l in out.lines().rev().take(8).collect::<Vec<_>>().into_iter().rev() {
|
|
shared.log(&format!(" build: {l}"));
|
|
}
|
|
let p = dir.join(exe);
|
|
if p.exists() { Ok(p) } else { Err(format!("{exe} was not written; see the log")) }
|
|
}
|
|
}
|
|
|
|
/// One `amd` line of igneum-gpu-telemetry (proto-opencl/gpu-telemetry.c): the fields the card row carries.
|
|
/// `ordinal_in_kind` is the line's rank among the lines of the same kind in one sample: the helper's `amd N` is
|
|
/// the rank over all kinds, so the parser tracks the kinds it has seen through `AmdTelemetrySample::new`
|
|
/// per sample; a single line parses with the rank 0 when it is the first of its kind in its `amd N` ordering.
|
|
#[derive(Debug, Clone, PartialEq)]
|
|
pub struct AmdTelemetry {
|
|
pub ordinal: usize,
|
|
pub ordinal_in_kind: usize,
|
|
pub bus: String,
|
|
pub kind: String,
|
|
pub name: String,
|
|
pub watts: f64,
|
|
pub temp_c: f64,
|
|
pub fan_rpm: f64,
|
|
pub fan_pct: f64,
|
|
pub mclk_mhz: f64,
|
|
pub gclk_mhz: f64,
|
|
pub util_pct: f64,
|
|
/// the limits in force (0.3.10 lines; -1.0 on older helpers): 100 + the power offset, the max GPU clock
|
|
pub plimit_pct: f64,
|
|
pub gmax_mhz: f64,
|
|
pub source: String,
|
|
}
|
|
|
|
/// Parses `amd <n> bus <b> kind <k> name "<name>" watts <w> temp_c <t> fan_rpm <r> fan_pct <p> mclk_mhz <m>
|
|
/// gclk_mhz <g> util_pct <u> source <s>`; a `-` value reads as -1.0. Anything else (info, end) gives None.
|
|
/// With one card per kind (the common case) `ordinal_in_kind` is 0 for the discrete card and 0 for the integrated
|
|
/// one whatever their `amd N`; with several discrete cards the helper's order within the kind is kept: the rank is
|
|
/// the number of earlier lines of the same kind, which the helper encodes by listing kinds contiguously (ADLX lists
|
|
/// GPUs in a fixed order, so the rank of a card is stable across samples).
|
|
pub fn parse_amd_telemetry(line: &str) -> Option<AmdTelemetry> {
|
|
let line = line.trim();
|
|
if !line.starts_with("amd ") {
|
|
return None;
|
|
}
|
|
let (head, rest) = line.split_once(" name \"")?;
|
|
let (name, tail) = rest.split_once('"')?;
|
|
let hp: Vec<&str> = head.split_whitespace().collect();
|
|
if hp.len() < 6 || hp[2] != "bus" || hp[4] != "kind" {
|
|
return None;
|
|
}
|
|
let ordinal: usize = hp[1].parse().ok()?;
|
|
let tp: Vec<&str> = tail.split_whitespace().collect();
|
|
let num = |key: &str| -> Option<f64> {
|
|
let i = tp.iter().position(|p| *p == key)?;
|
|
let v = tp.get(i + 1)?;
|
|
if *v == "-" { Some(-1.0) } else { v.parse::<f64>().ok() }
|
|
};
|
|
let watts = num("watts")?;
|
|
let temp_c = num("temp_c")?;
|
|
let fan_rpm = num("fan_rpm")?;
|
|
let fan_pct = num("fan_pct")?;
|
|
let mclk_mhz = num("mclk_mhz")?;
|
|
let gclk_mhz = num("gclk_mhz")?;
|
|
let util_pct = num("util_pct")?;
|
|
let plimit_pct = num("plimit_pct").unwrap_or(-1.0);
|
|
let gmax_mhz = num("gmax_mhz").unwrap_or(-1.0);
|
|
let source = tp.iter().position(|p| *p == "source").and_then(|i| tp.get(i + 1)).map(|s| s.to_string()).unwrap_or_default();
|
|
let kind = hp[5].to_string();
|
|
// the rank within the kind: the helper lists one integrated card at most and it comes first when present
|
|
// (ADLX order on every PC seen so far), so a discrete card's rank is its ordinal minus the integrated ones before it
|
|
let ordinal_in_kind = if kind == "discrete" && ordinal > 0 { ordinal - 1 } else if kind == "discrete" { 0 } else { 0 };
|
|
Some(AmdTelemetry { ordinal, ordinal_in_kind, bus: hp[3].to_string(), kind, name: name.to_string(), watts, temp_c, fan_rpm, fan_pct, mclk_mhz, gclk_mhz, util_pct, plimit_pct, gmax_mhz, source })
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod amd_telemetry_tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn a_sysfs_line_from_the_fixture_parses() {
|
|
// proto-opencl/gpu-telemetry.c on the Mac against a fixture tree, 5 October 2026
|
|
let l = "amd 0 bus 0000:0c:00.0 kind discrete name \"AMD Radeon RX 9070 XT\" watts 287.0 temp_c 61.0 fan_rpm 1180 fan_pct 30 mclk_mhz 1258 gclk_mhz 2450 util_pct 90 source sysfs";
|
|
let s = parse_amd_telemetry(l).unwrap();
|
|
assert_eq!((s.ordinal, s.ordinal_in_kind, s.bus.as_str(), s.kind.as_str(), s.name.as_str()), (0, 0, "0000:0c:00.0", "discrete", "AMD Radeon RX 9070 XT"));
|
|
assert_eq!((s.watts, s.temp_c, s.fan_rpm, s.fan_pct, s.mclk_mhz, s.gclk_mhz, s.util_pct), (287.0, 61.0, 1180.0, 30.0, 1258.0, 2450.0, 90.0));
|
|
assert_eq!(s.source, "sysfs");
|
|
}
|
|
|
|
#[test]
|
|
fn a_dash_reads_as_unknown_and_other_lines_give_none() {
|
|
let l = "amd 1 bus 98 kind discrete name \"AMD Radeon RX 9070 XT\" watts 250.3 temp_c 58.0 fan_rpm 900 fan_pct - mclk_mhz 1258 gclk_mhz 2460 util_pct 97.5 source adlx";
|
|
let s = parse_amd_telemetry(l).unwrap();
|
|
assert_eq!(s.fan_pct, -1.0);
|
|
assert_eq!(s.ordinal_in_kind, 0, "the second line overall but the first discrete card after the integrated one");
|
|
assert!(parse_amd_telemetry("end 3.2 ms 2 card(s)").is_none());
|
|
assert!(parse_amd_telemetry("info adlx: ADLXHelper_Initialize returned 1").is_none());
|
|
assert!(parse_amd_telemetry("amd 0 bus - kind - name \"x\" watts").is_none());
|
|
}
|
|
|
|
/// The 0.3.10 helper's lines (the tuning commands, usage header of proto-opencl/gpu-telemetry.c): the sample
|
|
/// line carries the limits in force, the `tune` line the ranges a card allows. An older line (no plimit_pct,
|
|
/// no gmax_mhz) still parses with -1 there.
|
|
#[test]
|
|
fn the_tune_line_and_the_limits_in_force_parse() {
|
|
let l = "amd 1 bus 98 kind discrete name \"AMD Radeon RX 9070 XT\" watts 198.9 temp_c 64.0 fan_rpm 657 fan_pct 25 mclk_mhz 2505 gclk_mhz 3290 util_pct 100 plimit_pct 100 gmax_mhz 3100 source adlx";
|
|
let s = parse_amd_telemetry(l).unwrap();
|
|
assert_eq!((s.plimit_pct, s.gmax_mhz, s.gclk_mhz), (100.0, 3100.0, 3290.0));
|
|
let old = "amd 0 bus 0000:0c:00.0 kind discrete name \"AMD Radeon RX 9070 XT\" watts 287.0 temp_c 61.0 fan_rpm 1180 fan_pct 30 mclk_mhz 1258 gclk_mhz 2450 util_pct 90 source sysfs";
|
|
assert_eq!(parse_amd_telemetry(old).unwrap().plimit_pct, -1.0);
|
|
let t = parse_amd_tune("tune 1 name \"AMD Radeon RX 9070 XT\" gmax 3100 gmax_range 500 3400 plimit 0 plimit_range -30 15 factory 1 ok").unwrap();
|
|
assert_eq!((t.ordinal, t.name.as_str(), t.gmax, t.gmax_min, t.gmax_max, t.plimit, t.plimit_min, t.plimit_max, t.factory, t.ok), (1, "AMD Radeon RX 9070 XT", 3100.0, 500.0, 3400.0, 0.0, -30.0, 15.0, true, true));
|
|
let e = parse_amd_tune("tune 0 name \"x\" gmax - gmax_range - - plimit - plimit_range - - factory 0 ADLX: GetManualGraphicsTuning returned 2").unwrap();
|
|
assert!(!e.ok);
|
|
assert_eq!(e.error, "ADLX: GetManualGraphicsTuning returned 2");
|
|
assert_eq!(e.gmax_max, -1.0);
|
|
assert!(parse_amd_tune("end 3.2 ms 2 card(s)").is_none());
|
|
assert!(parse_amd_tune("tune 0 name \"x\" gmax 1").is_none());
|
|
}
|
|
|
|
#[test]
|
|
fn the_perfcounter_fallback_line_parses() {
|
|
let l = "amd 0 bus luid_0x00000000_0x0000D4E3 kind - name \"-\" watts - temp_c - fan_rpm - fan_pct - mclk_mhz - gclk_mhz - util_pct 100 source perfcounter";
|
|
let s = parse_amd_telemetry(l).unwrap();
|
|
assert_eq!((s.watts, s.util_pct, s.source.as_str()), (-1.0, 100.0, "perfcounter"));
|
|
}
|
|
}
|