From 7778dd7cde6ee5588375d64f505bd517a582c2b9 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:38:07 +0000 Subject: [PATCH] Miner efficiency sweep: hash per watt per NVIDIA card (lever 3) src/sweep.rs (new): the cap steps 100% to 50% in 10% steps clamped to the card's limits, per-step rows (mean draw from nvidia-smi power.draw, mean worker interval rate), the choice (best MH/W, ties to the higher rate then the lower cap), the nvidia-smi power parser, a state machine on an explicit clock (15 s settle, 60 s hold, 30 s cap readback limit), the elevated helper scripts (one administrator prompt per sweep: a command file polled by one elevated process, self-restoring after 20 idle minutes), the unsupported reasons (Apple silicon, AMD). 9 unit tests with PC 1's recorded RTX 5090 numbers (575 W default, 460 W cap, 290 W draw, memory temperature [N/A]). Engine: scheduler (once after install, then weekly; one card at a time; only while the card mines, after 120 s steady, never under a remote job hold, a pause, or inside 600 s of the hour boundary), the cap-mode probe (direct when the engine runs elevated, else the helper), abort on any fault (card leaves mining, worker error, GPU 90 C, job, pause, quit) with the cap restored, the chosen cap held and recorded, SWEEP table lines in the app log, --sweep mode (sweep every supported card, print the table on stdout, leave the caps, quit). Cap floor 50% (was 60). A readback that matches the asked cap now counts as applied (PC 1 showed "cap NOT applied" for hours at 460 W). Dashboard: live eff MH/W on each tile, the sweep line (phase, last result, or why unsupported), Sweep now / Stop / Unpin, "pinned" and "chosen by the sweep" on the cap line, the Settings toggle, the cards-page note, slider min 50. A cap moved by hand pins the card: the sweep records but does not change it. PC 1 measurement: relay/playbooks/sweep-5090.ps1 (a run job, elevated, miners stopped; a second engine with --sweep in a scratch data folder, RESULT SWEEP lines) and docs/plans/miner-eff.md with the publish command. Not published. Untested on a card. Co-Authored-By: Claude Fable 5.1 --- app/igneum-app/src/config.rs | 39 ++- app/igneum-app/src/detect.rs | 31 ++ app/igneum-app/src/engine.rs | 561 ++++++++++++++++++++++++++++++- app/igneum-app/src/main.rs | 12 +- app/igneum-app/src/server.rs | 24 +- app/igneum-app/src/state.rs | 15 +- app/igneum-app/src/sweep.rs | 593 +++++++++++++++++++++++++++++++++ app/igneum-app/ui/app.css | 4 + app/igneum-app/ui/app.js | 29 +- app/igneum-app/ui/index.html | 7 +- docs/plans/miner-eff.md | 120 +++++++ relay/playbooks/sweep-5090.ps1 | 111 ++++++ 12 files changed, 1524 insertions(+), 22 deletions(-) create mode 100644 app/igneum-app/src/sweep.rs create mode 100644 docs/plans/miner-eff.md create mode 100644 relay/playbooks/sweep-5090.ps1 diff --git a/app/igneum-app/src/config.rs b/app/igneum-app/src/config.rs index 966315868..30e034ebc 100644 --- a/app/igneum-app/src/config.rs +++ b/app/igneum-app/src/config.rs @@ -12,9 +12,23 @@ use std::path::{Path, PathBuf}; pub struct CardPref { pub enabled: bool, pub identities: u32, - /// NVIDIA power cap, percent of the card's default limit (60 to 100); 0 = the default 80 + /// NVIDIA power cap, percent of the card's default limit (50 to 100); 0 = the default 80 #[serde(default)] pub power_pct: u32, + /// the user moved the slider: this cap stays; the efficiency sweep (src/sweep.rs) records but does not change it + #[serde(default)] + pub pinned: bool, + /// the last efficiency sweep: when (unix s), the cap it chose, and that step's numbers + #[serde(default)] + pub sweep_at: u64, + #[serde(default)] + pub sweep_pct: u32, + #[serde(default)] + pub sweep_eff: f64, + #[serde(default)] + pub sweep_watts: f64, + #[serde(default)] + pub sweep_mhs: f64, } #[derive(Clone, Serialize, Deserialize)] @@ -55,6 +69,13 @@ pub struct Settings { /// Prove the shards assigned to this machine's keys (src/prover.rs). Default off until the GPU numbers exist. #[serde(default)] pub prove: bool, + /// The efficiency sweep (src/sweep.rs): once after install, then weekly, each NVIDIA card's cap is stepped from + /// 100% to 50% on the live program and held at the best MH per watt. Default on. A pinned card is skipped. + #[serde(default = "yes")] + pub sweep: bool, + /// When this install first ran (unix s), for the "first hour after install" sweep. + #[serde(default)] + pub installed_at: u64, } fn one() -> u32 { @@ -66,13 +87,19 @@ fn yes() -> bool { impl Default for Settings { fn default() -> Settings { - Settings { setup_done: false, address: String::new(), address_source: String::new(), key_saved: false, identities: 1, cards: HashMap::new(), display_name: String::new(), vote: true, paused: false, accepted_total: 0, auto_update: true, remote_jobs: true, prove: false } + Settings { setup_done: false, address: String::new(), address_source: String::new(), key_saved: false, identities: 1, cards: HashMap::new(), display_name: String::new(), vote: true, paused: false, accepted_total: 0, auto_update: true, remote_jobs: true, prove: false, sweep: true, installed_at: 0 } } } impl Settings { pub fn load(path: &Path) -> Settings { - std::fs::read_to_string(path).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default() + let mut s: Settings = std::fs::read_to_string(path).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default(); + if s.installed_at == 0 { + // an install from before the sweep existed counts as installed now: it gets its first-hour sweep + s.installed_at = crate::platform::unix_now(); + s.save(path); + } + s } pub fn save(&self, path: &Path) { if let Some(d) = path.parent() { @@ -148,6 +175,9 @@ pub struct Runtime { pub app_dir: PathBuf, pub log_dir: PathBuf, pub status_secs: u32, + /// `--sweep`: run the efficiency sweep on every supported card as soon as it mines, print the table on stdout, + /// leave the chosen caps in force and quit (the PC measurement job; docs/plans/miner-eff.md). + pub sweep_only: bool, /// The machine's hostname: a friendly label only, never part of a key or a payout address. pub host: String, /// Per-install random id (16 hex, app data dir/machine-id, locked to the user). Identity labels, vote keys and @@ -181,7 +211,8 @@ impl Runtime { let log_dir = crate::platform::log_root(); let status_secs = env("IGNEUM_APP_STATUS_SECS").and_then(|v| v.parse().ok()).unwrap_or(30); let machine_id = machine_id(&app_dir); - Runtime { network, rpc_port, p2p_port, peers, unsynced_mining, devnet_suffix, node_dir, app_dir, log_dir, status_secs, host: crate::platform::host_label(), machine_id } + let sweep_only = env("IGNEUM_APP_SWEEP").map(|v| v == "1").unwrap_or(false); + Runtime { network, rpc_port, p2p_port, peers, unsynced_mining, devnet_suffix, node_dir, app_dir, log_dir, status_secs, sweep_only, host: crate::platform::host_label(), machine_id } } /// The first 8 hex of the machine id: what goes into labels. pub fn id8(&self) -> String { diff --git a/app/igneum-app/src/detect.rs b/app/igneum-app/src/detect.rs index fd2e1b521..1af73c031 100644 --- a/app/igneum-app/src/detect.rs +++ b/app/igneum-app/src/detect.rs @@ -99,6 +99,34 @@ pub fn apply_defaults(c: &mut CardState) { } } +/// Marks whether the efficiency sweep (src/sweep.rs) can run on a card, with the reason when it cannot. Called +/// once the power limits are known. +pub fn mark_sweep_support(c: &mut CardState) { + match crate::sweep::unsupported_reason(&c.vendor, c.power_default_w, &c.device) { + None => { + c.sweep_supported = true; + if c.sweep_state.is_empty() || c.sweep_state == "unsupported" { + c.sweep_state = "idle".into(); + c.sweep_note = String::new(); + } + } + Some(why) => { + c.sweep_supported = false; + c.sweep_state = "unsupported".into(); + c.sweep_note = why.into(); + } + } +} + +/// `nvidia-smi --query-gpu=index,power.draw`: index -> watts now (the one-shot form; the telemetry child streams it). +#[cfg(not(target_os = "macos"))] +#[allow(dead_code)] +pub fn nvidia_power_draw() -> std::collections::HashMap { + run_timeout(Command::new(crate::platform::tool("nvidia-smi")).args(["--query-gpu=index,power.draw", "--format=csv,noheader,nounits"]), None, Duration::from_secs(10)) + .map(|t| crate::sweep::parse_power_draw(&t)) + .unwrap_or_default() +} + /// NVIDIA power limits per card index: (default, current, min, max) in watts. #[cfg(target_os = "macos")] pub fn nvidia_power_limits() -> std::collections::HashMap { @@ -163,6 +191,7 @@ pub fn detect(bins: &Bins, notes: &mut Vec) -> Vec { c.vram_mb = mem.trim().parse::().map(|b| b / (1024 * 1024)).unwrap_or(0); } apply_defaults(&mut c); + mark_sweep_support(&mut c); cards.push(c); cards } @@ -203,6 +232,7 @@ pub fn detect(bins: &Bins, notes: &mut Vec) -> Vec { c.power_min_w = *lo; c.power_max_w = *hi; } + mark_sweep_support(c); } } None => notes.push("nvidia-smi is not on this PC (no NVIDIA driver): no NVIDIA card".into()), @@ -233,6 +263,7 @@ pub fn detect(bins: &Bins, notes: &mut Vec) -> Vec { c.kind = if looks_integrated(name) { "integrated".into() } else { "discrete".into() }; c.path = "prebuilt".into(); apply_defaults(&mut c); + mark_sweep_support(&mut c); cards.push(c); } } diff --git a/app/igneum-app/src/engine.rs b/app/igneum-app/src/engine.rs index fe4b8e20c..4029c2f9c 100644 --- a/app/igneum-app/src/engine.rs +++ b/app/igneum-app/src/engine.rs @@ -58,6 +58,18 @@ pub enum Cmd { ElevatedDone(Result<(), String>), /// the user asked for the cap again (the Retry button) ApplyPower, + /// the efficiency sweep (src/sweep.rs): start on one card (by key), stop, pin or unpin a card's cap, the toggle + SweepStart(String), + SweepStop, + SweepPin(String, bool), + SweepEnable(bool), + /// the probe that decides how caps are set during a sweep: Ok(true) = nvidia-smi -pl works from this process + /// (the engine runs elevated), Ok(false) = the elevated helper is needed; Err = nvidia-smi did not answer + SweepCapMode(Result), + /// the elevated helper process ended (Ok at `quit`, Err when the prompt was cancelled or it failed) + SweepHelperDone(Result<(), String>), + /// a direct `nvidia-smi -pl` for the sweep finished: what it printed + SweepCapSet(String), Quit, } @@ -97,7 +109,7 @@ impl Shared { st.mining.paused = settings.paused; st.mining.accepted_total = settings.accepted_total; st.address = address_state(&settings, &wallet_path); - st.settings = crate::state::SettingsState { identities: settings.identities, vote: settings.vote, start_at_login: crate::platform::start_at_login_is_on(), auto_update: settings.auto_update, remote_jobs: settings.remote_jobs, prove: settings.prove }; + st.settings = crate::state::SettingsState { identities: settings.identities, vote: settings.vote, start_at_login: crate::platform::start_at_login_is_on(), auto_update: settings.auto_update, remote_jobs: settings.remote_jobs, prove: settings.prove, sweep: settings.sweep }; st.live_page = packaged.live_page.clone(); st.finality.message = "waiting for the miner".into(); st.clock.hint = crate::platform::clock_hint().into(); @@ -416,6 +428,21 @@ pub struct Engine { last_stability: Instant, last_settings_save: Instant, last_error_event: Instant, + // the efficiency sweep (src/sweep.rs): one card at a time + sweep: Option, + /// a sweep waiting for the cap-mode probe: (card index, forced) + sweep_pending: Option<(usize, bool)>, + /// how caps are set: Some(true) directly, Some(false) through the elevated helper, None = not probed yet + sweep_direct: Option, + /// the elevated helper is running (its thread reports SweepHelperDone when it ends) + sweep_helper: bool, + sweep_seq: u64, + /// cards asked for by hand (Sweep now) or by --sweep, by key + sweep_queue: Vec, + sweep_next_check: Instant, + /// after an abort, a card waits before the scheduler tries it again + sweep_retry: std::collections::HashMap, + sweep_attempts: std::collections::HashMap, } impl Engine { @@ -487,6 +514,15 @@ impl Engine { last_stability: now, last_settings_save: now, last_error_event: now - Duration::from_secs(600), + sweep: None, + sweep_pending: None, + sweep_direct: None, + sweep_helper: false, + sweep_seq: 0, + sweep_queue: Vec::new(), + sweep_next_check: now + Duration::from_secs(20), + sweep_retry: std::collections::HashMap::new(), + sweep_attempts: std::collections::HashMap::new(), } } @@ -530,7 +566,10 @@ impl Engine { } } self.shared.send(Cmd::Detect); - if setup_done && !self.quitting { + if self.shared.runtime.sweep_only { + self.shared.log("--sweep: the efficiency sweep runs on every supported card as soon as it mines; the table goes to stdout and this log; the engine quits after it"); + } + if (setup_done || self.shared.runtime.sweep_only) && !self.quitting { self.shared.send(Cmd::Start); } loop { @@ -582,11 +621,30 @@ impl Engine { c.identities = p.identities.clamp(1, 64); c.reason = String::new(); if c.vendor == "nvidia" && p.power_pct > 0 { - c.power_pct = p.power_pct.clamp(60, 100); + c.power_pct = p.power_pct.clamp(crate::sweep::MIN_PCT, 100); } + c.pinned = p.pinned; + c.sweep_pct = p.sweep_pct; + c.sweep_eff = p.sweep_eff; + c.sweep_watts = p.sweep_watts; + c.sweep_mhs = p.sweep_mhs; + c.sweep_at = p.sweep_at as f64; } } + let supported: Vec = st.mining.cards.iter().filter(|c| c.sweep_supported && c.enabled).map(|c| c.key.clone()).collect(); + let unsupported: Vec = st.mining.cards.iter().filter(|c| !c.sweep_supported).map(|c| format!("{}: {}", c.name, c.sweep_note)).collect(); drop(st); + for u in &unsupported { + self.shared.log(&format!("efficiency sweep {u}")); + } + if self.shared.runtime.sweep_only { + if supported.is_empty() { + self.sweep_say("SWEEP none reason=no_supported_card"); + self.shared.send(Cmd::Quit); + } else { + self.sweep_queue = supported; + } + } if self.running { self.plan_miners(); } @@ -768,11 +826,80 @@ impl Engine { } } Cmd::ApplyPower => { + if self.sweep.is_some() || self.sweep_pending.is_some() { + self.shared.event("info", "a sweep is setting the caps right now; retry once it is done"); + return; + } for c in self.st().mining.cards.iter_mut().filter(|c| c.vendor == "nvidia") { c.power_applied = false; // force the step again } self.apply_power_limits("retry"); } + Cmd::SweepStart(key) => { + let (name, ok, why) = { + let st = self.st(); + match st.mining.cards.iter().find(|c| c.key == key) { + Some(c) if !c.sweep_supported => (c.name.clone(), false, c.sweep_note.clone()), + Some(c) if !c.enabled => (c.name.clone(), false, "the card is switched off".to_string()), + Some(c) => (c.name.clone(), true, String::new()), + None => (key.clone(), false, "no such card".to_string()), + } + }; + if !ok { + self.shared.event("error", &format!("{name}: no sweep: {why}")); + } else if !self.sweep_queue.contains(&key) { + self.sweep_queue.push(key); + self.sweep_next_check = Instant::now(); + self.shared.event("info", &format!("{name}: efficiency sweep queued; it starts once the card has mined for {} s with over {} s to the hour boundary", crate::sweep::STABLE_S, crate::sweep::NEEDS_S)); + } + } + Cmd::SweepStop => { + self.sweep_queue.clear(); + if self.sweep.is_some() || self.sweep_pending.is_some() { + self.sweep_abort("stopped from the dashboard"); + } + } + Cmd::SweepPin(key, pinned) => { + let name = { + let mut settings = self.shared.settings.lock().unwrap(); + let mut st = self.st(); + let Some(c) = st.mining.cards.iter_mut().find(|c| c.key == key) else { return }; + c.pinned = pinned; + let e = settings.cards.entry(key.clone()).or_insert_with(|| crate::config::CardPref { enabled: c.enabled, identities: c.identities, ..Default::default() }); + e.pinned = pinned; + if pinned { + e.power_pct = c.power_pct; + } + c.name.clone() + }; + self.shared.save_settings(); + self.shared.event("info", &if pinned { format!("{name}: cap pinned; the sweep records but does not change it") } else { format!("{name}: the sweep chooses the cap again from its next run") }); + } + Cmd::SweepEnable(on) => { + self.shared.settings.lock().unwrap().sweep = on; + self.shared.save_settings(); + self.st().settings.sweep = on; + self.shared.event("info", if on { "efficiency sweep on: once after install, then weekly" } else { "efficiency sweep off; Sweep now on a card still runs one" }); + } + Cmd::SweepCapMode(r) => self.sweep_mode_known(r), + Cmd::SweepHelperDone(r) => { + self.sweep_helper = false; + if let Err(e) = &r { + self.shared.log(&format!("sweep helper: {e}")); + if self.sweep.is_some() || self.sweep_pending.is_some() { + self.sweep_abort(&format!("the elevated helper did not run ({e})")); + } + } else if self.sweep.is_some() { + self.sweep_abort("the elevated helper exited early"); + } else { + self.shared.log("sweep helper: exited"); + } + } + Cmd::SweepCapSet(text) => { + if !text.contains("All done") { + self.shared.log(&format!("sweep: nvidia-smi -pl answered: {}", short(&text.replace('\n', " "), 200))); + } + } Cmd::Quit => { self.quitting = true; self.st().quitting = true; @@ -1117,6 +1244,7 @@ impl Engine { fn apply_cards(&mut self, choices: Vec) { let mut changed = Vec::new(); let mut power_changed = false; + let mut pin_aborts = false; { let mut settings = self.shared.settings.lock().unwrap(); let mut st = self.st(); @@ -1124,13 +1252,19 @@ impl Engine { let Some(c) = st.mining.cards.iter_mut().find(|c| c.key == ch.key) else { continue }; let identities = ch.identities.clamp(1, 64); let enabled = ch.enabled && c.kind != "unknown"; - let power_pct = ch.power_pct.map(|p| p.clamp(60, 100)).unwrap_or(c.power_pct); + let power_pct = ch.power_pct.map(|p| p.clamp(crate::sweep::MIN_PCT, 100)).unwrap_or(c.power_pct); if c.vendor == "nvidia" && power_pct != c.power_pct { c.power_pct = power_pct; c.power_applied = false; power_changed = true; + // a cap set by hand is pinned: the sweep records MH per watt but leaves it (unpin on the tile) + c.pinned = true; + if self.sweep.as_ref().map(|r| r.card == c.index).unwrap_or(false) { + pin_aborts = true; + } } - settings.cards.insert(ch.key.clone(), crate::config::CardPref { enabled, identities, power_pct: if c.vendor == "nvidia" { power_pct } else { 0 } }); + let prev = settings.cards.get(&ch.key).cloned().unwrap_or_default(); + settings.cards.insert(ch.key.clone(), crate::config::CardPref { enabled, identities, power_pct: if c.vendor == "nvidia" { power_pct } else { 0 }, pinned: c.pinned, ..prev }); if c.enabled != enabled || c.identities != identities { c.enabled = enabled; c.identities = identities; @@ -1163,6 +1297,9 @@ impl Engine { self.shared.event("ok", &format!("{name}: switched on, {identities} identit{}", if identities == 1 { "y" } else { "ies" })); } } + if pin_aborts { + self.sweep_abort("the cap was set by hand"); + } if self.running { self.plan_miners(); if power_changed { @@ -1177,12 +1314,17 @@ impl Engine { if self.power_busy { return; } + if self.sweep.is_some() || self.sweep_pending.is_some() { + // the sweep owns the caps until it ends; it applies the chosen one itself + self.shared.log(&format!("power cap ({why}): deferred, a sweep is running")); + return; + } let mut cmds = Vec::new(); let mut what = Vec::new(); { let mut st = self.st(); for c in st.mining.cards.iter_mut().filter(|c| c.vendor == "nvidia" && c.enabled && c.power_default_w > 0.0) { - let pct = if c.power_pct == 0 { 80 } else { c.power_pct.clamp(60, 100) }; + let pct = if c.power_pct == 0 { 80 } else { c.power_pct.clamp(crate::sweep::MIN_PCT, 100) }; c.power_pct = pct; let watts = requested_watts(c); if (c.power_limit_w - watts).abs() < 1.0 && c.power_applied { @@ -1300,6 +1442,7 @@ impl Engine { } let f = |s: &str| s.parse::().unwrap_or(0.0); let (draw, tgpu, tmem, limit) = (f(p[1]), f(p[2]), f(p[3]), f(p[4])); + let sweeping = self.sweep.is_some() || self.sweep_pending.is_some(); let mut st = self.st(); let Some(c) = st.mining.cards.iter_mut().find(|c| c.vendor == "nvidia" && c.device == p[0]) else { return }; c.power_w = draw; @@ -1307,11 +1450,22 @@ impl Engine { c.temp_mem = tmem; if limit > 0.0 { c.power_limit_w = limit; + // the readback is the truth: a cap the elevated step set after the judgement (or one set by hand + // outside the app) that matches what was asked counts as applied + if !sweeping && !c.power_applied && c.enabled && c.power_default_w > 0.0 && (limit - requested_watts(c)).abs() < 1.0 { + c.power_applied = true; + c.power_note = format!("power cap: {} W in force (read back)", limit as u64); + } } c.telemetry_at = crate::platform::unix_now_f(); let idx = c.index; let mining = c.state == "mining"; drop(st); + if let Some(run) = self.sweep.as_mut() { + if run.card == idx { + run.sample_draw(draw); + } + } if mining { let s = self.stability.entry(idx).or_default(); s.draws.push(draw); @@ -1349,6 +1503,378 @@ impl Engine { } } + // ---- the efficiency sweep (src/sweep.rs) --------------------------------------------------------------------- + + /// A SWEEP line: the app log, and stdout under --sweep (the PC job reads it there). + fn sweep_say(&self, line: &str) { + self.shared.log(line); + if self.shared.runtime.sweep_only { + println!("{line}"); + let _ = std::io::stdout().flush(); + } + } + + fn sweep_dir(&self) -> PathBuf { + self.shared.runtime.app_dir.join("sweep") + } + + /// Every 10 s: drive the running sweep, else start one that is due. One card at a time. Never while a remote + /// job holds the GPU, never under a pause, never inside the last 10 minutes before the hour boundary. + fn tick_sweep(&mut self, now: Instant) { + if self.sweep.is_some() { + self.sweep_drive(now); + return; + } + if self.sweep_pending.is_some() || now < self.sweep_next_check { + return; + } + self.sweep_next_check = now + Duration::from_secs(10); + if self.quitting || !self.running || self.power_busy || self.job_hold || self.jobs.holds_miners() { + return; + } + let (auto_on, paused) = { + let s = self.shared.settings.lock().unwrap(); + let st = self.st(); + (s.sweep, st.mining.paused) + }; + if paused { + return; + } + let unix = crate::platform::unix_now(); + let (eta_known, eta_s) = { + let st = self.st(); + (st.node.daa > 0 && st.program.boundary_daa > 0, st.program.eta_s) + }; + let boundary_ok = !eta_known || eta_s > crate::sweep::NEEDS_S; + let prefs = self.shared.settings.lock().unwrap().cards.clone(); + let cards = self.st().mining.cards.clone(); + let mut pick: Option<(usize, bool)> = None; + for c in cards.iter() { + if !(c.enabled && c.sweep_supported && c.vendor == "nvidia") { + continue; + } + let forced = self.sweep_queue.contains(&c.key); + let due = prefs.get(&c.key).map(|p| p.sweep_at == 0 || unix.saturating_sub(p.sweep_at) >= crate::sweep::PERIOD_S).unwrap_or(true); + if !forced && !(auto_on && !c.pinned && due) { + continue; + } + if self.sweep_retry.get(&c.index).map(|t| now < *t).unwrap_or(false) { + continue; + } + let stable = self.miners.iter().find(|m| m.card == c.index).map(|m| m.proc.as_ref().map(|p| p.started.elapsed() >= Duration::from_secs(crate::sweep::STABLE_S)).unwrap_or(false) && m.last_status.is_some()).unwrap_or(false); + let wait = if c.state != "mining" { + Some(format!("sweep: waits for the card to mine ({})", c.state)) + } else if !stable { + Some(format!("sweep: waits for {} s of steady mining", crate::sweep::STABLE_S)) + } else if !boundary_ok { + Some(format!("sweep: starts after the hour boundary ({eta_s} s)")) + } else { + None + }; + match wait { + Some(w) => { + if forced { + if let Some(cc) = self.st().mining.cards.get_mut(c.index) { + cc.sweep_note = w; + } + } + } + None => { + pick = Some((c.index, forced)); + break; + } + } + } + if let Some((idx, forced)) = pick { + self.sweep_begin(idx, forced); + } else if self.shared.runtime.sweep_only && self.sweep_queue.is_empty() && self.sweep_pending.is_none() { + self.shared.send(Cmd::Quit); + } + } + + /// Step 1 of a sweep: find out how caps can be set from this process (one nvidia-smi -pl at the current limit, + /// a no-op). Elevated engines (the PC job runs with --elevated) set them directly; others need the helper. + fn sweep_begin(&mut self, idx: usize, forced: bool) { + let Some(c) = self.st().mining.cards.get(idx).cloned() else { return }; + self.sweep_pending = Some((idx, forced)); + self.sweep_queue.retain(|k| k != &c.key); + *self.sweep_attempts.entry(idx).or_default() += 1; + if let Some(cc) = self.st().mining.cards.get_mut(idx) { + cc.sweep_state = "running".into(); + cc.sweep_note = "sweep: checking how the cap is set".into(); + } + if let Some(direct) = self.sweep_direct { + self.sweep_mode_known(Ok(direct)); + return; + } + let smi = crate::platform::tool("nvidia-smi"); + let device = c.device.clone(); + let current = if c.power_limit_w > 0.0 { c.power_limit_w } else { c.power_default_w }.round() as u64; + let shared = self.shared.clone(); + std::thread::spawn(move || { + let r = match crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "-pl", ¤t.to_string()]), None, Duration::from_secs(20)) { + Some(out) => Ok(out.contains("All done")), + None => Err("nvidia-smi did not answer".to_string()), + }; + shared.send(Cmd::SweepCapMode(r)); + }); + } + + /// Step 2: start the helper when needed, then the run. + fn sweep_mode_known(&mut self, r: Result) { + let Some((idx, forced)) = self.sweep_pending else { return }; + let direct = match r { + Ok(d) => d, + Err(e) => { + self.sweep_abort(&format!("cannot set caps: {e}")); + return; + } + }; + self.sweep_direct = Some(direct); + let Some(c) = self.st().mining.cards.get(idx).cloned() else { + self.sweep_pending = None; + return; + }; + if !direct && !self.sweep_helper { + if let Err(e) = self.sweep_helper_start(&c) { + self.sweep_abort(&format!("the elevated helper could not start: {e}")); + return; + } + } + let steps = crate::sweep::plan_steps(c.power_default_w, c.power_min_w, c.power_max_w); + if steps.is_empty() { + self.sweep_abort("no steps: the card reported no default power limit"); + return; + } + let label = self.miners.iter().find(|m| m.card == idx).map(|m| m.label.clone()).unwrap_or_else(|| format!("card-{idx}")); + let before_w = if c.power_limit_w > 0.0 { c.power_limit_w } else { requested_watts(&c) }; + let now = Instant::now(); + let run = crate::sweep::Run::new(idx, &c.key, &c.device, &label, steps.clone(), c.power_pct, before_w, forced, crate::sweep::Timing::from_env(), now); + self.sweep_pending = None; + self.sweep_say(&format!( + "SWEEP start card={label} name={} steps={} default={:.0} min={:.0} max={:.0} before={:.0} mode={}", + c.name.replace(' ', "_"), + steps.iter().map(|s| s.pct.to_string()).collect::>().join(","), + c.power_default_w, + c.power_min_w, + c.power_max_w, + before_w, + if direct { "direct" } else { "helper" } + )); + self.shared.event("info", &format!("{}: efficiency sweep started: {} caps from {}% down, {} s each on the live program", c.name, steps.len(), steps[0].pct, (run.timing.settle + run.timing.hold).as_secs())); + self.sweep = Some(run); + self.sweep_drive(now); + } + + /// The elevated helper (src/sweep.rs helper_script_*): one administrator prompt; it polls /sweep/cmd.txt. + fn sweep_helper_start(&mut self, c: &CardState) -> Result<(), String> { + let dir = self.sweep_dir(); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + let _ = std::fs::write(dir.join("cmd.txt"), ""); + let restore = (if c.power_limit_w > 0.0 { c.power_limit_w } else { c.power_default_w }).round() as u64; + let smi = crate::platform::tool("nvidia-smi").display().to_string(); + let line = if cfg!(windows) { + let script = dir.join("helper.ps1"); + std::fs::write(&script, [b"\xEF\xBB\xBF".as_slice(), crate::sweep::helper_script_windows().replace('\n', "\r\n").as_bytes()].concat()).map_err(|e| e.to_string())?; + format!("\"{}\" -NoProfile -ExecutionPolicy Bypass -File \"{}\" \"{}\" \"{}\" {} {}", crate::platform::tool("powershell").display(), script.display(), dir.display(), smi, c.device, restore) + } else { + let script = dir.join("helper.sh"); + std::fs::write(&script, crate::sweep::helper_script_unix()).map_err(|e| e.to_string())?; + format!("sh \"{}\" \"{}\" \"{}\" {} {}", script.display(), dir.display(), smi, c.device, restore) + }; + self.shared.log(&format!("sweep helper (administrator prompt): {line}")); + self.sweep_helper = true; + let shared = self.shared.clone(); + std::thread::spawn(move || { + let r = crate::platform::run_elevated(&line); + shared.send(Cmd::SweepHelperDone(r)); + }); + Ok(()) + } + + fn sweep_helper_quit(&mut self) { + if self.sweep_helper { + let _ = std::fs::write(self.sweep_dir().join("cmd.txt"), "quit\n"); + } + } + + /// Sets a cap for the sweep: directly (this process is elevated) or through the helper's command file. The + /// telemetry readback (power.limit every 5 s) tells the run when it took. + fn sweep_set_cap(&mut self, device: &str, watts: f64) { + self.sweep_seq += 1; + let w = watts.round() as u64; + if self.sweep_direct == Some(true) { + let smi = crate::platform::tool("nvidia-smi"); + let device = device.to_string(); + let shared = self.shared.clone(); + std::thread::spawn(move || { + let out = crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "-pl", &w.to_string()]), None, Duration::from_secs(20)).unwrap_or_else(|| "nvidia-smi did not answer".into()); + shared.send(Cmd::SweepCapSet(out)); + }); + } else { + let _ = std::fs::write(self.sweep_dir().join("cmd.txt"), format!("{} {w}\n", self.sweep_seq)); + } + self.shared.log(&format!("sweep: cap {w} W requested on device {device}")); + } + + /// Faults and conditions first, then the state machine. + fn sweep_drive(&mut self, now: Instant) { + let Some((idx, device, started)) = self.sweep.as_ref().map(|r| (r.card, r.device.clone(), r.started)) else { return }; + let card = self.st().mining.cards.get(idx).cloned(); + let Some(c) = card else { + self.sweep_abort("the card disappeared"); + return; + }; + let slot_error = self.miners.iter().find(|m| m.card == idx).and_then(|m| m.error_at).map(|t| t > started).unwrap_or(false); + let fault = if self.quitting { + Some("the app is quitting".to_string()) + } else if self.job_hold || self.jobs.holds_miners() { + Some("a remote job took the GPU".into()) + } else if self.st().mining.paused { + Some("mining paused".into()) + } else if c.state != "mining" { + Some(format!("the card left mining ({})", if c.state == "starting" || c.state == "restarting" { "worker restart, likely the hour boundary" } else { &c.state })) + } else if slot_error { + Some("the worker reported an error".into()) + } else if c.temp_gpu >= 90.0 { + Some(format!("GPU at {:.0} C", c.temp_gpu)) + } else { + None + }; + if let Some(why) = fault { + self.sweep_abort(&why); + return; + } + let outs = self.sweep.as_mut().map(|r| r.tick(now, c.power_limit_w)).unwrap_or_default(); + let label = self.sweep.as_ref().map(|r| r.label.clone()).unwrap_or_default(); + for o in outs { + match o { + crate::sweep::Out::Apply(w) => self.sweep_set_cap(&device, w), + crate::sweep::Out::Row(row) => { + self.sweep_say(&row.line(&label)); + if let Some(cc) = self.st().mining.cards.get_mut(idx) { + if row.usable() { + cc.sweep_note = format!("sweep: {}% done · {:.3} MH/W", row.pct, row.eff); + } + } + } + crate::sweep::Out::Finished(row) => { + self.sweep_finish(row); + return; + } + crate::sweep::Out::Failed(e) => { + self.sweep_abort(&e); + return; + } + } + } + if let Some(words) = self.sweep.as_ref().map(|r| r.words(now)) { + if let Some(cc) = self.st().mining.cards.get_mut(idx) { + cc.sweep_note = words; + } + } + } + + /// The chosen cap is in force: record it on the card and in the settings, say so, let the helper go. + fn sweep_finish(&mut self, row: crate::sweep::Row) { + let Some(run) = self.sweep.take() else { return }; + let idx = run.card; + let unix = crate::platform::unix_now(); + let (name, pinned, pinned_w) = { + let mut settings = self.shared.settings.lock().unwrap(); + let mut st = self.st(); + let Some(c) = st.mining.cards.get_mut(idx) else { return }; + c.sweep_pct = row.pct; + c.sweep_eff = row.eff; + c.sweep_watts = row.watts; + c.sweep_mhs = row.mhs; + c.sweep_at = unix as f64; + c.sweep_state = "idle".into(); + let pinned_w = requested_watts(c); + if c.pinned { + c.sweep_note = format!("sweep: best {}% · {:.3} MH/W; cap pinned at {}%", row.pct, row.eff, c.power_pct); + } else { + c.power_pct = row.pct; + c.power_limit_w = row.limit; + c.power_applied = true; + c.power_note = format!("power cap: {} W held by the sweep (best MH per watt)", row.limit as u64); + c.sweep_note = format!("sweep: best {}% · {:.3} MH/W", row.pct, row.eff); + } + let e = settings.cards.entry(c.key.clone()).or_insert_with(|| crate::config::CardPref { enabled: c.enabled, identities: c.identities, ..Default::default() }); + e.sweep_at = unix; + e.sweep_pct = row.pct; + e.sweep_eff = row.eff; + e.sweep_watts = row.watts; + e.sweep_mhs = row.mhs; + if !c.pinned { + e.power_pct = row.pct; + } + (c.name.clone(), c.pinned, pinned_w) + }; + self.shared.save_settings(); + self.sweep_say(&format!("SWEEP chosen card={} cap={} limit={:.0} watts={:.1} mhs={:.2} eff={:.4}{}", run.label, row.pct, row.limit, row.watts, row.mhs, row.eff, if pinned { " pinned=1" } else { "" })); + if pinned { + self.sweep_set_cap(&run.device.clone(), pinned_w); + self.shared.event("ok", &format!("{name}: best efficiency at {}% ({:.0} W cap): {:.1} MH/s at {:.0} W = {:.3} MH/W. The cap stays pinned where you set it.", row.pct, row.limit, row.mhs, row.watts, row.eff)); + } else { + self.shared.event("ok", &format!("{name}: best efficiency at {}% ({:.0} W cap): {:.1} MH/s at {:.0} W = {:.3} MH/W; cap held there", row.pct, row.limit, row.mhs, row.watts, row.eff)); + } + self.sweep_helper_quit(); + if self.shared.runtime.sweep_only && self.sweep_queue.is_empty() { + self.shared.send(Cmd::Quit); + } + } + + /// Stops a sweep: the cap goes back to what it was, the log says why, the card waits an hour before the + /// scheduler tries again (a forced sweep under --sweep retries twice). + fn sweep_abort(&mut self, why: &str) { + let pending = self.sweep_pending.take(); + let run = self.sweep.take(); + let (idx, label, device, before_w, before_pct, forced) = match (&run, pending) { + (Some(r), _) => (r.card, r.label.clone(), r.device.clone(), r.before_w, r.before_pct, r.forced), + (None, Some((idx, forced))) => { + let c = self.st().mining.cards.get(idx).cloned(); + let (label, device, w, pct) = c.map(|c| (format!("card-{idx}"), c.device.clone(), if c.power_limit_w > 0.0 { c.power_limit_w } else { requested_watts(&c) }, c.power_pct)).unwrap_or_default(); + (idx, label, device, w, pct, forced) + } + _ => return, + }; + self.sweep_say(&format!("SWEEP aborted card={label} reason={}", why.replace(' ', "_"))); + let name = { + let mut st = self.st(); + match st.mining.cards.get_mut(idx) { + Some(c) => { + c.sweep_state = "idle".into(); + c.sweep_note = format!("sweep stopped: {why}"); + c.power_pct = before_pct; + c.power_applied = false; + c.power_note = format!("cap restored to {} W after the sweep stopped", before_w as u64); + c.name.clone() + } + None => label.clone(), + } + }; + if run.is_some() && before_w > 0.0 && !device.is_empty() { + self.sweep_set_cap(&device, before_w); + } + self.sweep_helper_quit(); + self.shared.event("error", &format!("{name}: efficiency sweep stopped ({why}); cap back to {} W", before_w as u64)); + let attempts = self.sweep_attempts.get(&idx).copied().unwrap_or(0); + if forced && self.shared.runtime.sweep_only && attempts < 3 && !self.quitting { + // the PC job: the hour boundary or a restart interrupted it; try again once the card is steady + let key = self.st().mining.cards.get(idx).map(|c| c.key.clone()); + if let Some(key) = key { + self.sweep_queue.push(key); + } + self.sweep_retry.insert(idx, Instant::now() + Duration::from_secs(30)); + } else { + self.sweep_retry.insert(idx, Instant::now() + Duration::from_secs(3600)); + if self.shared.runtime.sweep_only && self.sweep_queue.is_empty() { + self.shared.send(Cmd::Quit); + } + } + } + // ---- the tick ---------------------------------------------------------------------------------------------- fn tick(&mut self) { @@ -1362,6 +1888,7 @@ impl Engine { self.tick_watch(now); self.tick_miners(now); self.tick_telemetry(now); + self.tick_sweep(now); } self.derive(now); if now.duration_since(self.last_status) >= Duration::from_secs(self.shared.runtime.status_secs as u64) { @@ -1727,10 +2254,12 @@ impl Engine { let mut st = self.st(); st.node.last_reading_age_s = self.node_last_reading.map(|t| now.duration_since(t).as_secs_f64()).unwrap_or(-1.0); let mut total = 0.0; - for c in st.mining.cards.iter() { + for c in st.mining.cards.iter_mut() { if c.state == "mining" { total += c.hash_now; } + // live hash per watt (the number next to the draw on the tile); needs a draw reading under a minute old + c.eff_mhw = if c.state == "mining" && c.power_w > 1.0 && c.hash_now > 0.0 && unix - c.telemetry_at < 60.0 { c.hash_now / c.power_w } else { 0.0 }; } st.mining.hash_total = total; let cut = unix - 3600.0; @@ -2014,6 +2543,11 @@ impl Engine { let avg = kv_f64(text, "hash").unwrap_or(0.0); // the v4 miner's interval rate: "now=12.34 MH/s wall (...)"; older miners: the average let now_rate = kv_f64(text, "now").unwrap_or(avg); + if let Some(run) = self.sweep.as_mut() { + if run.card == card { + run.sample_rate(now_rate); + } + } let mut st = self.st(); if let Some(c) = st.mining.cards.get_mut(card) { c.hash_avg = avg; @@ -2155,12 +2689,21 @@ impl Engine { fn shutdown(&mut self) { self.shared.log("quit: stopping the miners, then the node"); self.jobs.abort(&self.shared, "the app is quitting"); + if self.sweep.is_some() || self.sweep_pending.is_some() { + self.sweep_abort("the app is quitting"); + } + self.sweep_helper_quit(); self.stop_miners("quit"); self.stability_line(); if let Some(mut t) = self.telemetry.take() { t.stop(2); } - self.restore_power_limits(); + if self.shared.runtime.sweep_only { + // the measurement leaves the chosen cap in force for the app that takes the card back + self.shared.log("--sweep: the chosen caps stay in force (not restored)"); + } else { + self.restore_power_limits(); + } if let Some(mut w) = self.watch.take() { w.stop(2); } @@ -2193,7 +2736,7 @@ impl Engine { /// The watts a card's cap asks for: power_pct of the default limit, inside the card's min and max. fn requested_watts(c: &CardState) -> f64 { - let pct = if c.power_pct == 0 { 80 } else { c.power_pct.clamp(60, 100) }; + let pct = if c.power_pct == 0 { 80 } else { c.power_pct.clamp(crate::sweep::MIN_PCT, 100) }; let mut w = c.power_default_w * pct as f64 / 100.0; if c.power_min_w > 0.0 { w = w.max(c.power_min_w); diff --git a/app/igneum-app/src/main.rs b/app/igneum-app/src/main.rs index 48c749988..9d67751ec 100644 --- a/app/igneum-app/src/main.rs +++ b/app/igneum-app/src/main.rs @@ -3,9 +3,11 @@ //! --wrapper, reads `URL ...` and `STATE {...}` lines from its stdout and writes `quit` on its stdin. Without a host //! (--open) the dashboard opens in the default browser. //! -//! igneum-app [--wrapper | --open | --no-open | --launch] [--print-url] +//! igneum-app [--wrapper | --open | --no-open | --launch | --sweep] [--print-url] //! --launch is what the Start Menu entry runs on Windows: it hands over to "Igneum Miner.exe" (the window host) when //! that is installed next to the engine, else runs the engine with the dashboard in the default browser. +//! --sweep runs the efficiency sweep (src/sweep.rs) on every supported card as soon as it mines, prints the SWEEP +//! table on stdout, leaves the chosen caps in force and quits; no browser opens (the PC measurement job). //! //! Environment (tests): IGNEUM_APP_NETWORK devnet|simnet, IGNEUM_APP_RPC_PORT, IGNEUM_APP_P2P_PORT, IGNEUM_APP_PEERS //! (comma list, empty for none), IGNEUM_APP_UNSYNCED=1, IGNEUM_APP_DATA, IGNEUM_APP_LOGS, IGNEUM_APP_BIN, IGNEUM_APP_NODE_DIR. @@ -25,6 +27,7 @@ mod update; mod jobs; mod jobrun; mod prover; +mod sweep; use std::io::{BufRead, Write}; use std::sync::mpsc::channel; @@ -33,7 +36,12 @@ use std::sync::Arc; fn main() { let args: Vec = std::env::args().skip(1).collect(); let wrapper = args.iter().any(|a| a == "--wrapper"); - let no_open = wrapper || args.iter().any(|a| a == "--no-open"); + let sweep = args.iter().any(|a| a == "--sweep"); + if sweep { + // the runtime reads it (config::Runtime::from_env); the engine starts at once and quits after the sweep + std::env::set_var("IGNEUM_APP_SWEEP", "1"); + } + let no_open = wrapper || sweep || args.iter().any(|a| a == "--no-open"); if args.iter().any(|a| a == "--version" || a == "-V") { println!("igneum-app {}", engine::VERSION); return; diff --git a/app/igneum-app/src/server.rs b/app/igneum-app/src/server.rs index ddd271e97..3b7c0ff5e 100644 --- a/app/igneum-app/src/server.rs +++ b/app/igneum-app/src/server.rs @@ -244,7 +244,7 @@ fn api_post(shared: &Arc, path: &str, body: Value) -> Result, path: &str, body: Value) -> Result { + let key = s("key").ok_or("key missing")?; + shared.send(Cmd::SweepStart(key)); + Ok(json!({ "ok": true })) + } + "/api/sweep/stop" => { + shared.send(Cmd::SweepStop); + Ok(json!({ "ok": true })) + } + "/api/sweep/pin" => { + let key = s("key").ok_or("key missing")?; + let pinned = body.get("pinned").and_then(|v| v.as_bool()).ok_or("pinned missing")?; + shared.send(Cmd::SweepPin(key, pinned)); + Ok(json!({ "ok": true })) + } + "/api/sweep/enable" => { + let on = body.get("on").and_then(|v| v.as_bool()).ok_or("on missing")?; + shared.send(Cmd::SweepEnable(on)); + Ok(json!({ "ok": true })) + } "/api/clock/sync" => { shared.send(Cmd::ClockSync); Ok(json!({ "ok": true })) diff --git a/app/igneum-app/src/state.rs b/app/igneum-app/src/state.rs index b5b25b559..9df1ca188 100644 --- a/app/igneum-app/src/state.rs +++ b/app/igneum-app/src/state.rs @@ -64,12 +64,23 @@ pub struct CardState { pub power_min_w: f64, pub power_max_w: f64, pub power_before_w: f64, // the limit before the app touched it (restored on quit) - pub power_pct: u32, // the chosen cap, percent of the default (60 to 100) + pub power_pct: u32, // the chosen cap, percent of the default (50 to 100) pub power_applied: bool, pub power_note: String, pub temp_gpu: f64, pub temp_mem: f64, pub telemetry_at: f64, + // hash per watt (src/sweep.rs) + pub eff_mhw: f64, // live: hash_now over power_w, MH per watt; 0 = unknown + pub sweep_supported: bool, // NVIDIA with readable limits; the note says why not otherwise + pub sweep_state: String, // idle | running | unsupported + pub sweep_note: String, // the phase in words while running, else why unsupported or the last outcome + pub sweep_pct: u32, // the cap the last sweep chose (0 = never swept) + pub sweep_eff: f64, // MH per watt at that cap + pub sweep_watts: f64, + pub sweep_mhs: f64, + pub sweep_at: f64, // unix s of the last sweep + pub pinned: bool, // the user set the cap by hand; the sweep records but does not change it } #[derive(Clone, Serialize, Default)] @@ -158,6 +169,8 @@ pub struct SettingsState { pub remote_jobs: bool, /// the prover service (src/prover.rs) pub prove: bool, + /// the efficiency sweep (src/sweep.rs): once after install, then weekly + pub sweep: bool, } /// One remote job this machine ran (the ledger entry), for the Settings history and the last-job strip. diff --git a/app/igneum-app/src/sweep.rs b/app/igneum-app/src/sweep.rs new file mode 100644 index 000000000..52c0adbbf --- /dev/null +++ b/app/igneum-app/src/sweep.rs @@ -0,0 +1,593 @@ +//! The efficiency sweep: hash per watt, the number miners compare. For one NVIDIA card at a time the engine steps +//! the power cap from 100% of the card's default limit down to 50% in 10% steps, holds each step for 60 s after a +//! 15 s settle, averages the card's reported draw (nvidia-smi power.draw, the telemetry child) and the worker's +//! interval rate (the STATUS lines), computes MH per watt per step, picks the best step and leaves the cap there. +//! It runs on the live kernel: the worker is never restarted and the hour's program is never lost. It never starts +//! while a remote job holds the GPU, and it aborts the moment the card leaves "mining" (a fault, an hour boundary, +//! a pause, a job). +//! +//! This file is the logic: the step plan, the per-step rows, the choice, the parser for nvidia-smi's power query and +//! the state machine, all driven by an explicit clock so the tests run without a card. The engine (src/engine.rs, +//! `tick_sweep` and the `Cmd::Sweep*` commands) owns the processes: it sets the cap (directly when the engine runs +//! elevated, else through one elevated helper that polls a command file, so a sweep costs one administrator prompt +//! instead of six), reads the limit back through the telemetry child, feeds the samples in and writes the table to +//! the app log: +//! SWEEP start card=