diff --git a/app/igneum-app/src/config.rs b/app/igneum-app/src/config.rs index 966315868..30e034ebc 100644 --- a/app/igneum-app/src/config.rs +++ b/app/igneum-app/src/config.rs @@ -12,9 +12,23 @@ use std::path::{Path, PathBuf}; pub struct CardPref { pub enabled: bool, pub identities: u32, - /// NVIDIA power cap, percent of the card's default limit (60 to 100); 0 = the default 80 + /// NVIDIA power cap, percent of the card's default limit (50 to 100); 0 = the default 80 #[serde(default)] pub power_pct: u32, + /// the user moved the slider: this cap stays; the efficiency sweep (src/sweep.rs) records but does not change it + #[serde(default)] + pub pinned: bool, + /// the last efficiency sweep: when (unix s), the cap it chose, and that step's numbers + #[serde(default)] + pub sweep_at: u64, + #[serde(default)] + pub sweep_pct: u32, + #[serde(default)] + pub sweep_eff: f64, + #[serde(default)] + pub sweep_watts: f64, + #[serde(default)] + pub sweep_mhs: f64, } #[derive(Clone, Serialize, Deserialize)] @@ -55,6 +69,13 @@ pub struct Settings { /// Prove the shards assigned to this machine's keys (src/prover.rs). Default off until the GPU numbers exist. #[serde(default)] pub prove: bool, + /// The efficiency sweep (src/sweep.rs): once after install, then weekly, each NVIDIA card's cap is stepped from + /// 100% to 50% on the live program and held at the best MH per watt. Default on. A pinned card is skipped. + #[serde(default = "yes")] + pub sweep: bool, + /// When this install first ran (unix s), for the "first hour after install" sweep. + #[serde(default)] + pub installed_at: u64, } fn one() -> u32 { @@ -66,13 +87,19 @@ fn yes() -> bool { impl Default for Settings { fn default() -> Settings { - Settings { setup_done: false, address: String::new(), address_source: String::new(), key_saved: false, identities: 1, cards: HashMap::new(), display_name: String::new(), vote: true, paused: false, accepted_total: 0, auto_update: true, remote_jobs: true, prove: false } + Settings { setup_done: false, address: String::new(), address_source: String::new(), key_saved: false, identities: 1, cards: HashMap::new(), display_name: String::new(), vote: true, paused: false, accepted_total: 0, auto_update: true, remote_jobs: true, prove: false, sweep: true, installed_at: 0 } } } impl Settings { pub fn load(path: &Path) -> Settings { - std::fs::read_to_string(path).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default() + let mut s: Settings = std::fs::read_to_string(path).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default(); + if s.installed_at == 0 { + // an install from before the sweep existed counts as installed now: it gets its first-hour sweep + s.installed_at = crate::platform::unix_now(); + s.save(path); + } + s } pub fn save(&self, path: &Path) { if let Some(d) = path.parent() { @@ -148,6 +175,9 @@ pub struct Runtime { pub app_dir: PathBuf, pub log_dir: PathBuf, pub status_secs: u32, + /// `--sweep`: run the efficiency sweep on every supported card as soon as it mines, print the table on stdout, + /// leave the chosen caps in force and quit (the PC measurement job; docs/plans/miner-eff.md). + pub sweep_only: bool, /// The machine's hostname: a friendly label only, never part of a key or a payout address. pub host: String, /// Per-install random id (16 hex, app data dir/machine-id, locked to the user). Identity labels, vote keys and @@ -181,7 +211,8 @@ impl Runtime { let log_dir = crate::platform::log_root(); let status_secs = env("IGNEUM_APP_STATUS_SECS").and_then(|v| v.parse().ok()).unwrap_or(30); let machine_id = machine_id(&app_dir); - Runtime { network, rpc_port, p2p_port, peers, unsynced_mining, devnet_suffix, node_dir, app_dir, log_dir, status_secs, host: crate::platform::host_label(), machine_id } + let sweep_only = env("IGNEUM_APP_SWEEP").map(|v| v == "1").unwrap_or(false); + Runtime { network, rpc_port, p2p_port, peers, unsynced_mining, devnet_suffix, node_dir, app_dir, log_dir, status_secs, sweep_only, host: crate::platform::host_label(), machine_id } } /// The first 8 hex of the machine id: what goes into labels. pub fn id8(&self) -> String { diff --git a/app/igneum-app/src/detect.rs b/app/igneum-app/src/detect.rs index fd2e1b521..1af73c031 100644 --- a/app/igneum-app/src/detect.rs +++ b/app/igneum-app/src/detect.rs @@ -99,6 +99,34 @@ pub fn apply_defaults(c: &mut CardState) { } } +/// Marks whether the efficiency sweep (src/sweep.rs) can run on a card, with the reason when it cannot. Called +/// once the power limits are known. +pub fn mark_sweep_support(c: &mut CardState) { + match crate::sweep::unsupported_reason(&c.vendor, c.power_default_w, &c.device) { + None => { + c.sweep_supported = true; + if c.sweep_state.is_empty() || c.sweep_state == "unsupported" { + c.sweep_state = "idle".into(); + c.sweep_note = String::new(); + } + } + Some(why) => { + c.sweep_supported = false; + c.sweep_state = "unsupported".into(); + c.sweep_note = why.into(); + } + } +} + +/// `nvidia-smi --query-gpu=index,power.draw`: index -> watts now (the one-shot form; the telemetry child streams it). +#[cfg(not(target_os = "macos"))] +#[allow(dead_code)] +pub fn nvidia_power_draw() -> std::collections::HashMap { + run_timeout(Command::new(crate::platform::tool("nvidia-smi")).args(["--query-gpu=index,power.draw", "--format=csv,noheader,nounits"]), None, Duration::from_secs(10)) + .map(|t| crate::sweep::parse_power_draw(&t)) + .unwrap_or_default() +} + /// NVIDIA power limits per card index: (default, current, min, max) in watts. #[cfg(target_os = "macos")] pub fn nvidia_power_limits() -> std::collections::HashMap { @@ -163,6 +191,7 @@ pub fn detect(bins: &Bins, notes: &mut Vec) -> Vec { c.vram_mb = mem.trim().parse::().map(|b| b / (1024 * 1024)).unwrap_or(0); } apply_defaults(&mut c); + mark_sweep_support(&mut c); cards.push(c); cards } @@ -203,6 +232,7 @@ pub fn detect(bins: &Bins, notes: &mut Vec) -> Vec { c.power_min_w = *lo; c.power_max_w = *hi; } + mark_sweep_support(c); } } None => notes.push("nvidia-smi is not on this PC (no NVIDIA driver): no NVIDIA card".into()), @@ -233,6 +263,7 @@ pub fn detect(bins: &Bins, notes: &mut Vec) -> Vec { c.kind = if looks_integrated(name) { "integrated".into() } else { "discrete".into() }; c.path = "prebuilt".into(); apply_defaults(&mut c); + mark_sweep_support(&mut c); cards.push(c); } } diff --git a/app/igneum-app/src/engine.rs b/app/igneum-app/src/engine.rs index fe4b8e20c..4029c2f9c 100644 --- a/app/igneum-app/src/engine.rs +++ b/app/igneum-app/src/engine.rs @@ -58,6 +58,18 @@ pub enum Cmd { ElevatedDone(Result<(), String>), /// the user asked for the cap again (the Retry button) ApplyPower, + /// the efficiency sweep (src/sweep.rs): start on one card (by key), stop, pin or unpin a card's cap, the toggle + SweepStart(String), + SweepStop, + SweepPin(String, bool), + SweepEnable(bool), + /// the probe that decides how caps are set during a sweep: Ok(true) = nvidia-smi -pl works from this process + /// (the engine runs elevated), Ok(false) = the elevated helper is needed; Err = nvidia-smi did not answer + SweepCapMode(Result), + /// the elevated helper process ended (Ok at `quit`, Err when the prompt was cancelled or it failed) + SweepHelperDone(Result<(), String>), + /// a direct `nvidia-smi -pl` for the sweep finished: what it printed + SweepCapSet(String), Quit, } @@ -97,7 +109,7 @@ impl Shared { st.mining.paused = settings.paused; st.mining.accepted_total = settings.accepted_total; st.address = address_state(&settings, &wallet_path); - st.settings = crate::state::SettingsState { identities: settings.identities, vote: settings.vote, start_at_login: crate::platform::start_at_login_is_on(), auto_update: settings.auto_update, remote_jobs: settings.remote_jobs, prove: settings.prove }; + st.settings = crate::state::SettingsState { identities: settings.identities, vote: settings.vote, start_at_login: crate::platform::start_at_login_is_on(), auto_update: settings.auto_update, remote_jobs: settings.remote_jobs, prove: settings.prove, sweep: settings.sweep }; st.live_page = packaged.live_page.clone(); st.finality.message = "waiting for the miner".into(); st.clock.hint = crate::platform::clock_hint().into(); @@ -416,6 +428,21 @@ pub struct Engine { last_stability: Instant, last_settings_save: Instant, last_error_event: Instant, + // the efficiency sweep (src/sweep.rs): one card at a time + sweep: Option, + /// a sweep waiting for the cap-mode probe: (card index, forced) + sweep_pending: Option<(usize, bool)>, + /// how caps are set: Some(true) directly, Some(false) through the elevated helper, None = not probed yet + sweep_direct: Option, + /// the elevated helper is running (its thread reports SweepHelperDone when it ends) + sweep_helper: bool, + sweep_seq: u64, + /// cards asked for by hand (Sweep now) or by --sweep, by key + sweep_queue: Vec, + sweep_next_check: Instant, + /// after an abort, a card waits before the scheduler tries it again + sweep_retry: std::collections::HashMap, + sweep_attempts: std::collections::HashMap, } impl Engine { @@ -487,6 +514,15 @@ impl Engine { last_stability: now, last_settings_save: now, last_error_event: now - Duration::from_secs(600), + sweep: None, + sweep_pending: None, + sweep_direct: None, + sweep_helper: false, + sweep_seq: 0, + sweep_queue: Vec::new(), + sweep_next_check: now + Duration::from_secs(20), + sweep_retry: std::collections::HashMap::new(), + sweep_attempts: std::collections::HashMap::new(), } } @@ -530,7 +566,10 @@ impl Engine { } } self.shared.send(Cmd::Detect); - if setup_done && !self.quitting { + if self.shared.runtime.sweep_only { + self.shared.log("--sweep: the efficiency sweep runs on every supported card as soon as it mines; the table goes to stdout and this log; the engine quits after it"); + } + if (setup_done || self.shared.runtime.sweep_only) && !self.quitting { self.shared.send(Cmd::Start); } loop { @@ -582,11 +621,30 @@ impl Engine { c.identities = p.identities.clamp(1, 64); c.reason = String::new(); if c.vendor == "nvidia" && p.power_pct > 0 { - c.power_pct = p.power_pct.clamp(60, 100); + c.power_pct = p.power_pct.clamp(crate::sweep::MIN_PCT, 100); } + c.pinned = p.pinned; + c.sweep_pct = p.sweep_pct; + c.sweep_eff = p.sweep_eff; + c.sweep_watts = p.sweep_watts; + c.sweep_mhs = p.sweep_mhs; + c.sweep_at = p.sweep_at as f64; } } + let supported: Vec = st.mining.cards.iter().filter(|c| c.sweep_supported && c.enabled).map(|c| c.key.clone()).collect(); + let unsupported: Vec = st.mining.cards.iter().filter(|c| !c.sweep_supported).map(|c| format!("{}: {}", c.name, c.sweep_note)).collect(); drop(st); + for u in &unsupported { + self.shared.log(&format!("efficiency sweep {u}")); + } + if self.shared.runtime.sweep_only { + if supported.is_empty() { + self.sweep_say("SWEEP none reason=no_supported_card"); + self.shared.send(Cmd::Quit); + } else { + self.sweep_queue = supported; + } + } if self.running { self.plan_miners(); } @@ -768,11 +826,80 @@ impl Engine { } } Cmd::ApplyPower => { + if self.sweep.is_some() || self.sweep_pending.is_some() { + self.shared.event("info", "a sweep is setting the caps right now; retry once it is done"); + return; + } for c in self.st().mining.cards.iter_mut().filter(|c| c.vendor == "nvidia") { c.power_applied = false; // force the step again } self.apply_power_limits("retry"); } + Cmd::SweepStart(key) => { + let (name, ok, why) = { + let st = self.st(); + match st.mining.cards.iter().find(|c| c.key == key) { + Some(c) if !c.sweep_supported => (c.name.clone(), false, c.sweep_note.clone()), + Some(c) if !c.enabled => (c.name.clone(), false, "the card is switched off".to_string()), + Some(c) => (c.name.clone(), true, String::new()), + None => (key.clone(), false, "no such card".to_string()), + } + }; + if !ok { + self.shared.event("error", &format!("{name}: no sweep: {why}")); + } else if !self.sweep_queue.contains(&key) { + self.sweep_queue.push(key); + self.sweep_next_check = Instant::now(); + self.shared.event("info", &format!("{name}: efficiency sweep queued; it starts once the card has mined for {} s with over {} s to the hour boundary", crate::sweep::STABLE_S, crate::sweep::NEEDS_S)); + } + } + Cmd::SweepStop => { + self.sweep_queue.clear(); + if self.sweep.is_some() || self.sweep_pending.is_some() { + self.sweep_abort("stopped from the dashboard"); + } + } + Cmd::SweepPin(key, pinned) => { + let name = { + let mut settings = self.shared.settings.lock().unwrap(); + let mut st = self.st(); + let Some(c) = st.mining.cards.iter_mut().find(|c| c.key == key) else { return }; + c.pinned = pinned; + let e = settings.cards.entry(key.clone()).or_insert_with(|| crate::config::CardPref { enabled: c.enabled, identities: c.identities, ..Default::default() }); + e.pinned = pinned; + if pinned { + e.power_pct = c.power_pct; + } + c.name.clone() + }; + self.shared.save_settings(); + self.shared.event("info", &if pinned { format!("{name}: cap pinned; the sweep records but does not change it") } else { format!("{name}: the sweep chooses the cap again from its next run") }); + } + Cmd::SweepEnable(on) => { + self.shared.settings.lock().unwrap().sweep = on; + self.shared.save_settings(); + self.st().settings.sweep = on; + self.shared.event("info", if on { "efficiency sweep on: once after install, then weekly" } else { "efficiency sweep off; Sweep now on a card still runs one" }); + } + Cmd::SweepCapMode(r) => self.sweep_mode_known(r), + Cmd::SweepHelperDone(r) => { + self.sweep_helper = false; + if let Err(e) = &r { + self.shared.log(&format!("sweep helper: {e}")); + if self.sweep.is_some() || self.sweep_pending.is_some() { + self.sweep_abort(&format!("the elevated helper did not run ({e})")); + } + } else if self.sweep.is_some() { + self.sweep_abort("the elevated helper exited early"); + } else { + self.shared.log("sweep helper: exited"); + } + } + Cmd::SweepCapSet(text) => { + if !text.contains("All done") { + self.shared.log(&format!("sweep: nvidia-smi -pl answered: {}", short(&text.replace('\n', " "), 200))); + } + } Cmd::Quit => { self.quitting = true; self.st().quitting = true; @@ -1117,6 +1244,7 @@ impl Engine { fn apply_cards(&mut self, choices: Vec) { let mut changed = Vec::new(); let mut power_changed = false; + let mut pin_aborts = false; { let mut settings = self.shared.settings.lock().unwrap(); let mut st = self.st(); @@ -1124,13 +1252,19 @@ impl Engine { let Some(c) = st.mining.cards.iter_mut().find(|c| c.key == ch.key) else { continue }; let identities = ch.identities.clamp(1, 64); let enabled = ch.enabled && c.kind != "unknown"; - let power_pct = ch.power_pct.map(|p| p.clamp(60, 100)).unwrap_or(c.power_pct); + let power_pct = ch.power_pct.map(|p| p.clamp(crate::sweep::MIN_PCT, 100)).unwrap_or(c.power_pct); if c.vendor == "nvidia" && power_pct != c.power_pct { c.power_pct = power_pct; c.power_applied = false; power_changed = true; + // a cap set by hand is pinned: the sweep records MH per watt but leaves it (unpin on the tile) + c.pinned = true; + if self.sweep.as_ref().map(|r| r.card == c.index).unwrap_or(false) { + pin_aborts = true; + } } - settings.cards.insert(ch.key.clone(), crate::config::CardPref { enabled, identities, power_pct: if c.vendor == "nvidia" { power_pct } else { 0 } }); + let prev = settings.cards.get(&ch.key).cloned().unwrap_or_default(); + settings.cards.insert(ch.key.clone(), crate::config::CardPref { enabled, identities, power_pct: if c.vendor == "nvidia" { power_pct } else { 0 }, pinned: c.pinned, ..prev }); if c.enabled != enabled || c.identities != identities { c.enabled = enabled; c.identities = identities; @@ -1163,6 +1297,9 @@ impl Engine { self.shared.event("ok", &format!("{name}: switched on, {identities} identit{}", if identities == 1 { "y" } else { "ies" })); } } + if pin_aborts { + self.sweep_abort("the cap was set by hand"); + } if self.running { self.plan_miners(); if power_changed { @@ -1177,12 +1314,17 @@ impl Engine { if self.power_busy { return; } + if self.sweep.is_some() || self.sweep_pending.is_some() { + // the sweep owns the caps until it ends; it applies the chosen one itself + self.shared.log(&format!("power cap ({why}): deferred, a sweep is running")); + return; + } let mut cmds = Vec::new(); let mut what = Vec::new(); { let mut st = self.st(); for c in st.mining.cards.iter_mut().filter(|c| c.vendor == "nvidia" && c.enabled && c.power_default_w > 0.0) { - let pct = if c.power_pct == 0 { 80 } else { c.power_pct.clamp(60, 100) }; + let pct = if c.power_pct == 0 { 80 } else { c.power_pct.clamp(crate::sweep::MIN_PCT, 100) }; c.power_pct = pct; let watts = requested_watts(c); if (c.power_limit_w - watts).abs() < 1.0 && c.power_applied { @@ -1300,6 +1442,7 @@ impl Engine { } let f = |s: &str| s.parse::().unwrap_or(0.0); let (draw, tgpu, tmem, limit) = (f(p[1]), f(p[2]), f(p[3]), f(p[4])); + let sweeping = self.sweep.is_some() || self.sweep_pending.is_some(); let mut st = self.st(); let Some(c) = st.mining.cards.iter_mut().find(|c| c.vendor == "nvidia" && c.device == p[0]) else { return }; c.power_w = draw; @@ -1307,11 +1450,22 @@ impl Engine { c.temp_mem = tmem; if limit > 0.0 { c.power_limit_w = limit; + // the readback is the truth: a cap the elevated step set after the judgement (or one set by hand + // outside the app) that matches what was asked counts as applied + if !sweeping && !c.power_applied && c.enabled && c.power_default_w > 0.0 && (limit - requested_watts(c)).abs() < 1.0 { + c.power_applied = true; + c.power_note = format!("power cap: {} W in force (read back)", limit as u64); + } } c.telemetry_at = crate::platform::unix_now_f(); let idx = c.index; let mining = c.state == "mining"; drop(st); + if let Some(run) = self.sweep.as_mut() { + if run.card == idx { + run.sample_draw(draw); + } + } if mining { let s = self.stability.entry(idx).or_default(); s.draws.push(draw); @@ -1349,6 +1503,378 @@ impl Engine { } } + // ---- the efficiency sweep (src/sweep.rs) --------------------------------------------------------------------- + + /// A SWEEP line: the app log, and stdout under --sweep (the PC job reads it there). + fn sweep_say(&self, line: &str) { + self.shared.log(line); + if self.shared.runtime.sweep_only { + println!("{line}"); + let _ = std::io::stdout().flush(); + } + } + + fn sweep_dir(&self) -> PathBuf { + self.shared.runtime.app_dir.join("sweep") + } + + /// Every 10 s: drive the running sweep, else start one that is due. One card at a time. Never while a remote + /// job holds the GPU, never under a pause, never inside the last 10 minutes before the hour boundary. + fn tick_sweep(&mut self, now: Instant) { + if self.sweep.is_some() { + self.sweep_drive(now); + return; + } + if self.sweep_pending.is_some() || now < self.sweep_next_check { + return; + } + self.sweep_next_check = now + Duration::from_secs(10); + if self.quitting || !self.running || self.power_busy || self.job_hold || self.jobs.holds_miners() { + return; + } + let (auto_on, paused) = { + let s = self.shared.settings.lock().unwrap(); + let st = self.st(); + (s.sweep, st.mining.paused) + }; + if paused { + return; + } + let unix = crate::platform::unix_now(); + let (eta_known, eta_s) = { + let st = self.st(); + (st.node.daa > 0 && st.program.boundary_daa > 0, st.program.eta_s) + }; + let boundary_ok = !eta_known || eta_s > crate::sweep::NEEDS_S; + let prefs = self.shared.settings.lock().unwrap().cards.clone(); + let cards = self.st().mining.cards.clone(); + let mut pick: Option<(usize, bool)> = None; + for c in cards.iter() { + if !(c.enabled && c.sweep_supported && c.vendor == "nvidia") { + continue; + } + let forced = self.sweep_queue.contains(&c.key); + let due = prefs.get(&c.key).map(|p| p.sweep_at == 0 || unix.saturating_sub(p.sweep_at) >= crate::sweep::PERIOD_S).unwrap_or(true); + if !forced && !(auto_on && !c.pinned && due) { + continue; + } + if self.sweep_retry.get(&c.index).map(|t| now < *t).unwrap_or(false) { + continue; + } + let stable = self.miners.iter().find(|m| m.card == c.index).map(|m| m.proc.as_ref().map(|p| p.started.elapsed() >= Duration::from_secs(crate::sweep::STABLE_S)).unwrap_or(false) && m.last_status.is_some()).unwrap_or(false); + let wait = if c.state != "mining" { + Some(format!("sweep: waits for the card to mine ({})", c.state)) + } else if !stable { + Some(format!("sweep: waits for {} s of steady mining", crate::sweep::STABLE_S)) + } else if !boundary_ok { + Some(format!("sweep: starts after the hour boundary ({eta_s} s)")) + } else { + None + }; + match wait { + Some(w) => { + if forced { + if let Some(cc) = self.st().mining.cards.get_mut(c.index) { + cc.sweep_note = w; + } + } + } + None => { + pick = Some((c.index, forced)); + break; + } + } + } + if let Some((idx, forced)) = pick { + self.sweep_begin(idx, forced); + } else if self.shared.runtime.sweep_only && self.sweep_queue.is_empty() && self.sweep_pending.is_none() { + self.shared.send(Cmd::Quit); + } + } + + /// Step 1 of a sweep: find out how caps can be set from this process (one nvidia-smi -pl at the current limit, + /// a no-op). Elevated engines (the PC job runs with --elevated) set them directly; others need the helper. + fn sweep_begin(&mut self, idx: usize, forced: bool) { + let Some(c) = self.st().mining.cards.get(idx).cloned() else { return }; + self.sweep_pending = Some((idx, forced)); + self.sweep_queue.retain(|k| k != &c.key); + *self.sweep_attempts.entry(idx).or_default() += 1; + if let Some(cc) = self.st().mining.cards.get_mut(idx) { + cc.sweep_state = "running".into(); + cc.sweep_note = "sweep: checking how the cap is set".into(); + } + if let Some(direct) = self.sweep_direct { + self.sweep_mode_known(Ok(direct)); + return; + } + let smi = crate::platform::tool("nvidia-smi"); + let device = c.device.clone(); + let current = if c.power_limit_w > 0.0 { c.power_limit_w } else { c.power_default_w }.round() as u64; + let shared = self.shared.clone(); + std::thread::spawn(move || { + let r = match crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "-pl", ¤t.to_string()]), None, Duration::from_secs(20)) { + Some(out) => Ok(out.contains("All done")), + None => Err("nvidia-smi did not answer".to_string()), + }; + shared.send(Cmd::SweepCapMode(r)); + }); + } + + /// Step 2: start the helper when needed, then the run. + fn sweep_mode_known(&mut self, r: Result) { + let Some((idx, forced)) = self.sweep_pending else { return }; + let direct = match r { + Ok(d) => d, + Err(e) => { + self.sweep_abort(&format!("cannot set caps: {e}")); + return; + } + }; + self.sweep_direct = Some(direct); + let Some(c) = self.st().mining.cards.get(idx).cloned() else { + self.sweep_pending = None; + return; + }; + if !direct && !self.sweep_helper { + if let Err(e) = self.sweep_helper_start(&c) { + self.sweep_abort(&format!("the elevated helper could not start: {e}")); + return; + } + } + let steps = crate::sweep::plan_steps(c.power_default_w, c.power_min_w, c.power_max_w); + if steps.is_empty() { + self.sweep_abort("no steps: the card reported no default power limit"); + return; + } + let label = self.miners.iter().find(|m| m.card == idx).map(|m| m.label.clone()).unwrap_or_else(|| format!("card-{idx}")); + let before_w = if c.power_limit_w > 0.0 { c.power_limit_w } else { requested_watts(&c) }; + let now = Instant::now(); + let run = crate::sweep::Run::new(idx, &c.key, &c.device, &label, steps.clone(), c.power_pct, before_w, forced, crate::sweep::Timing::from_env(), now); + self.sweep_pending = None; + self.sweep_say(&format!( + "SWEEP start card={label} name={} steps={} default={:.0} min={:.0} max={:.0} before={:.0} mode={}", + c.name.replace(' ', "_"), + steps.iter().map(|s| s.pct.to_string()).collect::>().join(","), + c.power_default_w, + c.power_min_w, + c.power_max_w, + before_w, + if direct { "direct" } else { "helper" } + )); + self.shared.event("info", &format!("{}: efficiency sweep started: {} caps from {}% down, {} s each on the live program", c.name, steps.len(), steps[0].pct, (run.timing.settle + run.timing.hold).as_secs())); + self.sweep = Some(run); + self.sweep_drive(now); + } + + /// The elevated helper (src/sweep.rs helper_script_*): one administrator prompt; it polls /sweep/cmd.txt. + fn sweep_helper_start(&mut self, c: &CardState) -> Result<(), String> { + let dir = self.sweep_dir(); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + let _ = std::fs::write(dir.join("cmd.txt"), ""); + let restore = (if c.power_limit_w > 0.0 { c.power_limit_w } else { c.power_default_w }).round() as u64; + let smi = crate::platform::tool("nvidia-smi").display().to_string(); + let line = if cfg!(windows) { + let script = dir.join("helper.ps1"); + std::fs::write(&script, [b"\xEF\xBB\xBF".as_slice(), crate::sweep::helper_script_windows().replace('\n', "\r\n").as_bytes()].concat()).map_err(|e| e.to_string())?; + format!("\"{}\" -NoProfile -ExecutionPolicy Bypass -File \"{}\" \"{}\" \"{}\" {} {}", crate::platform::tool("powershell").display(), script.display(), dir.display(), smi, c.device, restore) + } else { + let script = dir.join("helper.sh"); + std::fs::write(&script, crate::sweep::helper_script_unix()).map_err(|e| e.to_string())?; + format!("sh \"{}\" \"{}\" \"{}\" {} {}", script.display(), dir.display(), smi, c.device, restore) + }; + self.shared.log(&format!("sweep helper (administrator prompt): {line}")); + self.sweep_helper = true; + let shared = self.shared.clone(); + std::thread::spawn(move || { + let r = crate::platform::run_elevated(&line); + shared.send(Cmd::SweepHelperDone(r)); + }); + Ok(()) + } + + fn sweep_helper_quit(&mut self) { + if self.sweep_helper { + let _ = std::fs::write(self.sweep_dir().join("cmd.txt"), "quit\n"); + } + } + + /// Sets a cap for the sweep: directly (this process is elevated) or through the helper's command file. The + /// telemetry readback (power.limit every 5 s) tells the run when it took. + fn sweep_set_cap(&mut self, device: &str, watts: f64) { + self.sweep_seq += 1; + let w = watts.round() as u64; + if self.sweep_direct == Some(true) { + let smi = crate::platform::tool("nvidia-smi"); + let device = device.to_string(); + let shared = self.shared.clone(); + std::thread::spawn(move || { + let out = crate::detect::run_timeout(std::process::Command::new(&smi).args(["-i", &device, "-pl", &w.to_string()]), None, Duration::from_secs(20)).unwrap_or_else(|| "nvidia-smi did not answer".into()); + shared.send(Cmd::SweepCapSet(out)); + }); + } else { + let _ = std::fs::write(self.sweep_dir().join("cmd.txt"), format!("{} {w}\n", self.sweep_seq)); + } + self.shared.log(&format!("sweep: cap {w} W requested on device {device}")); + } + + /// Faults and conditions first, then the state machine. + fn sweep_drive(&mut self, now: Instant) { + let Some((idx, device, started)) = self.sweep.as_ref().map(|r| (r.card, r.device.clone(), r.started)) else { return }; + let card = self.st().mining.cards.get(idx).cloned(); + let Some(c) = card else { + self.sweep_abort("the card disappeared"); + return; + }; + let slot_error = self.miners.iter().find(|m| m.card == idx).and_then(|m| m.error_at).map(|t| t > started).unwrap_or(false); + let fault = if self.quitting { + Some("the app is quitting".to_string()) + } else if self.job_hold || self.jobs.holds_miners() { + Some("a remote job took the GPU".into()) + } else if self.st().mining.paused { + Some("mining paused".into()) + } else if c.state != "mining" { + Some(format!("the card left mining ({})", if c.state == "starting" || c.state == "restarting" { "worker restart, likely the hour boundary" } else { &c.state })) + } else if slot_error { + Some("the worker reported an error".into()) + } else if c.temp_gpu >= 90.0 { + Some(format!("GPU at {:.0} C", c.temp_gpu)) + } else { + None + }; + if let Some(why) = fault { + self.sweep_abort(&why); + return; + } + let outs = self.sweep.as_mut().map(|r| r.tick(now, c.power_limit_w)).unwrap_or_default(); + let label = self.sweep.as_ref().map(|r| r.label.clone()).unwrap_or_default(); + for o in outs { + match o { + crate::sweep::Out::Apply(w) => self.sweep_set_cap(&device, w), + crate::sweep::Out::Row(row) => { + self.sweep_say(&row.line(&label)); + if let Some(cc) = self.st().mining.cards.get_mut(idx) { + if row.usable() { + cc.sweep_note = format!("sweep: {}% done · {:.3} MH/W", row.pct, row.eff); + } + } + } + crate::sweep::Out::Finished(row) => { + self.sweep_finish(row); + return; + } + crate::sweep::Out::Failed(e) => { + self.sweep_abort(&e); + return; + } + } + } + if let Some(words) = self.sweep.as_ref().map(|r| r.words(now)) { + if let Some(cc) = self.st().mining.cards.get_mut(idx) { + cc.sweep_note = words; + } + } + } + + /// The chosen cap is in force: record it on the card and in the settings, say so, let the helper go. + fn sweep_finish(&mut self, row: crate::sweep::Row) { + let Some(run) = self.sweep.take() else { return }; + let idx = run.card; + let unix = crate::platform::unix_now(); + let (name, pinned, pinned_w) = { + let mut settings = self.shared.settings.lock().unwrap(); + let mut st = self.st(); + let Some(c) = st.mining.cards.get_mut(idx) else { return }; + c.sweep_pct = row.pct; + c.sweep_eff = row.eff; + c.sweep_watts = row.watts; + c.sweep_mhs = row.mhs; + c.sweep_at = unix as f64; + c.sweep_state = "idle".into(); + let pinned_w = requested_watts(c); + if c.pinned { + c.sweep_note = format!("sweep: best {}% · {:.3} MH/W; cap pinned at {}%", row.pct, row.eff, c.power_pct); + } else { + c.power_pct = row.pct; + c.power_limit_w = row.limit; + c.power_applied = true; + c.power_note = format!("power cap: {} W held by the sweep (best MH per watt)", row.limit as u64); + c.sweep_note = format!("sweep: best {}% · {:.3} MH/W", row.pct, row.eff); + } + let e = settings.cards.entry(c.key.clone()).or_insert_with(|| crate::config::CardPref { enabled: c.enabled, identities: c.identities, ..Default::default() }); + e.sweep_at = unix; + e.sweep_pct = row.pct; + e.sweep_eff = row.eff; + e.sweep_watts = row.watts; + e.sweep_mhs = row.mhs; + if !c.pinned { + e.power_pct = row.pct; + } + (c.name.clone(), c.pinned, pinned_w) + }; + self.shared.save_settings(); + self.sweep_say(&format!("SWEEP chosen card={} cap={} limit={:.0} watts={:.1} mhs={:.2} eff={:.4}{}", run.label, row.pct, row.limit, row.watts, row.mhs, row.eff, if pinned { " pinned=1" } else { "" })); + if pinned { + self.sweep_set_cap(&run.device.clone(), pinned_w); + self.shared.event("ok", &format!("{name}: best efficiency at {}% ({:.0} W cap): {:.1} MH/s at {:.0} W = {:.3} MH/W. The cap stays pinned where you set it.", row.pct, row.limit, row.mhs, row.watts, row.eff)); + } else { + self.shared.event("ok", &format!("{name}: best efficiency at {}% ({:.0} W cap): {:.1} MH/s at {:.0} W = {:.3} MH/W; cap held there", row.pct, row.limit, row.mhs, row.watts, row.eff)); + } + self.sweep_helper_quit(); + if self.shared.runtime.sweep_only && self.sweep_queue.is_empty() { + self.shared.send(Cmd::Quit); + } + } + + /// Stops a sweep: the cap goes back to what it was, the log says why, the card waits an hour before the + /// scheduler tries again (a forced sweep under --sweep retries twice). + fn sweep_abort(&mut self, why: &str) { + let pending = self.sweep_pending.take(); + let run = self.sweep.take(); + let (idx, label, device, before_w, before_pct, forced) = match (&run, pending) { + (Some(r), _) => (r.card, r.label.clone(), r.device.clone(), r.before_w, r.before_pct, r.forced), + (None, Some((idx, forced))) => { + let c = self.st().mining.cards.get(idx).cloned(); + let (label, device, w, pct) = c.map(|c| (format!("card-{idx}"), c.device.clone(), if c.power_limit_w > 0.0 { c.power_limit_w } else { requested_watts(&c) }, c.power_pct)).unwrap_or_default(); + (idx, label, device, w, pct, forced) + } + _ => return, + }; + self.sweep_say(&format!("SWEEP aborted card={label} reason={}", why.replace(' ', "_"))); + let name = { + let mut st = self.st(); + match st.mining.cards.get_mut(idx) { + Some(c) => { + c.sweep_state = "idle".into(); + c.sweep_note = format!("sweep stopped: {why}"); + c.power_pct = before_pct; + c.power_applied = false; + c.power_note = format!("cap restored to {} W after the sweep stopped", before_w as u64); + c.name.clone() + } + None => label.clone(), + } + }; + if run.is_some() && before_w > 0.0 && !device.is_empty() { + self.sweep_set_cap(&device, before_w); + } + self.sweep_helper_quit(); + self.shared.event("error", &format!("{name}: efficiency sweep stopped ({why}); cap back to {} W", before_w as u64)); + let attempts = self.sweep_attempts.get(&idx).copied().unwrap_or(0); + if forced && self.shared.runtime.sweep_only && attempts < 3 && !self.quitting { + // the PC job: the hour boundary or a restart interrupted it; try again once the card is steady + let key = self.st().mining.cards.get(idx).map(|c| c.key.clone()); + if let Some(key) = key { + self.sweep_queue.push(key); + } + self.sweep_retry.insert(idx, Instant::now() + Duration::from_secs(30)); + } else { + self.sweep_retry.insert(idx, Instant::now() + Duration::from_secs(3600)); + if self.shared.runtime.sweep_only && self.sweep_queue.is_empty() { + self.shared.send(Cmd::Quit); + } + } + } + // ---- the tick ---------------------------------------------------------------------------------------------- fn tick(&mut self) { @@ -1362,6 +1888,7 @@ impl Engine { self.tick_watch(now); self.tick_miners(now); self.tick_telemetry(now); + self.tick_sweep(now); } self.derive(now); if now.duration_since(self.last_status) >= Duration::from_secs(self.shared.runtime.status_secs as u64) { @@ -1727,10 +2254,12 @@ impl Engine { let mut st = self.st(); st.node.last_reading_age_s = self.node_last_reading.map(|t| now.duration_since(t).as_secs_f64()).unwrap_or(-1.0); let mut total = 0.0; - for c in st.mining.cards.iter() { + for c in st.mining.cards.iter_mut() { if c.state == "mining" { total += c.hash_now; } + // live hash per watt (the number next to the draw on the tile); needs a draw reading under a minute old + c.eff_mhw = if c.state == "mining" && c.power_w > 1.0 && c.hash_now > 0.0 && unix - c.telemetry_at < 60.0 { c.hash_now / c.power_w } else { 0.0 }; } st.mining.hash_total = total; let cut = unix - 3600.0; @@ -2014,6 +2543,11 @@ impl Engine { let avg = kv_f64(text, "hash").unwrap_or(0.0); // the v4 miner's interval rate: "now=12.34 MH/s wall (...)"; older miners: the average let now_rate = kv_f64(text, "now").unwrap_or(avg); + if let Some(run) = self.sweep.as_mut() { + if run.card == card { + run.sample_rate(now_rate); + } + } let mut st = self.st(); if let Some(c) = st.mining.cards.get_mut(card) { c.hash_avg = avg; @@ -2155,12 +2689,21 @@ impl Engine { fn shutdown(&mut self) { self.shared.log("quit: stopping the miners, then the node"); self.jobs.abort(&self.shared, "the app is quitting"); + if self.sweep.is_some() || self.sweep_pending.is_some() { + self.sweep_abort("the app is quitting"); + } + self.sweep_helper_quit(); self.stop_miners("quit"); self.stability_line(); if let Some(mut t) = self.telemetry.take() { t.stop(2); } - self.restore_power_limits(); + if self.shared.runtime.sweep_only { + // the measurement leaves the chosen cap in force for the app that takes the card back + self.shared.log("--sweep: the chosen caps stay in force (not restored)"); + } else { + self.restore_power_limits(); + } if let Some(mut w) = self.watch.take() { w.stop(2); } @@ -2193,7 +2736,7 @@ impl Engine { /// The watts a card's cap asks for: power_pct of the default limit, inside the card's min and max. fn requested_watts(c: &CardState) -> f64 { - let pct = if c.power_pct == 0 { 80 } else { c.power_pct.clamp(60, 100) }; + let pct = if c.power_pct == 0 { 80 } else { c.power_pct.clamp(crate::sweep::MIN_PCT, 100) }; let mut w = c.power_default_w * pct as f64 / 100.0; if c.power_min_w > 0.0 { w = w.max(c.power_min_w); diff --git a/app/igneum-app/src/main.rs b/app/igneum-app/src/main.rs index 48c749988..9d67751ec 100644 --- a/app/igneum-app/src/main.rs +++ b/app/igneum-app/src/main.rs @@ -3,9 +3,11 @@ //! --wrapper, reads `URL ...` and `STATE {...}` lines from its stdout and writes `quit` on its stdin. Without a host //! (--open) the dashboard opens in the default browser. //! -//! igneum-app [--wrapper | --open | --no-open | --launch] [--print-url] +//! igneum-app [--wrapper | --open | --no-open | --launch | --sweep] [--print-url] //! --launch is what the Start Menu entry runs on Windows: it hands over to "Igneum Miner.exe" (the window host) when //! that is installed next to the engine, else runs the engine with the dashboard in the default browser. +//! --sweep runs the efficiency sweep (src/sweep.rs) on every supported card as soon as it mines, prints the SWEEP +//! table on stdout, leaves the chosen caps in force and quits; no browser opens (the PC measurement job). //! //! Environment (tests): IGNEUM_APP_NETWORK devnet|simnet, IGNEUM_APP_RPC_PORT, IGNEUM_APP_P2P_PORT, IGNEUM_APP_PEERS //! (comma list, empty for none), IGNEUM_APP_UNSYNCED=1, IGNEUM_APP_DATA, IGNEUM_APP_LOGS, IGNEUM_APP_BIN, IGNEUM_APP_NODE_DIR. @@ -25,6 +27,7 @@ mod update; mod jobs; mod jobrun; mod prover; +mod sweep; use std::io::{BufRead, Write}; use std::sync::mpsc::channel; @@ -33,7 +36,12 @@ use std::sync::Arc; fn main() { let args: Vec = std::env::args().skip(1).collect(); let wrapper = args.iter().any(|a| a == "--wrapper"); - let no_open = wrapper || args.iter().any(|a| a == "--no-open"); + let sweep = args.iter().any(|a| a == "--sweep"); + if sweep { + // the runtime reads it (config::Runtime::from_env); the engine starts at once and quits after the sweep + std::env::set_var("IGNEUM_APP_SWEEP", "1"); + } + let no_open = wrapper || sweep || args.iter().any(|a| a == "--no-open"); if args.iter().any(|a| a == "--version" || a == "-V") { println!("igneum-app {}", engine::VERSION); return; diff --git a/app/igneum-app/src/server.rs b/app/igneum-app/src/server.rs index ddd271e97..3b7c0ff5e 100644 --- a/app/igneum-app/src/server.rs +++ b/app/igneum-app/src/server.rs @@ -244,7 +244,7 @@ fn api_post(shared: &Arc, path: &str, body: Value) -> Result, path: &str, body: Value) -> Result { + let key = s("key").ok_or("key missing")?; + shared.send(Cmd::SweepStart(key)); + Ok(json!({ "ok": true })) + } + "/api/sweep/stop" => { + shared.send(Cmd::SweepStop); + Ok(json!({ "ok": true })) + } + "/api/sweep/pin" => { + let key = s("key").ok_or("key missing")?; + let pinned = body.get("pinned").and_then(|v| v.as_bool()).ok_or("pinned missing")?; + shared.send(Cmd::SweepPin(key, pinned)); + Ok(json!({ "ok": true })) + } + "/api/sweep/enable" => { + let on = body.get("on").and_then(|v| v.as_bool()).ok_or("on missing")?; + shared.send(Cmd::SweepEnable(on)); + Ok(json!({ "ok": true })) + } "/api/clock/sync" => { shared.send(Cmd::ClockSync); Ok(json!({ "ok": true })) diff --git a/app/igneum-app/src/state.rs b/app/igneum-app/src/state.rs index b5b25b559..9df1ca188 100644 --- a/app/igneum-app/src/state.rs +++ b/app/igneum-app/src/state.rs @@ -64,12 +64,23 @@ pub struct CardState { pub power_min_w: f64, pub power_max_w: f64, pub power_before_w: f64, // the limit before the app touched it (restored on quit) - pub power_pct: u32, // the chosen cap, percent of the default (60 to 100) + pub power_pct: u32, // the chosen cap, percent of the default (50 to 100) pub power_applied: bool, pub power_note: String, pub temp_gpu: f64, pub temp_mem: f64, pub telemetry_at: f64, + // hash per watt (src/sweep.rs) + pub eff_mhw: f64, // live: hash_now over power_w, MH per watt; 0 = unknown + pub sweep_supported: bool, // NVIDIA with readable limits; the note says why not otherwise + pub sweep_state: String, // idle | running | unsupported + pub sweep_note: String, // the phase in words while running, else why unsupported or the last outcome + pub sweep_pct: u32, // the cap the last sweep chose (0 = never swept) + pub sweep_eff: f64, // MH per watt at that cap + pub sweep_watts: f64, + pub sweep_mhs: f64, + pub sweep_at: f64, // unix s of the last sweep + pub pinned: bool, // the user set the cap by hand; the sweep records but does not change it } #[derive(Clone, Serialize, Default)] @@ -158,6 +169,8 @@ pub struct SettingsState { pub remote_jobs: bool, /// the prover service (src/prover.rs) pub prove: bool, + /// the efficiency sweep (src/sweep.rs): once after install, then weekly + pub sweep: bool, } /// One remote job this machine ran (the ledger entry), for the Settings history and the last-job strip. diff --git a/app/igneum-app/src/sweep.rs b/app/igneum-app/src/sweep.rs new file mode 100644 index 000000000..52c0adbbf --- /dev/null +++ b/app/igneum-app/src/sweep.rs @@ -0,0 +1,593 @@ +//! The efficiency sweep: hash per watt, the number miners compare. For one NVIDIA card at a time the engine steps +//! the power cap from 100% of the card's default limit down to 50% in 10% steps, holds each step for 60 s after a +//! 15 s settle, averages the card's reported draw (nvidia-smi power.draw, the telemetry child) and the worker's +//! interval rate (the STATUS lines), computes MH per watt per step, picks the best step and leaves the cap there. +//! It runs on the live kernel: the worker is never restarted and the hour's program is never lost. It never starts +//! while a remote job holds the GPU, and it aborts the moment the card leaves "mining" (a fault, an hour boundary, +//! a pause, a job). +//! +//! This file is the logic: the step plan, the per-step rows, the choice, the parser for nvidia-smi's power query and +//! the state machine, all driven by an explicit clock so the tests run without a card. The engine (src/engine.rs, +//! `tick_sweep` and the `Cmd::Sweep*` commands) owns the processes: it sets the cap (directly when the engine runs +//! elevated, else through one elevated helper that polls a command file, so a sweep costs one administrator prompt +//! instead of six), reads the limit back through the telemetry child, feeds the samples in and writes the table to +//! the app log: +//! SWEEP start card=