From 735887a309d44bdb1d84b81ce85e2e62deb0eef2 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:21:51 +0000 Subject: [PATCH] Ember Tune: every card tuned for MH per watt out of the box, the fleet prior per card model in the signed manifest, the console and /miners priors table the project lead, 5 October 2026, 22:45 BST: "make sure we have ember tuning every single card for efficiency out of the box, the more data = the better the tune, make an awesome system." Built on lever 3 (docs/plans/miner-eff.md), lever 2's signed tuning section (docs/design/miner-tuning.md), the AMD telemetry helper (35e3d26, its --tune/--set-gmax/--set-plimit/ --reset contract) and the Power control switch (652e848). Design, data flow, tiers and the privacy line: docs/plans/ember-tune.md. - src/ember.rs (new): two knobs per card (power limit %, core clock cap MHz; memory clock never touched), the full plan (power ladder 100..50%, then the clock ladder 90..60% at the chosen power), the confirm plan (the fleet prior and one neighbour), the baseline plan (measure only), the marks (faulted, hot, memory_clock_dropped, unapplied, no_readings), the choice (best MH/W within 1% of the top rate, then rate, then draw), the fleet record (a hash of the install id, no address), the prior lookup and the kill switch (tuning.ember), the state machine on a fake clock. 9 unit tests. - engine.rs: tick_sweep schedules every NVIDIA, AMD and Apple card (120 s steady, 600 s to the boundary, no job hold, no pause, weekly, again after a driver major or program-class change, never under the manifest kill switch); the probe (nvidia-smi clocks.max.gr + driver_version and the direct/helper mode; igneum-gpu-telemetry --tune for AMD); tune_apply (nvidia-smi -pl / -lgc 0, / -rgc directly or through the helper; the AMD helper per request); Cmd::TuneProbe, Cmd::TuneSet; faults from rejected and mismatched hashes mark the step; the TUNE lines and the TUNE {json} record, uploaded with the log; the Tuned line on the card state. The NVIDIA helper starts only with Power control on: the --sweep job never counts as permission (no prompt on a PC with nobody there). - sweep.rs: the helper protocol gains lgc/rgc (clock cap and reset) and resets the clocks after 20 idle minutes. - state.rs, config.rs: the tune fields (clock cap, driver, class, source, the Tuned line); the nvidia-smi telemetry query carries clocks.gr and clocks.mem; the AMD sample line's plimit_pct and gmax_mhz are parsed. - ui: "Tuned: X MH/s at Y W (Z MH/W)" with the point, the source and when; measure-only cards say why; the Ember Tune switch; tune-line.test.mjs. - relay/lib/ember.mjs + relay/test/ember.test.mjs: the aggregation per (card model | driver major | program class): median point, MH/W, spread, samples, machines; five samples converge, an outlier does not move the median, baselines make no prior, de-duplication, the manifest merge keeps lever 2's cards. api/console.mjs fn=tuning and tools/console.mjs tuning; tools/tuning.mjs --priors [--write tuning.json] [--site] [--tuning-off]. - site: the fleet priors table on /miners (site/miner-priors.json), the lever text. - relay/playbooks/ember-tune-pc1.ps1: the PC 1 run (second engine with --sweep from a scratch copy of the install). Measured tonight: see the bench log entry that follows the PC 1 run. The 9070 XT left PC 1's bus at 20:40 UTC and the 5090 needs the administrator prompt the project lead cannot answer asleep, so tonight's PC 1 run is the baseline plan on the 5090 through the whole pipeline; the two-knob tune on both cards is owed. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 4 +- app/igneum-app/src/config.rs | 10 + app/igneum-app/src/detect.rs | 1 + app/igneum-app/src/ember.rs | 1033 ++++++++++++++++++++++++++ app/igneum-app/src/engine.rs | 646 ++++++++++++---- app/igneum-app/src/main.rs | 1 + app/igneum-app/src/state.rs | 15 + app/igneum-app/src/sweep.rs | 58 +- app/igneum-app/ui/app.js | 55 +- app/igneum-app/ui/index.html | 12 +- app/igneum-app/ui/tune-line.test.mjs | 44 ++ docs/plans/ember-tune.md | 164 ++++ relay/api/console.mjs | 15 + relay/lib/ember.mjs | 131 ++++ relay/playbooks/ember-tune-pc1.ps1 | 143 ++++ relay/test/ember.test.mjs | 110 +++ site/build.mjs | 21 +- site/miner-priors.json | 6 + site/miner.html | 6 +- site/miners.html | 10 +- tools/console.mjs | 7 + tools/tuning.mjs | 46 +- 22 files changed, 2354 insertions(+), 184 deletions(-) create mode 100644 app/igneum-app/src/ember.rs create mode 100644 app/igneum-app/ui/tune-line.test.mjs create mode 100644 docs/plans/ember-tune.md create mode 100644 relay/lib/ember.mjs create mode 100644 relay/playbooks/ember-tune-pc1.ps1 create mode 100644 relay/test/ember.test.mjs create mode 100644 site/miner-priors.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9879a98b0..185fa0f42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,6 @@ jobs: - name: ship tool self-test (version bump, the dl-both and public manifest helpers) run: node tools/ship-app.mjs --self-test - name: relay unit tests (parsers, secret compare, the wake endpoint) - run: node --test relay/test/parse.test.mjs relay/test/auth.test.mjs relay/test/wake.test.mjs + run: node --test relay/test/parse.test.mjs relay/test/auth.test.mjs relay/test/wake.test.mjs relay/test/ember.test.mjs - name: miner app notice strip and update card (ordering, keys, wording, timers, when the card shows) - run: node --test app/igneum-app/ui/notices.test.mjs app/igneum-app/ui/update-card.test.mjs + run: node --test app/igneum-app/ui/notices.test.mjs app/igneum-app/ui/update-card.test.mjs app/igneum-app/ui/tune-line.test.mjs diff --git a/app/igneum-app/src/config.rs b/app/igneum-app/src/config.rs index 050b79bb7..f2b06d7bf 100644 --- a/app/igneum-app/src/config.rs +++ b/app/igneum-app/src/config.rs @@ -29,6 +29,16 @@ pub struct CardPref { pub sweep_watts: f64, #[serde(default)] pub sweep_mhs: f64, + /// Ember Tune (src/ember.rs): the clock cap the last tune chose (0 = unlocked), the driver and program class it + /// ran under (a change makes the card due again), and the plan that produced it (full | confirm | baseline) + #[serde(default)] + pub sweep_clock_mhz: u32, + #[serde(default)] + pub sweep_driver: String, + #[serde(default)] + pub sweep_class: String, + #[serde(default)] + pub sweep_source: String, } #[derive(Clone, Serialize, Deserialize)] diff --git a/app/igneum-app/src/detect.rs b/app/igneum-app/src/detect.rs index 020d7a6ae..358b7e391 100644 --- a/app/igneum-app/src/detect.rs +++ b/app/igneum-app/src/detect.rs @@ -66,6 +66,7 @@ fn card(index: usize, name: &str, vendor: &str, worker: &str, detail: &str, devi device: device.into(), enabled: true, state: "off".into(), + amd_ordinal: -1, ..Default::default() } } diff --git a/app/igneum-app/src/ember.rs b/app/igneum-app/src/ember.rs new file mode 100644 index 000000000..d839135e5 --- /dev/null +++ b/app/igneum-app/src/ember.rs @@ -0,0 +1,1033 @@ +//! Ember Tune: every card tuned for MH per watt out of the box, and the fleet's results folded into a prior that a +//! new card starts from (docs/plans/ember-tune.md). Two knobs per card: the power limit (percent of the card's +//! default) and the core clock cap (MHz; 0 = unlocked). The memory clock is never touched, and a step that drags it +//! down is marked and cannot win. This file is the logic, driven by an explicit clock so the tests run without a +//! card: the plans (full, confirm, baseline), the per-step rows with their marks, the choice rule, the fleet record, +//! the prior lookup and the state machine. The engine (src/engine.rs, `tick_tune` and the `Cmd::Tune*` commands) +//! owns the processes: NVIDIA through nvidia-smi (`-pl`, `-lgc 0,`, `-rgc`; administrator rights, so only with +//! Power control on), AMD through igneum-gpu-telemetry (`--set-plimit`, `--set-gmax`, `--reset`; no elevation on +//! Windows), Apple measure only. +//! +//! The lines in the app log (and on stdout under --sweep, which the PC job reads): +//! TUNE start card=