735887a309
the project lead, 5 October 2026, 22:45 BST: "make sure we have ember tuning every single card for efficiency out of the box, the more data = the better the tune, make an awesome system." Built on lever 3 (docs/plans/miner-eff.md), lever 2's signed tuning section (docs/design/miner-tuning.md), the AMD telemetry helper (35e3d26, its --tune/--set-gmax/--set-plimit/ --reset contract) and the Power control switch (652e848). Design, data flow, tiers and the privacy line: docs/plans/ember-tune.md. - src/ember.rs (new): two knobs per card (power limit %, core clock cap MHz; memory clock never touched), the full plan (power ladder 100..50%, then the clock ladder 90..60% at the chosen power), the confirm plan (the fleet prior and one neighbour), the baseline plan (measure only), the marks (faulted, hot, memory_clock_dropped, unapplied, no_readings), the choice (best MH/W within 1% of the top rate, then rate, then draw), the fleet record (a hash of the install id, no address), the prior lookup and the kill switch (tuning.ember), the state machine on a fake clock. 9 unit tests. - engine.rs: tick_sweep schedules every NVIDIA, AMD and Apple card (120 s steady, 600 s to the boundary, no job hold, no pause, weekly, again after a driver major or program-class change, never under the manifest kill switch); the probe (nvidia-smi clocks.max.gr + driver_version and the direct/helper mode; igneum-gpu-telemetry --tune for AMD); tune_apply (nvidia-smi -pl / -lgc 0,<MHz> / -rgc directly or through the helper; the AMD helper per request); Cmd::TuneProbe, Cmd::TuneSet; faults from rejected and mismatched hashes mark the step; the TUNE lines and the TUNE {json} record, uploaded with the log; the Tuned line on the card state. The NVIDIA helper starts only with Power control on: the --sweep job never counts as permission (no prompt on a PC with nobody there). - sweep.rs: the helper protocol gains lgc/rgc (clock cap and reset) and resets the clocks after 20 idle minutes. - state.rs, config.rs: the tune fields (clock cap, driver, class, source, the Tuned line); the nvidia-smi telemetry query carries clocks.gr and clocks.mem; the AMD sample line's plimit_pct and gmax_mhz are parsed. - ui: "Tuned: X MH/s at Y W (Z MH/W)" with the point, the source and when; measure-only cards say why; the Ember Tune switch; tune-line.test.mjs. - relay/lib/ember.mjs + relay/test/ember.test.mjs: the aggregation per (card model | driver major | program class): median point, MH/W, spread, samples, machines; five samples converge, an outlier does not move the median, baselines make no prior, de-duplication, the manifest merge keeps lever 2's cards. api/console.mjs fn=tuning and tools/console.mjs tuning; tools/tuning.mjs --priors [--write tuning.json] [--site] [--tuning-off]. - site: the fleet priors table on /miners (site/miner-priors.json), the lever text. - relay/playbooks/ember-tune-pc1.ps1: the PC 1 run (second engine with --sweep from a scratch copy of the install). Measured tonight: see the bench log entry that follows the PC 1 run. The 9070 XT left PC 1's bus at 20:40 UTC and the 5090 needs the administrator prompt the project lead cannot answer asleep, so tonight's PC 1 run is the baseline plan on the 5090 through the whole pipeline; the two-knob tune on both cards is owed. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
110 lines
7.1 KiB
JavaScript
110 lines
7.1 KiB
JavaScript
// node --test relay/test/ember.test.mjs (no dependencies; CI runs it in the site job)
|
|
// The fleet aggregation of Ember Tune records (relay/lib/ember.mjs) on a fixture of records in the shape
|
|
// app/igneum-app/src/ember.rs record_json writes: known-good (five samples converge on one point), known-bad (an
|
|
// outlier does not move the median), the de-duplication of re-sent logs, the manifest merge and the prior lookup.
|
|
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { parseRecords, dedupe, aggregate, mergeTuning, priorFor, priorLine, median, spreadPct } from '../lib/ember.mjs';
|
|
|
|
const step = (clock_mhz, power_pct, watts, mhs, mark = 'ok') => ({ clock_mhz, power_pct, limit_w: 575 * power_pct / 100, watts, mhs, eff: Number((mhs / watts).toFixed(4)), gclk: clock_mhz || 2800, mclk: 10500, tmax: 68, faults: 0, mark });
|
|
const rec = (machine, ts, chosen, over = {}) => ({
|
|
ts, machine, app: '0.3.10', os: 'windows', card: 'NVIDIA_GeForce_RTX_5090', vendor: 'nvidia', driver: '581.57', driver_major: '581', class: 'l128w16',
|
|
key: 'NVIDIA_GeForce_RTX_5090|581|l128w16', plan: 'full', steps: [step(0, 100, 290, 124.0), chosen], chosen, before: step(0, 100, 290, 124.0), eff: chosen.eff, mhs: chosen.mhs, watts: chosen.watts, ...over,
|
|
});
|
|
// five machines, each landing near 2,470 MHz at 100%: 0.55 to 0.57 MH/W
|
|
const good = [
|
|
rec('a1', 1000, step(2472, 100, 220, 123.5)),
|
|
rec('b2', 1001, step(2472, 100, 222, 123.1)),
|
|
rec('c3', 1002, step(2781, 100, 236, 123.8)),
|
|
rec('d4', 1003, step(2472, 100, 218, 123.6)),
|
|
rec('e5', 1004, step(2163, 100, 212, 122.9)),
|
|
];
|
|
const outlier = rec('f6', 1005, step(1854, 50, 130, 118.0)); // 0.908 MH/W: a card with a broken draw reading
|
|
const baseline = rec('g7', 1006, step(0, 80, 290, 122.3), { plan: 'baseline', steps: [step(0, 80, 290, 122.3)], before: null });
|
|
const amd = (machine, ts, chosen) => rec(machine, ts, chosen, { card: 'AMD_Radeon_RX_9070_XT', vendor: 'amd', driver: '32.0.15801.1', driver_major: '32', key: 'AMD_Radeon_RX_9070_XT|32|l128w16', plan: 'confirm', before: null });
|
|
|
|
test('five samples converge on the median point and an outlier does not move it', () => {
|
|
const { priors, table } = aggregate(good, { minSamples: 5 });
|
|
const p = priors['NVIDIA_GeForce_RTX_5090|581|l128w16'];
|
|
assert.ok(p, 'a prior at the sample floor');
|
|
assert.equal(p.clock_mhz, 2470, 'the median clock cap, rounded to 10 MHz');
|
|
assert.equal(p.power_pct, 100);
|
|
assert.equal(p.samples, 5);
|
|
assert.equal(p.machines, 5);
|
|
assert.ok(p.eff > 0.55 && p.eff < 0.57, `eff ${p.eff}`);
|
|
assert.ok(p.spread_pct >= 0 && p.spread_pct < 3, `spread ${p.spread_pct}`);
|
|
assert.equal(p.before_eff, Number((124 / 290).toFixed(4)));
|
|
assert.ok(p.gain_pct > 25, `gain ${p.gain_pct}% over the untuned 100% point`);
|
|
assert.equal(p.updated, '1970-01-01T00:16:44Z');
|
|
// the outlier: 0.908 MH/W at 1,854 MHz joins; the median moves by one rank at most
|
|
const with6 = aggregate(good.concat(outlier), { minSamples: 5 }).priors['NVIDIA_GeForce_RTX_5090|581|l128w16'];
|
|
assert.equal(with6.samples, 6);
|
|
assert.equal(with6.clock_mhz, 2470);
|
|
assert.equal(with6.power_pct, 100);
|
|
assert.ok(with6.eff < 0.58, `the outlier's 0.908 MH/W did not drag the median: ${with6.eff}`);
|
|
assert.ok(with6.spread_pct < 5, `spread ${with6.spread_pct}`);
|
|
assert.equal(table[0].key, 'NVIDIA_GeForce_RTX_5090|581|l128w16');
|
|
});
|
|
|
|
test('under the floor there is no prior, and baseline records never make one', () => {
|
|
const { priors, table } = aggregate(good.slice(0, 4), { minSamples: 5 });
|
|
assert.deepEqual(priors, {});
|
|
assert.equal(table[0].samples, 4, 'the table still shows the count');
|
|
const b = aggregate([baseline, baseline], { minSamples: 1 });
|
|
assert.deepEqual(b.priors, {}, 'measure-only records say what a card does, never what to set');
|
|
assert.equal(b.table[0].baseline_samples, 1, 'the duplicate upload counted once');
|
|
assert.equal(b.table[0].baseline_eff, Number((122.3 / 290).toFixed(4)));
|
|
// a marked chosen step (a faulted or hot winner cannot exist, but a record with one is ignored)
|
|
const bad = rec('h8', 1007, step(2000, 100, 200, 120, 'faulted'));
|
|
assert.deepEqual(aggregate([bad], { minSamples: 1 }).priors, {});
|
|
});
|
|
|
|
test('records are parsed out of log text and de-duplicated on machine, card and time', () => {
|
|
const line = `1791230000 TUNE ${JSON.stringify(good[0])}`;
|
|
const text = ['1791229999 status: x', line, line, `1791230001 TUNE ${JSON.stringify(good[1])}`, '1791230002 TUNE {not json'].join('\n');
|
|
const rs = parseRecords(text);
|
|
assert.equal(rs.length, 3);
|
|
assert.equal(dedupe(rs).length, 2);
|
|
assert.equal(parseRecords('').length, 0);
|
|
});
|
|
|
|
test('the manifest merge keeps the kernel-variant cards and carries the settings', () => {
|
|
const existing = { updated: '2026-10-04T21:00:00Z', window_days: 7, cards: { NVIDIA_GeForce_RTX_5090: { variant: 'u2-ldg', race: true, candidates: ['u2-ldg', 'ldg', 'base'] } }, priors: { 'old|1|v2': { samples: 9 } } };
|
|
const { priors } = aggregate(good, { minSamples: 5 });
|
|
const t = mergeTuning(existing, priors, { rate_tolerance_pct: 1 });
|
|
assert.equal(t.cards.NVIDIA_GeForce_RTX_5090.variant, 'u2-ldg', 'lever 2 untouched');
|
|
assert.equal(t.window_days, 7);
|
|
assert.deepEqual(t.ember, { enabled: true, min_samples: 5, rate_tolerance_pct: 1 });
|
|
assert.ok(!t.priors['old|1|v2'], 'a key without samples in the window drops out');
|
|
assert.ok(t.priors['NVIDIA_GeForce_RTX_5090|581|l128w16']);
|
|
// the kill switch rides the same section
|
|
assert.equal(mergeTuning(existing, {}, { enabled: false }).ember.enabled, false);
|
|
assert.deepEqual(mergeTuning(null, {}).cards, {});
|
|
// the round trip: canonical JSON (what publish-manifest.sh signs) parses back to the same prior
|
|
const back = JSON.parse(JSON.stringify(t));
|
|
assert.deepEqual(priorFor(back, 'NVIDIA_GeForce_RTX_5090|581|l128w16'), { clock_mhz: 2470, power_pct: 100, eff: priors['NVIDIA_GeForce_RTX_5090|581|l128w16'].eff, samples: 5 });
|
|
assert.equal(priorFor(back, 'NVIDIA_GeForce_RTX_5090|581|l128w16', 6), null, 'six wanted, five there');
|
|
assert.equal(priorFor(back, 'nothing|0|v2'), null);
|
|
assert.equal(priorFor(null, 'x'), null);
|
|
});
|
|
|
|
test('AMD confirm records aggregate by their own key, and the line reads', () => {
|
|
const rs = [amd('p1', 2000, step(2600, 90, 177, 17.7)), amd('p2', 2001, step(2600, 90, 180, 17.6)), amd('p3', 2002, step(2500, 90, 170, 17.4))];
|
|
const { priors, table } = aggregate(rs, { minSamples: 3 });
|
|
const p = priors['AMD_Radeon_RX_9070_XT|32|l128w16'];
|
|
assert.equal(p.clock_mhz, 2600);
|
|
assert.equal(p.power_pct, 90);
|
|
assert.equal(p.vendor, 'amd');
|
|
assert.equal(p.gain_pct, undefined, 'confirm records carry no before step and no baseline was uploaded');
|
|
assert.match(priorLine(p), /^AMD Radeon RX 9070 XT \| driver 32 \| l128w16: 2600 MHz at 90%, 0\.\d+ MH\/W, 17\.6 MH\/s at 177 W, spread \d+(\.\d+)?%, 3 sample\(s\) from 3 machine\(s\)$/);
|
|
assert.equal(table.length, 1);
|
|
});
|
|
|
|
test('median and spread', () => {
|
|
assert.equal(median([3, 1, 2]), 2);
|
|
assert.equal(median([4, 1, 2, 3]), 2.5);
|
|
assert.equal(median([]), 0);
|
|
assert.equal(spreadPct([1, 1, 1]), 0);
|
|
assert.equal(spreadPct([10]), 0);
|
|
assert.equal(spreadPct([9, 10, 11]), 10);
|
|
});
|