igneum/tools/tuning.mjs

#!/usr/bin/env node
// Fleet learning for the GPU workers' kernel variants (docs/design/miner-tuning.md). Runs on the Mac.
//
// Every app logs one `TUNING {json}` line per hourly race (src/engine.rs race_line: card model, driver, program
// class, every variant's MH/s, the winner, power cap, MH per watt). The app log reaches the intake (Neon table
// miner_logs, the same one tools/logs.mjs reads). This script aggregates those records per card model and writes
// the tuning object the over-the-air manifest carries back to every machine (publish-manifest.sh --tuning):
//
//   node tools/tuning.mjs                    the table: per card model, per variant, samples, median MH/s, MH per watt
//   node tools/tuning.mjs --write tuning.json [--days 7] [--min-samples 3] [--by mhs|mhw] [--pin]
//        writes {"updated": ..., "cards": {"<card>": {"variant", "race", "candidates", "samples", "mhs", "base_mhs",
//        "gain_pct", "mh_per_w"}}}. The winner is the variant with the best median over the window; "candidates" are
//        the top three, which the workers race first at every prepare. --pin sets race false (the worker uses the
//        variant without a race; the record then carries only the self-test), the default keeps racing (the fleet keeps
//        learning while the card starts from the known best).
//   node tools/tuning.mjs --records [--days 7] [--card <model>]     the raw records, newest first
//   node tools/tuning.mjs --priors [--days 30] [--min-samples 5]     Ember Tune (docs/plans/ember-tune.md): the fleet
//        priors per (card model, driver major, program class) from the TUNE records (relay/lib/ember.mjs aggregate);
//        --write adds them to the tuning file under "priors" with the "ember" settings (kill switch --tuning-off,
//        --rate-tolerance N), beside the kernel-variant cards; --site writes site/miner-priors.json for /miners.
//
// Reads DATABASE_URL from ~/.config/igneum/env. No dependencies: Neon HTTP SQL over fetch.
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
import { homedir } from 'node:os';
import { join, dirname, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseRecords, aggregate, mergeTuning, priorLine } from '../relay/lib/ember.mjs';

process.stdout.on('error', e => { if (e.code === 'EPIPE') process.exit(0); throw e; });

const env = readFileSync(`${homedir()}/.config/igneum/env`, 'utf8');
const m = /^DATABASE_URL=(.*)$/m.exec(env);
if (!m) { console.error('DATABASE_URL not found in ~/.config/igneum/env'); process.exit(1); }
const url = m[1].trim().replace(/^['"]|['"]$/g, '');
const host = new URL(url).hostname.replace('-pooler', '');

async function sql(query, params = []) {
  const r = await fetch(`https://${host}/sql`, {
    method: 'POST',
    headers: { 'Neon-Connection-String': url, 'Content-Type': 'application/json' },
    body: JSON.stringify({ query, params }),
  });
  const j = await r.json();
  if (!r.ok) throw new Error(j.message || JSON.stringify(j));
  return j.rows;
}

const args = process.argv.slice(2);
const opt = (name, def) => { const i = args.indexOf(name); return i >= 0 && i + 1 < args.length ? args[i + 1] : def; };
const flag = name => args.includes(name);
const days = Number(opt('--days', 7));
const minSamples = Number(opt('--min-samples', 3));
const by = opt('--by', 'mhs');
const outFile = opt('--write', '');
const onlyCard = opt('--card', '');
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');

// ---- Ember Tune priors (--priors): the TUNE records of the window, folded per key --------------------------------
if (flag('--priors')) {
  const pdays = Number(opt('--days', 30));
  const pmin = Number(opt('--min-samples', 5));
  const prow = await sql(
    `SELECT machine, run_id, lines FROM miner_logs WHERE received_at > now() - ($1 || ' days')::interval AND lines LIKE '%TUNE {%' ORDER BY received_at DESC`,
    [String(pdays)]);
  const recs = prow.flatMap(r => parseRecords(r.lines)).filter(r => !onlyCard || r.card === onlyCard);
  const { priors, table } = aggregate(recs, { minSamples: pmin });
  if (!recs.length) console.log(`No TUNE records in the last ${pdays} day(s). The apps log one per finished tune (0.3.10 and later).`);
  else {
    console.log(`${recs.length} tune record(s) in the last ${pdays} day(s); a prior needs ${pmin} sample(s)`);
    console.table(table.map(t => ({ key: t.key, samples: t.samples, machines: t.machines, 'clock MHz': t.clock_mhz ?? '-', 'power %': t.power_pct ?? '-', 'MH/W': t.eff ?? '-', 'MH/s': t.mhs ?? '-', W: t.watts ?? '-', 'spread %': t.spread_pct ?? '-', 'untuned MH/W': t.before_eff ?? t.baseline_eff ?? '-', 'gain %': t.gain_pct ?? '-', prior: t.key in priors ? 'yes' : 'no' })));
    for (const p of Object.values(priors)) console.log(priorLine(p));
  }
  if (outFile) {
    const existing = existsSync(outFile) ? JSON.parse(readFileSync(outFile, 'utf8')) : {};
    const ember = {};
    if (flag('--tuning-off')) ember.enabled = false;
    if (flag('--tuning-on')) ember.enabled = true;
    if (opt('--rate-tolerance', '')) ember.rate_tolerance_pct = Number(opt('--rate-tolerance', '1'));
    ember.min_samples = pmin;
    const merged = mergeTuning(existing, priors, ember);
    writeFileSync(outFile, JSON.stringify(merged, null, 2) + '\n');
    console.log(`written ${outFile}: ${Object.keys(merged.cards).length} kernel-variant card(s) kept, ${Object.keys(priors).length} prior(s), ember ${JSON.stringify(merged.ember)}; publish with: packaging/ota/publish-manifest.sh --version <current> --tuning ${outFile} [--deploy]`);
  }
  if (flag('--site')) {
    const site = join(ROOT, 'site', 'miner-priors.json');
    const rows = table.map(t => ({ card: t.card.replace(/_/g, ' '), vendor: t.vendor, driver_major: t.driver_major, class: t.class, clock_mhz: t.clock_mhz ?? null, power_pct: t.power_pct ?? null, mh_per_w: t.eff ?? null, mh_s: t.mhs ?? null, watts: t.watts ?? null, spread_pct: t.spread_pct ?? null, samples: t.samples, machines: t.machines, untuned_mh_per_w: t.before_eff ?? t.baseline_eff ?? null, gain_pct: t.gain_pct ?? null, prior: t.key in priors, updated: t.updated || null }));
    const about = existsSync(site) ? JSON.parse(readFileSync(site, 'utf8'))._about : undefined;
    writeFileSync(site, JSON.stringify({ _about: about || 'Rows of the fleet priors table at /miners (site/build.mjs), written by tools/tuning.mjs --priors --site from the TUNE records every Igneum Miner uploads. One row per card model, driver major and program class: the median tuned point, MH per watt, the spread and the sample count. No machine names, no addresses.', generated: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'), min_samples: pmin, rows }, null, 2) + '\n');
    console.log(`written ${site} (${rows.length} row(s))`);
  }
  process.exit(0);
}

// Every app-log upload of the window; the TUNING lines out of them. The same race is uploaded many times (the log
// is re-sent every minute), so records are de-duplicated on (machine, card, epoch).
const rows = await sql(
  `SELECT machine, run_id, lines FROM miner_logs WHERE received_at > now() - ($1 || ' days')::interval AND lines LIKE '%TUNING {%' ORDER BY received_at DESC`,
  [String(days)]);
const seen = new Set();
const records = [];
for (const r of rows) {
  for (const line of r.lines.split('\n')) {
    const i = line.indexOf('TUNING {');
    if (i < 0) continue;
    let rec;
    try { rec = JSON.parse(line.slice(i + 7)); } catch { continue; }
    if (!rec.card || !rec.winner) continue;
    const key = `${rec.machine}|${rec.card}|${rec.epoch}`;
    if (seen.has(key)) continue;
    seen.add(key);
    rec.run_id = r.run_id;
    records.push(rec);
  }
}
records.sort((a, b) => (b.ts || 0) - (a.ts || 0));

if (flag('--records')) {
  for (const r of records) if (!onlyCard || r.card === onlyCard) console.log(JSON.stringify(r));
  process.exit(0);
}

const median = xs => { const s = [...xs].sort((a, b) => a - b); return s.length ? (s.length % 2 ? s[(s.length - 1) / 2] : (s[s.length / 2 - 1] + s[s.length / 2]) / 2) : 0; };

// Per card model (the worker's device name), per variant: every timed MH/s of every race (a race that could not time a
// variant leaves it null), plus MH per watt from the race's power draw when the card reported one.
const cards = {};
for (const r of records) {
  if (onlyCard && r.card !== onlyCard) continue;
  const c = cards[r.card] ??= { worker: r.worker, vendor: r.vendor, machines: new Set(), races: 0, variants: {}, base: [], power: [] };
  c.machines.add(r.machine);
  c.races += 1;
  if (r.base_mhs > 0) c.base.push(r.base_mhs);
  if (r.power_w > 0) c.power.push(r.power_w);
  for (const [name, mhs] of Object.entries(r.variants || {})) {
    if (typeof mhs !== 'number' || mhs <= 0) continue;
    const v = c.variants[name] ??= { mhs: [], mhw: [] };
    v.mhs.push(mhs);
    if (r.power_w > 0) v.mhw.push(mhs / r.power_w);
  }
}

const tuning = { updated: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'), window_days: days, cards: {} };
const tableRows = [];
for (const [card, c] of Object.entries(cards)) {
  const stats = Object.entries(c.variants).map(([name, v]) => ({ name, samples: v.mhs.length, mhs: median(v.mhs), mhw: v.mhw.length ? median(v.mhw) : 0 }));
  const baseMhs = stats.find(s => s.name === 'base')?.mhs || median(c.base);
  const eligible = stats.filter(s => s.samples >= minSamples);
  const ranked = (eligible.length ? eligible : stats).sort((a, b) => (by === 'mhw' ? b.mhw - a.mhw : b.mhs - a.mhs));
  for (const s of stats.sort((a, b) => b.mhs - a.mhs)) {
    tableRows.push({ card, variant: s.name, samples: s.samples, 'median MH/s': Number(s.mhs.toFixed(3)), 'vs base %': baseMhs ? Number(((s.mhs / baseMhs - 1) * 100).toFixed(2)) : null, 'MH/W': s.mhw ? Number(s.mhw.toFixed(3)) : null });
  }
  if (!ranked.length) continue;
  const best = ranked[0];
  tuning.cards[card] = {
    variant: best.name,
    race: !flag('--pin'),
    candidates: ranked.slice(0, 3).map(s => s.name),
    samples: best.samples,
    races: c.races,
    machines: c.machines.size,
    mhs: Number(best.mhs.toFixed(3)),
    base_mhs: Number(baseMhs.toFixed(3)),
    gain_pct: baseMhs ? Number(((best.mhs / baseMhs - 1) * 100).toFixed(2)) : 0,
    mh_per_w: best.mhw ? Number(best.mhw.toFixed(3)) : 0,
    worker: c.worker,
    by,
  };
}

if (!records.length) { console.log(`No TUNING records in the last ${days} day(s). The workers log one per hourly race once the app with the race is on the machines.`); process.exit(0); }
console.log(`${records.length} race record(s) from ${new Set(records.map(r => r.machine)).size} machine(s), last ${days} day(s), winner by ${by === 'mhw' ? 'MH per watt' : 'median MH/s'}, at least ${minSamples} sample(s) to be eligible`);
console.table(tableRows);
for (const [card, t] of Object.entries(tuning.cards)) console.log(`${card}: ${t.variant} (${t.samples} samples, ${t.mhs} MH/s, ${t.gain_pct >= 0 ? '+' : ''}${t.gain_pct}% over base ${t.base_mhs}${t.mh_per_w ? `, ${t.mh_per_w} MH/W` : ''}); candidates ${t.candidates.join(', ')}; race ${t.race}`);
if (outFile) {
  writeFileSync(outFile, JSON.stringify(tuning, null, 2) + '\n');
  console.log(`written ${outFile}; publish with: packaging/ota/publish-manifest.sh --version <current> --tuning ${outFile} [--deploy]`);
}