igneum/tools/tuning.mjs

#!/usr/bin/env node
// Fleet learning for the GPU workers' kernel variants (docs/design/miner-tuning.md). Runs on the Mac.
//
// Every app logs one `TUNING {json}` line per hourly race (src/engine.rs race_line: card model, driver, program
// class, every variant's MH/s, the winner, power cap, MH per watt). The app log reaches the intake (Neon table
// miner_logs, the same one tools/logs.mjs reads). This script aggregates those records per card model and writes
// the tuning object the over-the-air manifest carries back to every machine (publish-manifest.sh --tuning):
//
//   node tools/tuning.mjs                    the table: per card model, per variant, samples, median MH/s, MH per watt
//   node tools/tuning.mjs --write tuning.json [--days 7] [--min-samples 3] [--by mhs|mhw] [--pin]
//        writes {"updated": ..., "cards": {"<card>": {"variant", "race", "candidates", "samples", "mhs", "base_mhs",
//        "gain_pct", "mh_per_w"}}}. The winner is the variant with the best median over the window; "candidates" are
//        the top three, which the workers race first at every prepare. --pin sets race false (the worker uses the
//        variant without a race; the record then carries only the self-test), the default keeps racing (the fleet keeps
//        learning while the card starts from the known best).
//   node tools/tuning.mjs --records [--days 7] [--card <model>]     the raw records, newest first
//
// Reads DATABASE_URL from ~/.config/igneum/env. No dependencies: Neon HTTP SQL over fetch.
import { readFileSync, writeFileSync } from 'node:fs';
import { homedir } from 'node:os';

process.stdout.on('error', e => { if (e.code === 'EPIPE') process.exit(0); throw e; });

const env = readFileSync(`${homedir()}/.config/igneum/env`, 'utf8');
const m = /^DATABASE_URL=(.*)$/m.exec(env);
if (!m) { console.error('DATABASE_URL not found in ~/.config/igneum/env'); process.exit(1); }
const url = m[1].trim().replace(/^['"]|['"]$/g, '');
const host = new URL(url).hostname.replace('-pooler', '');

async function sql(query, params = []) {
  const r = await fetch(`https://${host}/sql`, {
    method: 'POST',
    headers: { 'Neon-Connection-String': url, 'Content-Type': 'application/json' },
    body: JSON.stringify({ query, params }),
  });
  const j = await r.json();
  if (!r.ok) throw new Error(j.message || JSON.stringify(j));
  return j.rows;
}

const args = process.argv.slice(2);
const opt = (name, def) => { const i = args.indexOf(name); return i >= 0 && i + 1 < args.length ? args[i + 1] : def; };
const flag = name => args.includes(name);
const days = Number(opt('--days', 7));
const minSamples = Number(opt('--min-samples', 3));
const by = opt('--by', 'mhs');
const outFile = opt('--write', '');
const onlyCard = opt('--card', '');

// Every app-log upload of the window; the TUNING lines out of them. The same race is uploaded many times (the log
// is re-sent every minute), so records are de-duplicated on (machine, card, epoch).
const rows = await sql(
  `SELECT machine, run_id, lines FROM miner_logs WHERE received_at > now() - ($1 || ' days')::interval AND lines LIKE '%TUNING {%' ORDER BY received_at DESC`,
  [String(days)]);
const seen = new Set();
const records = [];
for (const r of rows) {
  for (const line of r.lines.split('\n')) {
    const i = line.indexOf('TUNING {');
    if (i < 0) continue;
    let rec;
    try { rec = JSON.parse(line.slice(i + 7)); } catch { continue; }
    if (!rec.card || !rec.winner) continue;
    const key = `${rec.machine}|${rec.card}|${rec.epoch}`;
    if (seen.has(key)) continue;
    seen.add(key);
    rec.run_id = r.run_id;
    records.push(rec);
  }
}
records.sort((a, b) => (b.ts || 0) - (a.ts || 0));

if (flag('--records')) {
  for (const r of records) if (!onlyCard || r.card === onlyCard) console.log(JSON.stringify(r));
  process.exit(0);
}

const median = xs => { const s = [...xs].sort((a, b) => a - b); return s.length ? (s.length % 2 ? s[(s.length - 1) / 2] : (s[s.length / 2 - 1] + s[s.length / 2]) / 2) : 0; };

// Per card model (the worker's device name), per variant: every timed MH/s of every race (a race that could not time a
// variant leaves it null), plus MH per watt from the race's power draw when the card reported one.
const cards = {};
for (const r of records) {
  if (onlyCard && r.card !== onlyCard) continue;
  const c = cards[r.card] ??= { worker: r.worker, vendor: r.vendor, machines: new Set(), races: 0, variants: {}, base: [], power: [] };
  c.machines.add(r.machine);
  c.races += 1;
  if (r.base_mhs > 0) c.base.push(r.base_mhs);
  if (r.power_w > 0) c.power.push(r.power_w);
  for (const [name, mhs] of Object.entries(r.variants || {})) {
    if (typeof mhs !== 'number' || mhs <= 0) continue;
    const v = c.variants[name] ??= { mhs: [], mhw: [] };
    v.mhs.push(mhs);
    if (r.power_w > 0) v.mhw.push(mhs / r.power_w);
  }
}

const tuning = { updated: new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'), window_days: days, cards: {} };
const tableRows = [];
for (const [card, c] of Object.entries(cards)) {
  const stats = Object.entries(c.variants).map(([name, v]) => ({ name, samples: v.mhs.length, mhs: median(v.mhs), mhw: v.mhw.length ? median(v.mhw) : 0 }));
  const baseMhs = stats.find(s => s.name === 'base')?.mhs || median(c.base);
  const eligible = stats.filter(s => s.samples >= minSamples);
  const ranked = (eligible.length ? eligible : stats).sort((a, b) => (by === 'mhw' ? b.mhw - a.mhw : b.mhs - a.mhs));
  for (const s of stats.sort((a, b) => b.mhs - a.mhs)) {
    tableRows.push({ card, variant: s.name, samples: s.samples, 'median MH/s': Number(s.mhs.toFixed(3)), 'vs base %': baseMhs ? Number(((s.mhs / baseMhs - 1) * 100).toFixed(2)) : null, 'MH/W': s.mhw ? Number(s.mhw.toFixed(3)) : null });
  }
  if (!ranked.length) continue;
  const best = ranked[0];
  tuning.cards[card] = {
    variant: best.name,
    race: !flag('--pin'),
    candidates: ranked.slice(0, 3).map(s => s.name),
    samples: best.samples,
    races: c.races,
    machines: c.machines.size,
    mhs: Number(best.mhs.toFixed(3)),
    base_mhs: Number(baseMhs.toFixed(3)),
    gain_pct: baseMhs ? Number(((best.mhs / baseMhs - 1) * 100).toFixed(2)) : 0,
    mh_per_w: best.mhw ? Number(best.mhw.toFixed(3)) : 0,
    worker: c.worker,
    by,
  };
}

if (!records.length) { console.log(`No TUNING records in the last ${days} day(s). The workers log one per hourly race once the app with the race is on the machines.`); process.exit(0); }
console.log(`${records.length} race record(s) from ${new Set(records.map(r => r.machine)).size} machine(s), last ${days} day(s), winner by ${by === 'mhw' ? 'MH per watt' : 'median MH/s'}, at least ${minSamples} sample(s) to be eligible`);
console.table(tableRows);
for (const [card, t] of Object.entries(tuning.cards)) console.log(`${card}: ${t.variant} (${t.samples} samples, ${t.mhs} MH/s, ${t.gain_pct >= 0 ? '+' : ''}${t.gain_pct}% over base ${t.base_mhs}${t.mh_per_w ? `, ${t.mh_per_w} MH/W` : ''}); candidates ${t.candidates.join(', ')}; race ${t.race}`);
if (outFile) {
  writeFileSync(outFile, JSON.stringify(tuning, null, 2) + '\n');
  console.log(`written ${outFile}; publish with: packaging/ota/publish-manifest.sh --version <current> --tuning ${outFile} [--deploy]`);
}