igneum/tools/workers/lib.mjs

// Parsers shared by tools/workers/collect.mjs (on igneum-build-1) and tools/workers/push.mjs (on the Mac). Pure functions
// over text the machine gives us (/proc files, lock files, the builds JSONL), so node --test can run them against the
// fixtures in tools/workers/fixtures without a box. Every reader here takes the machine's own facts, never a reported number.

/// /proc/stat -> { total: [{user, nice, system, idle, iowait, irq, softirq, steal}], cores: [...] } (jiffies).
export function parseProcStat(text) {
  const cores = []; let total = null;
  for (const line of String(text).split('\n')) {
    if (!line.startsWith('cpu')) continue;
    const f = line.trim().split(/\s+/);
    const n = f.slice(1, 9).map(Number);
    const row = { user: n[0], nice: n[1], system: n[2], idle: n[3], iowait: n[4] || 0, irq: n[5] || 0, softirq: n[6] || 0, steal: n[7] || 0 };
    if (f[0] === 'cpu') total = row; else cores.push(row);
  }
  return { total, cores };
}

/// Busy percentage between two /proc/stat samples: 0 to 100 for the whole box and per core, in core order.
export function cpuBusy(before, after) {
  const pct = (a, b) => {
    if (!a || !b) return 0;
    const sum = r => r.user + r.nice + r.system + r.idle + r.iowait + r.irq + r.softirq + r.steal;
    const idle = r => r.idle + r.iowait;
    const dt = sum(b) - sum(a); const di = idle(b) - idle(a);
    if (dt <= 0) return 0;
    return Math.max(0, Math.min(100, Math.round((1 - di / dt) * 1000) / 10));
  };
  const n = Math.min(before.cores.length, after.cores.length);
  const per = []; for (let i = 0; i < n; i++) per.push(pct(before.cores[i], after.cores[i]));
  return { busy_pct: pct(before.total, after.total), per_core: per, cores_busy: per.filter(p => p >= 50).length };
}

/// /proc/meminfo -> kB numbers and a used percentage (used = total minus available, what a build can still take).
export function parseMeminfo(text) {
  const m = {}; for (const line of String(text).split('\n')) { const r = /^(\w+):\s+(\d+)/.exec(line); if (r) m[r[1]] = Number(r[2]); }
  const total = m.MemTotal || 0, avail = m.MemAvailable ?? m.MemFree ?? 0;
  return { total_kb: total, available_kb: avail, used_kb: Math.max(0, total - avail), used_pct: total ? Math.round(((total - avail) / total) * 1000) / 10 : 0, swap_total_kb: m.SwapTotal || 0, swap_free_kb: m.SwapFree || 0 };
}

/// /proc/net/dev -> [{ iface, rx_bytes, tx_bytes }] without lo.
export function parseNetDev(text) {
  const out = [];
  for (const line of String(text).split('\n').slice(2)) {
    const r = /^\s*([^:\s]+):\s*(.*)$/.exec(line); if (!r) continue;
    const f = r[2].trim().split(/\s+/).map(Number);
    if (r[1] === 'lo') continue;
    out.push({ iface: r[1], rx_bytes: f[0] || 0, tx_bytes: f[8] || 0 });
  }
  return out;
}

/// /proc/loadavg -> { load: [1, 5, 15], running, threads }.
export function parseLoadavg(text) {
  const f = String(text).trim().split(/\s+/);
  const rq = (f[3] || '0/0').split('/');
  return { load: [Number(f[0]) || 0, Number(f[1]) || 0, Number(f[2]) || 0], running: Number(rq[0]) || 0, threads: Number(rq[1]) || 0 };
}

/// The slot-file line both with-lock.sh and bs_remote_run write: "pid N since HH:MM:SSZ waited S s: <label>". Returns null
/// for an empty or foreign line. `day` (YYYY-MM-DD, UTC) anchors the clock time; a time later than `now` is yesterday.
export function parseSlotLine(line, { day, now } = {}) {
  const r = /^pid (\d+) since (\d\d):(\d\d):(\d\d)Z waited (\d+) s: ?(.*)$/.exec(String(line || '').trim());
  if (!r) return null;
  const nowMs = now ? new Date(now).getTime() : Date.now();
  const d = day || new Date(nowMs).toISOString().slice(0, 10);
  let since = Date.parse(`${d}T${r[2]}:${r[3]}:${r[4]}Z`);
  if (since > nowMs + 60_000) since -= 86_400_000;
  const label = r[6].trim();
  const agent = (/;\s*agent=([^;]+)\s*$/.exec(label) || [])[1] || null;
  return { pid: Number(r[1]), since: new Date(since).toISOString().replace(/\.\d{3}Z$/, 'Z'), waited_s: Number(r[5]), label: agent ? label.replace(/;\s*agent=[^;]+\s*$/, '').trim() : label, agent, ...splitLabel(label) };
}

/// "igneum-wt-x/vendor/igneum-node-bs cross x86_64-pc-windows-gnu" or "igneum-wt-x/app/igneum-app cargo build --release ..."
/// -> { worktree, crate, command, target }. A Mac with-lock label is a plain command line: worktree and crate are null.
export function splitLabel(label) {
  const s = String(label || '').replace(/;\s*agent=[^;]+\s*$/, '').trim();
  const r = /^(igneum(?:-wt-[\w.-]+)?)\/(\S+) (cargo .*|cross .*)$/.exec(s) || /^(igneum(?:-wt-[\w.-]+)?)\/(\S+) (.*)$/.exec(s);
  if (!r) return { worktree: null, crate: null, command: s || null, target: targetOf(s) };
  let command = r[3];
  if (command.startsWith('cross ')) command = `cargo build --release --target ${command.slice(6).trim()}`;
  return { worktree: r[1], crate: r[2], command, target: targetOf(command) };
}

export function targetOf(command) {
  const r = /--target[= ](\S+)/.exec(String(command || ''));
  if (r) return r[1];
  return /\bcargo\b/.test(String(command || '')) ? 'x86_64-unknown-linux-gnu' : null;
}

/// The build's kind, from the builds-log fields or a slot label: node-linux, node-windows, app, app-windows, prove, suite,
/// check, other. The same rule as the JSONL proposal, so both sides agree.
export function deriveKind({ kind, command, crate, tool, target } = {}) {
  if (kind) return kind;
  const cmd = String(command || ''); const cr = String(crate || '');
  const sub = (/\bcargo\s+(\S+)/.exec(cmd) || [])[1] || cmd.split(/\s+/)[0] || '';
  const tgt = target || targetOf(cmd);
  const win = tgt === 'x86_64-pc-windows-gnu' || tool === 'cross-remote';
  if (sub === 'test' || sub === 'nextest') return 'suite';
  if (sub === 'check' || sub === 'clippy') return 'check';
  if (/^vendor\//.test(cr) || /\bkaspad\b/.test(cmd)) return win ? 'node-windows' : 'node-linux';
  if (/igneum-app/.test(cr)) return win ? 'app-windows' : 'app';
  if (/igneum-prove/.test(cr) || /\bprove\b/.test(cr)) return 'prove';
  if (/igneum-pow/.test(cr)) return 'pow';
  return 'other';
}

/// A lock directory listing -> { count, held: [...], free: [...] }. `files` is [{ name, text, held }] where `held` says whether
/// the flock is taken (the caller knows: flock probe on the box, pid alive on the Mac). Stale lines without a live lock are free.
export function parseLockDir(files, { day, now, aliveFn } = {}) {
  const held = [], free = [];
  let count = 1;
  for (const f of files) {
    if (f.name === 'slots') { const n = parseInt(f.text, 10); if (n >= 1) count = n; continue; }
    if (!/^(build-\d+|run-\d+|build|measure)$/.test(f.name)) continue;
    const line = parseSlotLine((f.text || '').split('\n')[0], { day, now });
    const alive = line && aliveFn ? aliveFn(line.pid) : null;
    const isHeld = f.held ?? (line ? alive !== false : false);
    if (line && isHeld) held.push({ slot: f.name, ...line, kind: deriveKind(line) });
    else free.push(f.name);
  }
  held.sort((a, b) => a.slot.localeCompare(b.slot));
  return { count, held, free: free.sort() };
}

/// The builds JSONL (one object per line, the format sent to the build-server agent). Bad lines are skipped and counted.
export function parseBuildsJsonl(text, { limit = 200 } = {}) {
  const rows = []; let bad = 0;
  for (const line of String(text).split('\n')) {
    const t = line.trim(); if (!t) continue;
    try { const o = JSON.parse(t); if (o && typeof o === 'object') rows.push(normaliseBuild(o)); else bad++; } catch { bad++; }
  }
  rows.sort((a, b) => String(b.end || b.start || '').localeCompare(String(a.end || a.start || '')));
  return { rows: rows.slice(0, limit), bad, total: rows.length };
}

export function normaliseBuild(o) {
  const start = o.start || null, end = o.end || null;
  const secs = Number.isFinite(o.secs) ? o.secs : (start && end ? Math.round((Date.parse(end) - Date.parse(start)) / 1000) : null);
  const sc = o.sccache || {};
  const hits = Number(sc.hits ?? 0), misses = Number(sc.misses ?? 0);
  return {
    ...o,
    id: o.id || `${o.host || 'box'}-${start || ''}-${o.pid || ''}`,
    kind: deriveKind(o),
    target: o.target || targetOf(o.command),
    secs,
    ok: Number(o.exit) === 0,
    sccache: { hits, misses, hit_rate_pct: hits + misses ? Math.round((hits / (hits + misses)) * 1000) / 10 : null, hits_total: sc.hits_total ?? null, misses_total: sc.misses_total ?? null },
    artefacts: Array.isArray(o.artefacts) ? o.artefacts : [],
  };
}

/// `sccache --show-stats` text -> { requests, executed, hits, misses, hit_rate_pct, cache_size, max_size }.
export function parseSccacheStats(text) {
  const num = re => { const r = re.exec(String(text)); return r ? Number(r[1].replace(/,/g, '')) : null; };
  const str = re => { const r = re.exec(String(text)); return r ? r[1].trim() : null; };
  const hits = num(/^Cache hits\s{2,}(\d[\d,]*)/m), misses = num(/^Cache misses\s{2,}(\d[\d,]*)/m);
  return {
    requests: num(/^Compile requests\s+(\d[\d,]*)/m), executed: num(/^Compile requests executed\s+(\d[\d,]*)/m),
    hits, misses, hit_rate_pct: hits !== null && misses !== null && hits + misses ? Math.round((hits / (hits + misses)) * 1000) / 10 : (hits === 0 && misses ? 0 : null),
    cache_size: str(/^Cache size\s+(.+)$/m), max_size: str(/^Max cache size\s+(.+)$/m),
  };
}

/// /sys/class/hwmon readings -> [{ name, label, c }] for the sensors worth a tile: the CPU die, the NVMe drives, the NIC.
export function pickTemps(sensors) {
  const out = [];
  for (const s of sensors) {
    const c = Number(s.millic) / 1000;
    if (!Number.isFinite(c) || c <= 0 || c > 150) continue;
    const name = s.name === 'k10temp' || s.name === 'coretemp' ? 'cpu' : s.name === 'nvme' ? 'nvme' : s.name.startsWith('bnxt') || s.name.includes('eth') ? 'nic' : s.name;
    out.push({ name, label: s.label || s.name, c: Math.round(c * 10) / 10 });
  }
  // one per name: the hottest (two NVMe drives in RAID 1, two NIC ports)
  const best = new Map();
  for (const t of out) { const cur = best.get(t.name); if (!cur || t.c > cur.c) best.set(t.name, t); }
  const order = ['cpu', 'nvme', 'nic'];
  return [...best.values()].sort((a, b) => (order.indexOf(a.name) + 99) % 99 - (order.indexOf(b.name) + 99) % 99 || a.name.localeCompare(b.name));
}

/// A cargo process -> a running build: { pid, cwd, worktree, crate, command, target, kind, started_at, elapsed_s }.
/// `stat` is /proc/<pid>/stat text, `bootSec` the boot time in unix seconds, `clk` CLK_TCK.
export function cargoProcess({ pid, cwd, cmdline, stat, bootSec, clk = 100, now = Date.now(), root = '/srv/builds' }) {
  const args = String(cmdline || '').split('\0').filter(Boolean);
  if (!args.length || !/(^|\/)cargo$/.test(args[0])) return null;
  const command = ['cargo', ...args.slice(1)].join(' ');
  const rest = stat ? stat.slice(stat.lastIndexOf(')') + 2).split(' ') : [];
  const startTicks = Number(rest[19]);   // field 22 of /proc/<pid>/stat, after pid and comm
  const started = Number.isFinite(startTicks) && bootSec ? (bootSec + startTicks / clk) * 1000 : null;
  const rel = String(cwd || '').startsWith(root + '/') ? cwd.slice(root.length + 1) : null;
  const worktree = rel ? rel.split('/')[0] : null;
  const crate = rel ? rel.split('/').slice(1).join('/') || '.' : null;
  const target = targetOf(command);
  return { pid, cwd, worktree, crate, command, target, kind: deriveKind({ command, crate, target }), started_at: started ? new Date(started).toISOString().replace(/\.\d{3}Z$/, 'Z') : null, elapsed_s: started ? Math.max(0, Math.round((now - started) / 1000)) : null };
}

/// The relay job SUMMARY line and closing line of one report (as tools/jobs.mjs reads them) -> { job, kind, status, exit,
/// duration_s, started_at, finished_at, summary, errors[] } or null.
export function summaryOf(lines) {
  const ls = String(lines || '').split('\n');
  let s = null;
  for (const l of ls) { if (l.startsWith('SUMMARY ')) { try { s = JSON.parse(l.slice(8)); } catch { s = null; } break; } }
  const close = ls.slice(-40).map(l => /^job (\S+): (done|failed|timeout|aborted) \(exit (-?\d+)\) after (\d+) s: (.*)$/.exec(l)).filter(Boolean).pop();
  if (close) { s = s || {}; s.job = s.job || close[1]; s.status = close[2]; s.exit = Number(close[3]); s.duration_s = Number(close[4]); s.summary = close[5]; s.finished_at = s.finished_at || 'closed'; }
  else if (s && !s.finished_at) s.status = 'running';
  if (s) {
    s.errors = ls.filter(l => /^(Error:|BUILD FAILED|GATE FAILED)|\bFAILED\b|panicked at/.test(l) && !l.startsWith('SUMMARY ')).slice(0, 5);
    s.stage = (ls.map(l => /^STAGE (\S+) start/.exec(l)).filter(Boolean).pop() || [])[1] || null;
  }
  return s;
}

/// Duration bars want the same scale: the longest of the lot, at least a minute.
export const fmtSecs = s => { s = Math.max(0, Math.round(Number(s) || 0)); const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), r = s % 60; return h ? `${h} h ${String(m).padStart(2, '0')} min` : m ? `${m} min ${String(r).padStart(2, '0')} s` : `${r} s`; };

/// Mac baselines for the kind comparison: approximate, from the docs that measured them; a measurement on the same tree
/// replaces them when one is logged. Shared by collect.mjs (the box's file) and push.mjs (the edge copy).
export const BASELINES = {
  'node-linux': { secs: 690, source: 'approximate: infra/cross/build-linux.sh on this Mac, 8 to 15 min per target at nice 19 with 4 to 6 jobs (docs/plans/build-job.md, 4 Oct 2026)' },
  'node-windows': { secs: 690, source: 'approximate: proto-cuda/windows-node/cross-build.sh on this Mac, the same 8 to 15 min (docs/plans/build-job.md, 4 Oct 2026)' },
};