942a58a0ef
tools/workers/collect.mjs runs every 30 s on the box (infra/build-server/workers/igneum-workers.{service,timer},
install.sh) and writes /srv/workers/workers.json from the machine itself: /proc/stat deltas per core, meminfo, df on
/srv, net bytes, hwmon temperatures, the flock state of /srv/builds/_locks, cargo processes with worktree, target and
start time, flock waiters, /srv/builds/_log/builds.jsonl (the format agreed with the build-server agent), sccache
--show-stats, headline.json. tools/workers/push.mjs (launchd every 60 s) adds the Mac's with-lock slots and waiters
and the two PCs' relay job states from the intake, drops them on the box so its file is whole, merges the box's file
and writes the fleet folder's workers.json; it deploys only when the live copy is over 6 min old, otherwise the
fleet orchestrator's 5-minute deploy carries it. The page (tools/workers/page/workers.html, shape.js) tries
https://build.igneum.network/workers.json first and falls back to the folder copy; core strip, arcs, now building,
queue, recently done with closing lines, headline timings, analytics; UK time with UTC tooltips; phone width.
node --test tools/workers/test: 13 tests over /proc/stat, lock dir, JSONL and sccache fixtures and the page shaping.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
154 lines
11 KiB
JavaScript
154 lines
11 KiB
JavaScript
// node --test tools/workers/test : the collector's parsers against the fixtures, and the page's data shaping.
|
|
import { test } from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { readFileSync, readdirSync } from 'node:fs';
|
|
import { join, dirname } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { parseProcStat, cpuBusy, parseMeminfo, parseNetDev, parseLoadavg, parseSlotLine, splitLabel, deriveKind, parseLockDir, parseBuildsJsonl, parseSccacheStats, pickTemps, cargoProcess, summaryOf } from '../lib.mjs';
|
|
import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, closingLines, ukTime, headline, quantile } from '../shape.js';
|
|
|
|
const FX = join(dirname(fileURLToPath(import.meta.url)), '..', 'fixtures');
|
|
const fx = n => readFileSync(join(FX, n), 'utf8');
|
|
|
|
test('/proc/stat: cores and busy deltas', () => {
|
|
const a = parseProcStat(fx('proc-stat-0.txt')), b = parseProcStat(fx('proc-stat-1.txt'));
|
|
assert.equal(a.cores.length, 4);
|
|
const busy = cpuBusy(a, b);
|
|
assert.equal(busy.per_core.length, 4);
|
|
assert.equal(busy.per_core[0], 100); // 100 user jiffies, 0 idle
|
|
assert.equal(busy.per_core[1], 0); // idle only
|
|
assert.equal(busy.per_core[2], 100); // 50 user + 50 system
|
|
assert.equal(busy.per_core[3], 25); // 25 busy of 100
|
|
assert.equal(busy.cores_busy, 2);
|
|
// whole box: 1000 user + 200 system busy of 2000 -> 60 %
|
|
assert.equal(busy.busy_pct, 60);
|
|
// a zero delta never divides by zero
|
|
assert.equal(cpuBusy(a, a).busy_pct, 0);
|
|
});
|
|
|
|
test('/proc/meminfo, /proc/net/dev, /proc/loadavg', () => {
|
|
const m = parseMeminfo('MemTotal: 131556268 kB\nMemFree: 88262160 kB\nMemAvailable: 97647588 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n');
|
|
assert.equal(m.total_kb, 131556268); assert.equal(m.used_kb, 131556268 - 97647588); assert.equal(m.used_pct, 25.8);
|
|
const n = parseNetDev('Inter-| Receive | Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n lo: 17515759 1 0 0 0 0 0 0 17515759 1 0 0 0 0 0 0\nenp193s0f0np0: 994079034 2 0 0 0 0 0 0 80040958 3 0 0 0 0 0 0\n');
|
|
assert.deepEqual(n, [{ iface: 'enp193s0f0np0', rx_bytes: 994079034, tx_bytes: 80040958 }]);
|
|
assert.deepEqual(parseLoadavg('13.86 9.61 4.32 1/1235 36952\n'), { load: [13.86, 9.61, 4.32], running: 1, threads: 1235 });
|
|
});
|
|
|
|
test('slot line: pid, since, waited, label, agent, worktree, crate, command, target', () => {
|
|
const l = parseSlotLine('pid 36992 since 17:34:00Z waited 0 s: igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server', { day: '2026-10-06', now: '2026-10-06T18:00:00Z' });
|
|
assert.equal(l.pid, 36992); assert.equal(l.since, '2026-10-06T17:34:00Z'); assert.equal(l.waited_s, 0);
|
|
assert.equal(l.worktree, 'igneum-wt-build-server'); assert.equal(l.crate, 'vendor/igneum-node-bs');
|
|
assert.equal(l.target, 'x86_64-pc-windows-gnu'); assert.equal(l.agent, 'build-server');
|
|
assert.equal(deriveKind(l), 'node-windows');
|
|
// a clock time later than now is yesterday
|
|
const y = parseSlotLine('pid 5 since 23:50:00Z waited 2 s: x', { day: '2026-10-06', now: '2026-10-06T00:10:00Z' });
|
|
assert.equal(y.since, '2026-10-05T23:50:00Z');
|
|
assert.equal(parseSlotLine(''), null); assert.equal(parseSlotLine('garbage'), null);
|
|
// a Mac with-lock label is a bare command
|
|
const m = splitLabel('cargo build --release -p igneum-miner');
|
|
assert.equal(m.worktree, null); assert.equal(m.target, 'x86_64-unknown-linux-gnu');
|
|
});
|
|
|
|
test('kind rule matches the JSONL proposal', () => {
|
|
assert.equal(deriveKind({ command: 'cargo test --release -p kaspa-consensus-core', crate: 'vendor/igneum-node-v4' }), 'suite');
|
|
assert.equal(deriveKind({ command: 'cargo clippy --all-targets', crate: 'app/igneum-app' }), 'check');
|
|
assert.equal(deriveKind({ command: 'cargo build --release -p kaspad', crate: 'vendor/igneum-node-bs' }), 'node-linux');
|
|
assert.equal(deriveKind({ command: 'cargo build --release -p kaspad --target x86_64-pc-windows-gnu', crate: 'vendor/igneum-node-bs' }), 'node-windows');
|
|
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'app/igneum-app' }), 'app');
|
|
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'app/igneum-app', tool: 'cross-remote' }), 'app-windows');
|
|
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'proving/igneum-prove' }), 'prove');
|
|
assert.equal(deriveKind({ kind: 'suite', command: 'cargo build' }), 'suite'); // an explicit kind wins
|
|
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'tools/x' }), 'other');
|
|
});
|
|
|
|
test('lock dir fixture: one held slot, one stale line, one empty, count from slots', () => {
|
|
const dir = join(FX, 'locks');
|
|
const files = readdirSync(dir).map(name => ({ name, text: readFileSync(join(dir, name), 'utf8') }));
|
|
const alive = pid => pid === 36992; // only the real holder is alive; pid 1's line is stale
|
|
const r = parseLockDir(files, { day: '2026-10-06', now: '2026-10-06T18:00:00Z', aliveFn: alive });
|
|
assert.equal(r.count, 1);
|
|
assert.equal(r.held.length, 1); assert.equal(r.held[0].slot, 'build-0'); assert.equal(r.held[0].kind, 'node-windows'); assert.equal(r.held[0].agent, 'build-server');
|
|
assert.deepEqual(r.free, ['build-1', 'build-2']);
|
|
// an explicit flock verdict overrides the pid check
|
|
const r2 = parseLockDir(files.map(f => ({ ...f, held: f.name === 'build-1' })), { day: '2026-10-06', now: '2026-10-06T18:00:00Z' });
|
|
assert.deepEqual(r2.held.map(h => h.slot), ['build-1']);
|
|
});
|
|
|
|
test('builds JSONL fixture: five rows, one bad line, newest first, derived fields', () => {
|
|
const r = parseBuildsJsonl(fx('builds.jsonl'));
|
|
assert.equal(r.total, 5); assert.equal(r.bad, 1);
|
|
assert.equal(r.rows[0].id, 'igneum-build-1-1759774200-37300');
|
|
const win = r.rows.find(b => b.tool === 'cross-remote');
|
|
assert.equal(win.kind, 'node-windows'); assert.equal(win.ok, true); assert.equal(win.sccache.hit_rate_pct, 0); assert.equal(win.artefacts.length, 2);
|
|
const suite = r.rows.find(b => b.exit === 101);
|
|
assert.equal(suite.kind, 'suite'); assert.equal(suite.ok, false);
|
|
const lin = r.rows.find(b => b.id.endsWith('37001'));
|
|
assert.equal(lin.kind, 'node-linux'); assert.equal(lin.target, 'x86_64-unknown-linux-gnu'); assert.equal(lin.sccache.hit_rate_pct, 71.7);
|
|
// secs derived from start and end when missing
|
|
const d = parseBuildsJsonl('{"start":"2026-10-06T10:00:00Z","end":"2026-10-06T10:05:30Z","exit":0,"command":"cargo build"}');
|
|
assert.equal(d.rows[0].secs, 330);
|
|
});
|
|
|
|
test('sccache --show-stats', () => {
|
|
const s = parseSccacheStats(fx('sccache-stats.txt'));
|
|
assert.equal(s.requests, 2294); assert.equal(s.hits, 790); assert.equal(s.misses, 1886); assert.equal(s.hit_rate_pct, 29.5);
|
|
assert.equal(s.cache_size, '12 GiB'); assert.equal(s.max_size, '100 GiB');
|
|
assert.equal(parseSccacheStats('Cache hits 0\nCache misses 10\n').hit_rate_pct, 0);
|
|
});
|
|
|
|
test('hwmon pick: hottest per class, cpu first, nothing absurd', () => {
|
|
const t = pickTemps([{ name: 'bnxt_en', label: null, millic: 55000 }, { name: 'nvme', label: 'Composite', millic: 36850 }, { name: 'nvme', label: 'Composite', millic: 34850 }, { name: 'k10temp', label: 'Tctl', millic: 75125 }, { name: 'k10temp', label: 'Tccd1', millic: 0 }, { name: 'acpitz', label: null, millic: 999000 }]);
|
|
assert.deepEqual(t.map(x => [x.name, x.c]), [['cpu', 75.1], ['nvme', 36.9], ['nic', 55]]);
|
|
});
|
|
|
|
test('a cargo process becomes a running build with its worktree, target, kind and start', () => {
|
|
const stat = '37210 (cargo) S 37209 37210 36992 0 -1 4194304 1 0 0 0 1 2 0 0 20 0 2 0 62341 1 2 3 4 5 6 7 8 9';
|
|
const p = cargoProcess({ pid: 37210, cwd: '/srv/builds/igneum-wt-build-server/vendor/igneum-node-bs', cmdline: ['/home/build/.rustup/toolchains/1.99.0-x86_64-unknown-linux-gnu/bin/cargo','build','--release','-p','kaspad','--target','x86_64-pc-windows-gnu','-j','90',''].join('\0'), stat, bootSec: 1759772000, clk: 100, now: (1759772000 + 1000) * 1000 });
|
|
assert.equal(p.worktree, 'igneum-wt-build-server'); assert.equal(p.crate, 'vendor/igneum-node-bs');
|
|
assert.equal(p.target, 'x86_64-pc-windows-gnu'); assert.equal(p.kind, 'node-windows');
|
|
assert.equal(p.started_at, new Date((1759772000 + 623.41) * 1000).toISOString().replace(/\.\d{3}Z$/, 'Z'));
|
|
assert.equal(p.elapsed_s, 377);
|
|
assert.equal(cargoProcess({ pid: 1, cmdline: ['rustc','--crate-name','x'].join('\0') }), null);
|
|
});
|
|
|
|
test('relay job report: SUMMARY and closing line, running without finished_at', () => {
|
|
const done = summaryOf('IGNEUM-APP 0.3.14\nSUMMARY {"job":"build-20261006-171423","kind":"build","status":"done","exit":0,"duration_s":622,"started_at":"2026-10-06T17:15:09Z","finished_at":"2026-10-06T17:25:31Z","summary":"ok"}\nSTAGE linux start 2026-10-06T17:16:00Z cap 20 min\njob build-20261006-171423: done (exit 0) after 622 s: every stage ok\n');
|
|
assert.equal(done.status, 'done'); assert.equal(done.duration_s, 622); assert.equal(done.stage, 'linux');
|
|
const run = summaryOf('SUMMARY {"job":"j1","kind":"build","status":"running","started_at":"2026-10-06T17:15:09Z"}\nSTAGE windows start 2026-10-06T17:30:00Z cap 20 min\n');
|
|
assert.equal(run.status, 'running'); assert.equal(run.stage, 'windows');
|
|
assert.equal(summaryOf('nothing here'), null);
|
|
});
|
|
|
|
// ---- page shaping ----------------------------------------------------------------------------------------------------------
|
|
const builds = parseBuildsJsonl(fx('builds.jsonl')).rows;
|
|
|
|
test('shape: median and p90 by kind, builds per hour, hit-rate series', () => {
|
|
const d = durationByKind(builds);
|
|
const nl = d.find(x => x.kind === 'node-linux'); assert.equal(nl.n, 1); assert.equal(nl.median_s, 801); assert.equal(nl.p90_s, 801);
|
|
assert.equal(quantile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 0.9), 9.1);
|
|
const h = buildsPerHour(builds, '2026-10-06');
|
|
assert.equal(h.length, 24); assert.equal(h[17].n, 1); assert.equal(h[18].n, 4); assert.equal(h[18].failed, 1); assert.equal(h[3].n, 0);
|
|
const s = hitRateSeries(builds);
|
|
assert.deepEqual(s.map(x => x.pct), [0, 71.7, 77.6, 85.7, 93.8]);
|
|
});
|
|
|
|
test('shape: box against Mac, timeline bars, closing lines, headline', () => {
|
|
const c = compareKinds(builds, { 'node-linux': { secs: 720, source: 'infra/cross/build-linux.sh, 4 Oct' }, 'app-windows': { secs: 900, source: 'runner' } });
|
|
const nl = c.find(x => x.kind === 'node-linux'); assert.equal(nl.mac_s, 720); assert.equal(nl.speedup, 0.9);
|
|
const aw = c.find(x => x.kind === 'app-windows'); assert.equal(aw.box_s, null); assert.equal(aw.mac_s, 900);
|
|
const t = timeline(builds, { limit: 3 });
|
|
assert.equal(t.length, 3); assert.equal(t[0].id, 'igneum-build-1-1759774200-37300');
|
|
assert.equal(Math.max(...t.map(r => r.bar_pct)), 100);
|
|
const lines = closingLines(builds.find(b => b.tool === 'cross-remote'));
|
|
assert.ok(lines[0].startsWith('$ cargo build')); assert.ok(lines.some(l => /igneumd\.exe\s+50\.5 MB\s+sha256 0f3a9c2e1d8b7a6f/.test(l)));
|
|
const hl = headline({ clean_node: { box_s: 600, mac_s: 2400 } });
|
|
assert.equal(hl[0].speedup, 4); assert.equal(hl[1].box_s, null); assert.equal(hl.length, 3);
|
|
});
|
|
|
|
test('shape: UK local time with the UTC in the tooltip', () => {
|
|
const t = ukTime('2026-10-06T17:34:00Z');
|
|
assert.equal(t.text, '18:34'); // BST on 6 October
|
|
assert.equal(t.utc, '2026-10-06 17:34:00 UTC');
|
|
assert.equal(ukTime('2026-12-06T17:34:00Z').text, '17:34'); // GMT in December
|
|
assert.deepEqual(ukTime(null), { text: '', utc: '' });
|
|
});
|