igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

Worker dashboard: collector on igneum-build-1, Mac pusher, workers.html next to the fleet page (Josh, 6 October 2026)

Browse source 
tools/workers/collect.mjs runs every 30 s on the box (infra/build-server/workers/igneum-workers.{service,timer},
install.sh) and writes /srv/workers/workers.json from the machine itself: /proc/stat deltas per core, meminfo, df on
/srv, net bytes, hwmon temperatures, the flock state of /srv/builds/_locks, cargo processes with worktree, target and
start time, flock waiters, /srv/builds/_log/builds.jsonl (the format agreed with the build-server agent), sccache
--show-stats, headline.json. tools/workers/push.mjs (launchd every 60 s) adds the Mac's with-lock slots and waiters
and the two PCs' relay job states from the intake, drops them on the box so its file is whole, merges the box's file
and writes the fleet folder's workers.json; it deploys only when the live copy is over 6 min old, otherwise the
fleet orchestrator's 5-minute deploy carries it. The page (tools/workers/page/workers.html, shape.js) tries
https://build.igneum.network/workers.json first and falls back to the folder copy; core strip, arcs, now building,
queue, recently done with closing lines, headline timings, analytics; UK time with UTC tooltips; phone width.
node --test tools/workers/test: 13 tests over /proc/stat, lock dir, JSONL and sccache fixtures and the page shaping.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-josh 2026-10-06 18:59:07 +01:00
parent e1d0b23da4
commit d1036db7fb
21 changed files with 1376 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
infra/build-server/workers/igneum-workers.service Normal file
View file

@ -0,0 +1,21 @@
# The worker dashboard's collector on igneum-build-1: one pass, written by igneum-workers.timer every 30 s.
# Reads /proc, /sys, /srv/builds/_locks, the process table and /srv/builds/_log/builds.jsonl; writes /srv/workers/workers.json.
# Installed by infra/build-server/workers/install.sh (copies tools/workers/{lib,collect}.mjs to /srv/workers/bin).
[Unit]
Description=Igneum worker dashboard collector (one pass)
After=local-fs.target
[Service]
Type=oneshot
User=build
Group=build
Environment=HOME=/home/build
Environment=PATH=/home/build/.cargo/bin:/usr/local/bin:/usr/bin:/bin
Environment=SCCACHE_DIR=/srv/sccache
Environment=IGNEUM_WORKERS_DIR=/srv/workers
Environment=IGNEUM_BUILD_SLOTS_DIR=/srv/builds/_locks
Environment=IGNEUM_BUILD_ROOT=/srv/builds
WorkingDirectory=/srv/workers/bin
ExecStart=/usr/local/bin/node /srv/workers/bin/collect.mjs
TimeoutStartSec=25
Nice=10

13
infra/build-server/workers/igneum-workers.timer Normal file
View file

@ -0,0 +1,13 @@
# Fires the collector every 30 s from boot. `systemctl list-timers igneum-workers.timer` shows the next pass;
# `journalctl -u igneum-workers.service -n 20` the last errors.
[Unit]
Description=Igneum worker dashboard collector, every 30 s
[Timer]
OnBootSec=20s
OnUnitActiveSec=30s
AccuracySec=1s
Unit=igneum-workers.service
[Install]
WantedBy=timers.target

14
infra/build-server/workers/install.sh Executable file
View file

@ -0,0 +1,14 @@
#!/usr/bin/env bash
# Install or refresh the worker dashboard collector on igneum-build-1 from this Mac.
# infra/build-server/workers/install.sh copies tools/workers/{lib,collect}.mjs and the two units, enables the timer
# Needs root over ssh (root@<ip> with ~/.ssh/igneum_ed25519); the host ip comes from ~/.config/igneum/build-server (build@<ip>).
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../../.." && pwd)"
KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')"
IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server" >&2; exit 1; }
SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10)
"${SSH[@]}" "root@$IP" 'install -d -o build -g build /srv/workers /srv/workers/bin /srv/workers/sources /srv/builds/_log; chown build:build /srv/builds/_log'
scp -q -i "$KEY" "$ROOT/tools/workers/lib.mjs" "$ROOT/tools/workers/collect.mjs" "build@$IP:/srv/workers/bin/"
scp -q -i "$KEY" "$HERE/igneum-workers.service" "$HERE/igneum-workers.timer" "root@$IP:/etc/systemd/system/"
"${SSH[@]}" "root@$IP" 'systemctl daemon-reload && systemctl enable --now igneum-workers.timer >/dev/null 2>&1; systemctl start igneum-workers.service; systemctl is-active igneum-workers.timer; systemctl list-timers igneum-workers.timer --no-pager | sed -n 2p; ls -la /srv/workers/workers.json'

160
tools/workers/collect.mjs Normal file
View file

@ -0,0 +1,160 @@
#!/usr/bin/env node
// The worker dashboard's collector. Runs on igneum-build-1 every 30 s (infra/build-server/workers/igneum-workers.timer)
// as the build user and writes /srv/workers/workers.json: what the MACHINE says (uptime, load, /proc/stat deltas per core,
// memory, the /srv disk, network bytes, hwmon temperatures), the slot and lock state under /srv/builds/_locks, the cargo
// processes with their worktree and target, the slot waiters, the recent builds from /srv/builds/_log/builds.jsonl and
// sccache's own counters. Nothing here is a reported number: every field is read from /proc, /sys, the lock files, the
// process table or the log the build itself appended.
//
// node tools/workers/collect.mjs write /srv/workers/workers.json (IGNEUM_WORKERS_DIR overrides)
// node tools/workers/collect.mjs --stdout print instead of writing
// node tools/workers/collect.mjs --sample-ms 1000 the /proc/stat and network delta window (default 1000)
//
// The Mac side (tools/workers/push.mjs) drops its own facts at /srv/workers/sources/mac.json (the Mac's with-lock slots) and
// /srv/workers/sources/pcs.json (the two PCs' relay job states); when present and under 10 minutes old they are merged
// into the same workers.json, so one file answers for the whole shop whichever side serves it. Needs Node 22, no deps.
import { readFileSync, readdirSync, writeFileSync, renameSync, mkdirSync, existsSync, statSync, readlinkSync, openSync, closeSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { hostname } from 'node:os';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseProcStat, cpuBusy, parseMeminfo, parseNetDev, parseLoadavg, parseLockDir, parseBuildsJsonl, parseSccacheStats, pickTemps, cargoProcess, deriveKind, BASELINES } from './lib.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const argv = process.argv.slice(2);
const flag = (n, d) => { const i = argv.indexOf(`--${n}`); return i >= 0 ? (argv[i + 1] ?? true) : d; };
const OUT_DIR = process.env.IGNEUM_WORKERS_DIR || '/srv/workers';
const LOCKS = process.env.IGNEUM_BUILD_SLOTS_DIR || '/srv/builds/_locks';
const BUILDS = process.env.IGNEUM_BUILD_ROOT || '/srv/builds';
const LOG = process.env.IGNEUM_BUILDS_LOG || join(BUILDS, '_log', 'builds.jsonl');
const SAMPLE_MS = Number(flag('sample-ms', 1000));
const iso = ms => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
const read = p => { try { return readFileSync(p, 'utf8'); } catch { return ''; } };
const sh = (cmd, args, opts = {}) => { const r = spawnSync(cmd, args, { encoding: 'utf8', timeout: 8000, ...opts }); return r.status === 0 ? r.stdout : ''; };
// ---- machine facts ----------------------------------------------------------------------------------------------------
function machine() {
const stat0 = parseProcStat(read('/proc/stat')); const net0 = parseNetDev(read('/proc/net/dev')); const t0 = Date.now();
const end = t0 + SAMPLE_MS; while (Date.now() < end) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, end - Date.now());
const stat1 = parseProcStat(read('/proc/stat')); const net1 = parseNetDev(read('/proc/net/dev')); const dt = (Date.now() - t0) / 1000;
const cpu = cpuBusy(stat0, stat1);
const up = Number(read('/proc/uptime').split(' ')[0]) || 0;
const la = parseLoadavg(read('/proc/loadavg'));
const mem = parseMeminfo(read('/proc/meminfo'));
const df = sh('df', ['-B1', '--output=target,size,used,avail', BUILDS]).trim().split('\n').pop() || '';
const dfp = df.trim().split(/\s+/);
const disk = dfp.length >= 4 ? { mount: dfp[0], total_bytes: Number(dfp[1]), used_bytes: Number(dfp[2]), avail_bytes: Number(dfp[3]), used_pct: Number(dfp[1]) ? Math.round((Number(dfp[2]) / Number(dfp[1])) * 1000) / 10 : 0 } : null;
// the busiest interface carries the build traffic (rsync in, artefacts out)
const nets = net1.map(n => { const b = net0.find(x => x.iface === n.iface) || n; return { ...n, rx_bps: Math.max(0, Math.round((n.rx_bytes - b.rx_bytes) * 8 / dt)), tx_bps: Math.max(0, Math.round((n.tx_bytes - b.tx_bytes) * 8 / dt)) }; }).filter(n => n.rx_bytes + n.tx_bytes > 0).sort((a, b) => (b.rx_bytes + b.tx_bytes) - (a.rx_bytes + a.tx_bytes));
const sensors = [];
for (const h of safeList('/sys/class/hwmon')) {
const dir = join('/sys/class/hwmon', h); const name = read(join(dir, 'name')).trim();
for (const f of safeList(dir)) { const m = /^(temp\d+)_input$/.exec(f); if (!m) continue; sensors.push({ name, label: read(join(dir, `${m[1]}_label`)).trim() || null, millic: Number(read(join(dir, f))) }); }
}
const osr = read('/etc/os-release'); const pretty = (/^PRETTY_NAME="?([^"\n]+)"?/m.exec(osr) || [])[1] || null;
const clk = Number(sh('getconf', ['CLK_TCK']).trim()) || 100;
return { name: hostname(), os: pretty, cores: stat1.cores.length, uptime_s: Math.round(up), boot_at: iso(Date.now() - up * 1000), ...la, cpu, mem, disk, net: nets[0] || null, temps: pickTemps(sensors), clk, kernel: read('/proc/sys/kernel/osrelease').trim() || null };
}
const safeList = d => { try { return readdirSync(d); } catch { return []; } };
const alive = pid => { try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; } };
// flock probe without python: `flock -n <file> true` fails while the slot is held (the slot is held by another process's fd)
const flockHeld = p => { const r = spawnSync('flock', ['-n', p, 'true'], { timeout: 3000 }); return r.status !== 0; };
// ---- slots, waiters, running builds ---------------------------------------------------------------------------------------
function slots(now) {
const files = safeList(LOCKS).map(n => ({ name: n, text: read(join(LOCKS, n)), held: /^(build-\d+|run-\d+|build|measure)$/.test(n) ? flockHeld(join(LOCKS, n)) : undefined }));
return parseLockDir(files, { now, aliveFn: alive });
}
function procs(bootSec, clk, now) {
const running = [], waiters = []; let compilers = 0;
for (const d of safeList('/proc')) {
if (!/^\d+$/.test(d)) continue;
const pid = Number(d); const comm = read(`/proc/${d}/comm`).trim();
if (['rustc', 'cc1plus', 'cc1', 'ld', 'ld.lld', 'x86_64-w64-mingw32-ld'].includes(comm)) compilers++;
if (comm === 'cargo') {
let cwd = null; try { cwd = readlinkSync(`/proc/${d}/cwd`); } catch { cwd = null; }
const p = cargoProcess({ pid, cwd, cmdline: read(`/proc/${d}/cmdline`), stat: read(`/proc/${d}/stat`), bootSec, clk, now, root: BUILDS });
if (p) running.push(p);
}
if (comm === 'flock') {
const args = read(`/proc/${d}/cmdline`).split('\0').filter(Boolean);
if (args[1] === '-w' && /^\d+$/.test(args[2] || '')) {
const stat = read(`/proc/${d}/stat`); const rest = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
const ppid = Number(rest[1]); const startTicks = Number(rest[19]);
const since = Number.isFinite(startTicks) ? iso((bootSec + startTicks / clk) * 1000) : null;
const wait = read(join(LOCKS, `wait-${ppid}`)).trim(); // a waiter's own line, when the appender writes one
waiters.push({ pid: ppid, since, waiting_s: since ? Math.max(0, Math.round((now - Date.parse(since)) / 1000)) : null, label: wait || null, ...(wait ? { kind: deriveKind(parseLabelSafe(wait)) } : {}) });
}
}
}
// the innermost cargo per worktree: a `cargo build` spawns no nested cargo, but a `cargo test` runs test binaries, not cargo
running.sort((a, b) => String(a.started_at).localeCompare(String(b.started_at)));
return { running, waiters, compilers };
}
function parseLabelSafe(line) { const r = /: (.*)$/.exec(line); return { command: r ? r[1] : line }; }
function sccache() {
const bin = existsSync('/home/build/.cargo/bin/sccache') ? '/home/build/.cargo/bin/sccache' : 'sccache';
const out = sh(bin, ['--show-stats'], { env: { ...process.env, SCCACHE_DIR: process.env.SCCACHE_DIR || '/srv/sccache' } });
return out ? parseSccacheStats(out) : null;
}
// ---- sources dropped by the Mac ------------------------------------------------------------------------------------------
function source(name, now) {
const p = join(OUT_DIR, 'sources', `${name}.json`);
try {
const st = statSync(p); const age = Math.round((now - st.mtimeMs) / 1000);
const j = JSON.parse(readFileSync(p, 'utf8'));
return { data: j, meta: { at: iso(st.mtimeMs), age_s: age, stale: age > 600, ok: true } };
} catch (e) { return { data: null, meta: { ok: false, error: e.code === 'ENOENT' ? 'the Mac has not pushed yet' : String(e.message || e) } }; }
}
// ---- main -------------------------------------------------------------------------------------------------------------------
export function collect() {
const now = Date.now();
const m = machine();
const bootSec = Math.round((now - m.uptime_s * 1000) / 1000);
const s = slots(now);
const p = procs(bootSec, m.clk, now);
// attach the slot holder to its build (the label names the worktree and crate the cargo runs in)
for (const r of p.running) {
const h = s.held.find(h => h.worktree === r.worktree && (!h.crate || h.crate === r.crate || !r.crate));
r.slot = h ? h.slot : null; r.agent = h ? h.agent : null; r.queued_at = h ? h.since : null; r.waited_s = h ? h.waited_s : null;
}
const logText = read(LOG);
const recent = parseBuildsJsonl(logText, { limit: 200 });
const mac = source('mac', now), pcs = source('pcs', now);
let headline = null; try { headline = JSON.parse(readFileSync(join(OUT_DIR, 'headline.json'), 'utf8')); } catch { headline = null; }
const doc = {
v: 1,
generated_at: iso(now),
generated_by: 'collect.mjs on ' + m.name,
box: {
...m, clk: undefined,
collected_at: iso(now),
sccache: sccache(),
slots: { count: s.count, held: s.held, free: s.free },
running: p.running,
queue: p.waiters,
compilers: p.compilers,
recent: recent.rows,
log: { path: LOG, present: !!logText, bad_lines: recent.bad, total: recent.total, note: logText ? null : 'builds.jsonl is not written yet: the build-server agent adds the append to bs_remote_run; until then this lane fills from nothing' },
},
mac: mac.data, pcs: pcs.data,
headline, baselines: BASELINES,
sources: { box: { ok: true, at: iso(now) }, mac: mac.meta, pcs: pcs.meta },
};
return doc;
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
const doc = collect();
const text = JSON.stringify(doc);
if (flag('stdout', false)) { process.stdout.write(text + '\n'); }
else {
mkdirSync(OUT_DIR, { recursive: true }); mkdirSync(join(OUT_DIR, 'sources'), { recursive: true });
const tmp = join(OUT_DIR, '.workers.json.tmp'); writeFileSync(tmp, text); renameSync(tmp, join(OUT_DIR, 'workers.json'));
const fd = openSync(join(OUT_DIR, 'last-run'), 'w'); closeSync(fd);
}
}

6
tools/workers/fixtures/builds.jsonl Normal file
View file

@ -0,0 +1,6 @@
{"v":1,"id":"igneum-build-1-1759770840-36992","host":"igneum-build-1","tool":"cross-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu -j 90","target":"x86_64-pc-windows-gnu","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:33:58Z","start":"2026-10-06T17:34:00Z","end":"2026-10-06T17:51:12Z","secs":1032,"exit":0,"compiles":1954,"sccache":{"hits":0,"misses":1886,"hits_total":0,"misses_total":1886},"load_end":[13.86,9.61,4.32],"artefacts":[{"path":"target/x86_64-pc-windows-gnu/release/igneumd.exe","bytes":52911104,"sha256":"0f3a9c2e1d8b7a6f5e4d3c2b1a0f9e8d7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f"},{"path":"target/x86_64-pc-windows-gnu/release/igneum-miner.exe","bytes":9811456,"sha256":"1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b"}]}
{"v":1,"id":"igneum-build-1-1759772000-37001","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow -j 90","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","slot":0,"wait_s":0,"start":"2026-10-06T17:53:20Z","end":"2026-10-06T18:06:41Z","secs":801,"exit":0,"compiles":1102,"sccache":{"hits":790,"misses":312,"hits_total":790,"misses_total":2198},"load_end":[61.2,40.1,22.0],"artefacts":[{"path":"target/release/igneumd","bytes":61120512,"sha256":"2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c"},{"path":"target/release/igneum-miner","bytes":11010048,"sha256":"3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d"}]}
{"v":1,"id":"igneum-build-1-1759773000-37100","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ship0314","crate":"app/igneum-app","command":"cargo build --release -j 90","branch":"release-0.3.14","sha":"952ce8f","label":"igneum-wt-ship0314/app/igneum-app cargo build --release","slot":0,"wait_s":412,"queued_at":"2026-10-06T18:03:10Z","start":"2026-10-06T18:10:02Z","end":"2026-10-06T18:13:48Z","secs":226,"exit":0,"compiles":388,"sccache":{"hits":301,"misses":87,"hits_total":1091,"misses_total":2285},"load_end":[70.3,55.8,30.2],"artefacts":[{"path":"target/release/igneum-app","bytes":30801920,"sha256":"4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e"}]}
{"v":1,"id":"igneum-build-1-1759773600-37200","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ca3-v4-node","crate":"vendor/igneum-node-v4","command":"cargo test --release -p kaspa-consensus-core --lib -j 90","branch":"ca3-v4-node","sha":"f39a8eb","label":"igneum-wt-ca3-v4-node/vendor/igneum-node-v4 cargo test --release -p kaspa-consensus-core --lib","slot":0,"wait_s":0,"start":"2026-10-06T18:14:00Z","end":"2026-10-06T18:19:33Z","secs":333,"exit":101,"compiles":210,"sccache":{"hits":180,"misses":30,"hits_total":1271,"misses_total":2315},"load_end":[55.0,50.2,33.1],"artefacts":[]}
not json at all
{"v":1,"id":"igneum-build-1-1759774200-37300","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-proving-v2","crate":"proving/igneum-prove","command":"cargo build --release -j 90","branch":"proving-v2","sha":"621f258","label":"igneum-wt-proving-v2/proving/igneum-prove cargo build --release","slot":0,"start":"2026-10-06T18:21:00Z","end":"2026-10-06T18:29:10Z","secs":490,"exit":0,"compiles":640,"sccache":{"hits":600,"misses":40,"hits_total":1871,"misses_total":2355},"load_end":[60.1,52.0,38.4],"artefacts":[{"path":"target/release/igneum-prove-host","bytes":80000000,"sha256":"5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f"}]}

1
tools/workers/fixtures/locks/build-0 Normal file
View file

@ -0,0 +1 @@
pid 36992 since 17:34:00Z waited 0 s: igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server

1
tools/workers/fixtures/locks/build-1 Normal file
View file

@ -0,0 +1 @@
pid 1 since 09:00:00Z waited 3 s: igneum-wt-old/app/igneum-app cargo build --release

0
tools/workers/fixtures/locks/build-2 Normal file
View file

1
tools/workers/fixtures/locks/slots Normal file
View file

@ -0,0 +1 @@
1

8
tools/workers/fixtures/proc-stat-0.txt Normal file
View file

@ -0,0 +1,8 @@
cpu 513633 175 77715 5794769 1804 0 128 0 0 0
cpu0 10000 0 1000 50000 10 0 1 0 0 0
cpu1 20000 0 2000 40000 10 0 1 0 0 0
cpu2 100 0 10 60000 10 0 1 0 0 0
cpu3 30000 0 500 30000 10 0 1 0 0 0
intr 1 2 3
ctxt 99
btime 1759772000

8
tools/workers/fixtures/proc-stat-1.txt Normal file
View file

@ -0,0 +1,8 @@
cpu 514633 175 77915 5795569 1804 0 128 0 0 0
cpu0 10100 0 1000 50000 10 0 1 0 0 0
cpu1 20000 0 2000 40100 10 0 1 0 0 0
cpu2 150 0 60 60000 10 0 1 0 0 0
cpu3 30025 0 500 30075 10 0 1 0 0 0
intr 1 2 3
ctxt 99
btime 1759772000

27
tools/workers/fixtures/sccache-stats.txt Normal file
View file

@ -0,0 +1,27 @@
Compile requests 2294
Compile requests executed 1954
Cache hits 790
Cache misses 1886
Cache misses (Assembler) 27
Cache misses (C/C++) 777
Cache misses (Rust) 1082
Cache hits rate 29.52 %
Cache timeouts 0
Cache read errors 0
Forced recaches 0
Cache write errors 0
Compilation failures 0
Cache errors 0
Non-cacheable compilations 0
Non-cacheable calls 340
Non-compilation calls 0
Unsupported compiler calls 0
Average cache write 0.002 s
Average compiler 1.624 s
Average cache read hit 0.001 s
Failed distributed compilations 0
Cache location Local disk: "/srv/sccache"
Use direct/preprocessor mode? yes
Version (client) 0.10.0
Cache size 12 GiB
Max cache size 100 GiB

1
tools/workers/fixtures/workers.demo.json Normal file
View file

File diff suppressed because one or more lines are too long

16
tools/workers/launchd/com.igneum.workers-push.plist Normal file
View file

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<!-- The worker dashboard's Mac pusher: tools/workers/push.mjs every 60 s. Installed by tools/workers/launchd/install.sh
into ~/Library/LaunchAgents (the __ROOT__ marker becomes the checkout path). Log: ~/Library/Logs/igneum-workers-push.log -->
<plist version="1.0"><dict>
<key>Label</key><string>com.igneum.workers-push</string>
<key>ProgramArguments</key><array><string>/opt/homebrew/bin/node</string><string>__ROOT__/tools/workers/push.mjs</string></array>
<key>StartInterval</key><integer>60</integer>
<key>RunAtLoad</key><true/>
<key>WorkingDirectory</key><string>__ROOT__</string>
<key>EnvironmentVariables</key><dict><key>PATH</key><string>/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin</string><key>HOME</key><string>__HOME__</string></dict>
<key>StandardOutPath</key><string>__HOME__/Library/Logs/igneum-workers-push.log</string>
<key>StandardErrorPath</key><string>__HOME__/Library/Logs/igneum-workers-push.log</string>
<key>Nice</key><integer>10</integer>
<key>ProcessType</key><string>Background</string>
</dict></plist>

19
tools/workers/launchd/install.sh Executable file
View file

@ -0,0 +1,19 @@
#!/usr/bin/env bash
# Install or refresh the Mac pusher's launchd agent (every 60 s). Run from any checkout: the plist points at the MAIN
# checkout's tools/workers/push.mjs (/Users/joshm/Projects/igneum) unless IGNEUM_WORKERS_ROOT names another, so the agent
# keeps working after this worktree is removed.
# tools/workers/launchd/install.sh install and (re)load
# tools/workers/launchd/install.sh --remove unload and delete
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="${IGNEUM_WORKERS_ROOT:-/Users/joshm/Projects/igneum}"
LABEL=com.igneum.workers-push; DEST="$HOME/Library/LaunchAgents/$LABEL.plist"
if [ "${1:-}" = --remove ]; then launchctl bootout "gui/$(id -u)/$LABEL" 2>/dev/null || true; rm -f "$DEST"; echo "removed $LABEL"; exit 0; fi
[ -f "$ROOT/tools/workers/push.mjs" ] || { echo "no $ROOT/tools/workers/push.mjs (merge workers-dash to master first, or set IGNEUM_WORKERS_ROOT)" >&2; exit 1; }
mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs"
sed -e "s#__ROOT__#$ROOT#g" -e "s#__HOME__#$HOME#g" "$HERE/$LABEL.plist" > "$DEST"
plutil -lint "$DEST" >/dev/null
launchctl bootout "gui/$(id -u)/$LABEL" 2>/dev/null || true
launchctl bootstrap "gui/$(id -u)" "$DEST"
launchctl kickstart -k "gui/$(id -u)/$LABEL"
sleep 3; launchctl print "gui/$(id -u)/$LABEL" | grep -E 'state|last exit|runs' | head -4; tail -2 "$HOME/Library/Logs/igneum-workers-push.log" | sed 's/dl\/[A-Za-z0-9_-]\{8,\}/dl\/<token>/g'

221
tools/workers/lib.mjs Normal file
View file

@ -0,0 +1,221 @@
// Parsers shared by tools/workers/collect.mjs (on igneum-build-1) and tools/workers/push.mjs (on the Mac). Pure functions
// over text the machine gives us (/proc files, lock files, the builds JSONL), so node --test can run them against the
// fixtures in tools/workers/fixtures without a box. Every reader here takes the machine's own facts, never a reported number.
/// /proc/stat -> { total: [{user, nice, system, idle, iowait, irq, softirq, steal}], cores: [...] } (jiffies).
export function parseProcStat(text) {
const cores = []; let total = null;
for (const line of String(text).split('\n')) {
if (!line.startsWith('cpu')) continue;
const f = line.trim().split(/\s+/);
const n = f.slice(1, 9).map(Number);
const row = { user: n[0], nice: n[1], system: n[2], idle: n[3], iowait: n[4] || 0, irq: n[5] || 0, softirq: n[6] || 0, steal: n[7] || 0 };
if (f[0] === 'cpu') total = row; else cores.push(row);
}
return { total, cores };
}
/// Busy percentage between two /proc/stat samples: 0 to 100 for the whole box and per core, in core order.
export function cpuBusy(before, after) {
const pct = (a, b) => {
if (!a || !b) return 0;
const sum = r => r.user + r.nice + r.system + r.idle + r.iowait + r.irq + r.softirq + r.steal;
const idle = r => r.idle + r.iowait;
const dt = sum(b) - sum(a); const di = idle(b) - idle(a);
if (dt <= 0) return 0;
return Math.max(0, Math.min(100, Math.round((1 - di / dt) * 1000) / 10));
};
const n = Math.min(before.cores.length, after.cores.length);
const per = []; for (let i = 0; i < n; i++) per.push(pct(before.cores[i], after.cores[i]));
return { busy_pct: pct(before.total, after.total), per_core: per, cores_busy: per.filter(p => p >= 50).length };
}
/// /proc/meminfo -> kB numbers and a used percentage (used = total minus available, what a build can still take).
export function parseMeminfo(text) {
const m = {}; for (const line of String(text).split('\n')) { const r = /^(\w+):\s+(\d+)/.exec(line); if (r) m[r[1]] = Number(r[2]); }
const total = m.MemTotal || 0, avail = m.MemAvailable ?? m.MemFree ?? 0;
return { total_kb: total, available_kb: avail, used_kb: Math.max(0, total - avail), used_pct: total ? Math.round(((total - avail) / total) * 1000) / 10 : 0, swap_total_kb: m.SwapTotal || 0, swap_free_kb: m.SwapFree || 0 };
}
/// /proc/net/dev -> [{ iface, rx_bytes, tx_bytes }] without lo.
export function parseNetDev(text) {
const out = [];
for (const line of String(text).split('\n').slice(2)) {
const r = /^\s*([^:\s]+):\s*(.*)$/.exec(line); if (!r) continue;
const f = r[2].trim().split(/\s+/).map(Number);
if (r[1] === 'lo') continue;
out.push({ iface: r[1], rx_bytes: f[0] || 0, tx_bytes: f[8] || 0 });
}
return out;
}
/// /proc/loadavg -> { load: [1, 5, 15], running, threads }.
export function parseLoadavg(text) {
const f = String(text).trim().split(/\s+/);
const rq = (f[3] || '0/0').split('/');
return { load: [Number(f[0]) || 0, Number(f[1]) || 0, Number(f[2]) || 0], running: Number(rq[0]) || 0, threads: Number(rq[1]) || 0 };
}
/// The slot-file line both with-lock.sh and bs_remote_run write: "pid N since HH:MM:SSZ waited S s: <label>". Returns null
/// for an empty or foreign line. `day` (YYYY-MM-DD, UTC) anchors the clock time; a time later than `now` is yesterday.
export function parseSlotLine(line, { day, now } = {}) {
const r = /^pid (\d+) since (\d\d):(\d\d):(\d\d)Z waited (\d+) s: ?(.*)$/.exec(String(line || '').trim());
if (!r) return null;
const nowMs = now ? new Date(now).getTime() : Date.now();
const d = day || new Date(nowMs).toISOString().slice(0, 10);
let since = Date.parse(`${d}T${r[2]}:${r[3]}:${r[4]}Z`);
if (since > nowMs + 60_000) since -= 86_400_000;
const label = r[6].trim();
const agent = (/;\s*agent=([^;]+)\s*$/.exec(label) || [])[1] || null;
return { pid: Number(r[1]), since: new Date(since).toISOString().replace(/\.\d{3}Z$/, 'Z'), waited_s: Number(r[5]), label: agent ? label.replace(/;\s*agent=[^;]+\s*$/, '').trim() : label, agent, ...splitLabel(label) };
}
/// "igneum-wt-x/vendor/igneum-node-bs cross x86_64-pc-windows-gnu" or "igneum-wt-x/app/igneum-app cargo build --release ..."
/// -> { worktree, crate, command, target }. A Mac with-lock label is a plain command line: worktree and crate are null.
export function splitLabel(label) {
const s = String(label || '').replace(/;\s*agent=[^;]+\s*$/, '').trim();
const r = /^(igneum(?:-wt-[\w.-]+)?)\/(\S+) (cargo .*|cross .*)$/.exec(s) || /^(igneum(?:-wt-[\w.-]+)?)\/(\S+) (.*)$/.exec(s);
if (!r) return { worktree: null, crate: null, command: s || null, target: targetOf(s) };
let command = r[3];
if (command.startsWith('cross ')) command = `cargo build --release --target ${command.slice(6).trim()}`;
return { worktree: r[1], crate: r[2], command, target: targetOf(command) };
}
export function targetOf(command) {
const r = /--target[= ](\S+)/.exec(String(command || ''));
if (r) return r[1];
return /\bcargo\b/.test(String(command || '')) ? 'x86_64-unknown-linux-gnu' : null;
}
/// The build's kind, from the builds-log fields or a slot label: node-linux, node-windows, app, app-windows, prove, suite,
/// check, other. The same rule as the JSONL proposal, so both sides agree.
export function deriveKind({ kind, command, crate, tool, target } = {}) {
if (kind) return kind;
const cmd = String(command || ''); const cr = String(crate || '');
const sub = (/\bcargo\s+(\S+)/.exec(cmd) || [])[1] || cmd.split(/\s+/)[0] || '';
const tgt = target || targetOf(cmd);
const win = tgt === 'x86_64-pc-windows-gnu' || tool === 'cross-remote';
if (sub === 'test' || sub === 'nextest') return 'suite';
if (sub === 'check' || sub === 'clippy') return 'check';
if (/^vendor\//.test(cr) || /\bkaspad\b/.test(cmd)) return win ? 'node-windows' : 'node-linux';
if (/igneum-app/.test(cr)) return win ? 'app-windows' : 'app';
if (/igneum-prove/.test(cr) || /\bprove\b/.test(cr)) return 'prove';
if (/igneum-pow/.test(cr)) return 'pow';
return 'other';
}
/// A lock directory listing -> { count, held: [...], free: [...] }. `files` is [{ name, text, held }] where `held` says whether
/// the flock is taken (the caller knows: flock probe on the box, pid alive on the Mac). Stale lines without a live lock are free.
export function parseLockDir(files, { day, now, aliveFn } = {}) {
const held = [], free = [];
let count = 1;
for (const f of files) {
if (f.name === 'slots') { const n = parseInt(f.text, 10); if (n >= 1) count = n; continue; }
if (!/^(build-\d+|run-\d+|build|measure)$/.test(f.name)) continue;
const line = parseSlotLine((f.text || '').split('\n')[0], { day, now });
const alive = line && aliveFn ? aliveFn(line.pid) : null;
const isHeld = f.held ?? (line ? alive !== false : false);
if (line && isHeld) held.push({ slot: f.name, ...line, kind: deriveKind(line) });
else free.push(f.name);
}
held.sort((a, b) => a.slot.localeCompare(b.slot));
return { count, held, free: free.sort() };
}
/// The builds JSONL (one object per line, the format sent to the build-server agent). Bad lines are skipped and counted.
export function parseBuildsJsonl(text, { limit = 200 } = {}) {
const rows = []; let bad = 0;
for (const line of String(text).split('\n')) {
const t = line.trim(); if (!t) continue;
try { const o = JSON.parse(t); if (o && typeof o === 'object') rows.push(normaliseBuild(o)); else bad++; } catch { bad++; }
}
rows.sort((a, b) => String(b.end || b.start || '').localeCompare(String(a.end || a.start || '')));
return { rows: rows.slice(0, limit), bad, total: rows.length };
}
export function normaliseBuild(o) {
const start = o.start || null, end = o.end || null;
const secs = Number.isFinite(o.secs) ? o.secs : (start && end ? Math.round((Date.parse(end) - Date.parse(start)) / 1000) : null);
const sc = o.sccache || {};
const hits = Number(sc.hits ?? 0), misses = Number(sc.misses ?? 0);
return {
...o,
id: o.id || `${o.host || 'box'}-${start || ''}-${o.pid || ''}`,
kind: deriveKind(o),
target: o.target || targetOf(o.command),
secs,
ok: Number(o.exit) === 0,
sccache: { hits, misses, hit_rate_pct: hits + misses ? Math.round((hits / (hits + misses)) * 1000) / 10 : null, hits_total: sc.hits_total ?? null, misses_total: sc.misses_total ?? null },
artefacts: Array.isArray(o.artefacts) ? o.artefacts : [],
};
}
/// `sccache --show-stats` text -> { requests, executed, hits, misses, hit_rate_pct, cache_size, max_size }.
export function parseSccacheStats(text) {
const num = re => { const r = re.exec(String(text)); return r ? Number(r[1].replace(/,/g, '')) : null; };
const str = re => { const r = re.exec(String(text)); return r ? r[1].trim() : null; };
const hits = num(/^Cache hits\s{2,}(\d[\d,]*)/m), misses = num(/^Cache misses\s{2,}(\d[\d,]*)/m);
return {
requests: num(/^Compile requests\s+(\d[\d,]*)/m), executed: num(/^Compile requests executed\s+(\d[\d,]*)/m),
hits, misses, hit_rate_pct: hits !== null && misses !== null && hits + misses ? Math.round((hits / (hits + misses)) * 1000) / 10 : (hits === 0 && misses ? 0 : null),
cache_size: str(/^Cache size\s+(.+)$/m), max_size: str(/^Max cache size\s+(.+)$/m),
};
}
/// /sys/class/hwmon readings -> [{ name, label, c }] for the sensors worth a tile: the CPU die, the NVMe drives, the NIC.
export function pickTemps(sensors) {
const out = [];
for (const s of sensors) {
const c = Number(s.millic) / 1000;
if (!Number.isFinite(c) || c <= 0 || c > 150) continue;
const name = s.name === 'k10temp' || s.name === 'coretemp' ? 'cpu' : s.name === 'nvme' ? 'nvme' : s.name.startsWith('bnxt') || s.name.includes('eth') ? 'nic' : s.name;
out.push({ name, label: s.label || s.name, c: Math.round(c * 10) / 10 });
}
// one per name: the hottest (two NVMe drives in RAID 1, two NIC ports)
const best = new Map();
for (const t of out) { const cur = best.get(t.name); if (!cur || t.c > cur.c) best.set(t.name, t); }
const order = ['cpu', 'nvme', 'nic'];
return [...best.values()].sort((a, b) => (order.indexOf(a.name) + 99) % 99 - (order.indexOf(b.name) + 99) % 99 || a.name.localeCompare(b.name));
}
/// A cargo process -> a running build: { pid, cwd, worktree, crate, command, target, kind, started_at, elapsed_s }.
/// `stat` is /proc/<pid>/stat text, `bootSec` the boot time in unix seconds, `clk` CLK_TCK.
export function cargoProcess({ pid, cwd, cmdline, stat, bootSec, clk = 100, now = Date.now(), root = '/srv/builds' }) {
const args = String(cmdline || '').split('\0').filter(Boolean);
if (!args.length || !/(^|\/)cargo$/.test(args[0])) return null;
const command = ['cargo', ...args.slice(1)].join(' ');
const rest = stat ? stat.slice(stat.lastIndexOf(')') + 2).split(' ') : [];
const startTicks = Number(rest[19]); // field 22 of /proc/<pid>/stat, after pid and comm
const started = Number.isFinite(startTicks) && bootSec ? (bootSec + startTicks / clk) * 1000 : null;
const rel = String(cwd || '').startsWith(root + '/') ? cwd.slice(root.length + 1) : null;
const worktree = rel ? rel.split('/')[0] : null;
const crate = rel ? rel.split('/').slice(1).join('/') || '.' : null;
const target = targetOf(command);
return { pid, cwd, worktree, crate, command, target, kind: deriveKind({ command, crate, target }), started_at: started ? new Date(started).toISOString().replace(/\.\d{3}Z$/, 'Z') : null, elapsed_s: started ? Math.max(0, Math.round((now - started) / 1000)) : null };
}
/// The relay job SUMMARY line and closing line of one report (as tools/jobs.mjs reads them) -> { job, kind, status, exit,
/// duration_s, started_at, finished_at, summary, errors[] } or null.
export function summaryOf(lines) {
const ls = String(lines || '').split('\n');
let s = null;
for (const l of ls) { if (l.startsWith('SUMMARY ')) { try { s = JSON.parse(l.slice(8)); } catch { s = null; } break; } }
const close = ls.slice(-40).map(l => /^job (\S+): (done|failed|timeout|aborted) \(exit (-?\d+)\) after (\d+) s: (.*)$/.exec(l)).filter(Boolean).pop();
if (close) { s = s || {}; s.job = s.job || close[1]; s.status = close[2]; s.exit = Number(close[3]); s.duration_s = Number(close[4]); s.summary = close[5]; s.finished_at = s.finished_at || 'closed'; }
else if (s && !s.finished_at) s.status = 'running';
if (s) {
s.errors = ls.filter(l => /^(Error:|BUILD FAILED|GATE FAILED)|\bFAILED\b|panicked at/.test(l) && !l.startsWith('SUMMARY ')).slice(0, 5);
s.stage = (ls.map(l => /^STAGE (\S+) start/.exec(l)).filter(Boolean).pop() || [])[1] || null;
}
return s;
}
/// Duration bars want the same scale: the longest of the lot, at least a minute.
export const fmtSecs = s => { s = Math.max(0, Math.round(Number(s) || 0)); const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), r = s % 60; return h ? `${h} h ${String(m).padStart(2, '0')} min` : m ? `${m} min ${String(r).padStart(2, '0')} s` : `${r} s`; };
/// Mac baselines for the kind comparison: approximate, from the docs that measured them; a measurement on the same tree
/// replaces them when one is logged. Shared by collect.mjs (the box's file) and push.mjs (the edge copy).
export const BASELINES = {
'node-linux': { secs: 690, source: 'approximate: infra/cross/build-linux.sh on this Mac, 8 to 15 min per target at nice 19 with 4 to 6 jobs (docs/plans/build-job.md, 4 Oct 2026)' },
'node-windows': { secs: 690, source: 'approximate: proto-cuda/windows-node/cross-build.sh on this Mac, the same 8 to 15 min (docs/plans/build-job.md, 4 Oct 2026)' },
};

366
tools/workers/page/workers.html Normal file
View file

@ -0,0 +1,366 @@
<!doctype html>
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover"><meta name="robots" content="noindex">
<title>Igneum Workers</title>
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Unbounded:wght@500;700&family=IBM+Plex+Mono:wght@400;500;600&family=IBM+Plex+Sans:wght@400;500;600&display=swap">
<style>
:root {
--bg: #0c0c0e; --panel: #141418; --panel-2: #1b1b21; --ink: #f1ede4; --muted: #8e8b86; --line: #26262d;
--ember: #f2541b; --ember-2: #ffb15a; --good: #5fd08a; --warn: #f2c14e; --bad: #ff6b5e; --cool: #6aa9ff;
--display: "Unbounded", "IBM Plex Sans", -apple-system, sans-serif;
--mono: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
--sans: "IBM Plex Sans", -apple-system, "Segoe UI", Helvetica, Arial, sans-serif;
color-scheme: dark;
}
* { box-sizing: border-box; }
body { background: var(--bg); color: var(--ink); font-family: var(--sans); font-size: 14px; line-height: 1.45; margin: 0; }
.wrap { max-width: 1180px; margin: 0 auto; padding-inline: 16px; padding-block: 18px 48px; }
header { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 10px 20px; }
.brand { display: flex; align-items: center; gap: 12px; }
.brand svg { width: 34px; height: 34px; }
.brand h1 { font-family: var(--display); font-weight: 700; font-size: 20px; letter-spacing: 0.02em; margin: 0; }
.brand h1 small { font-family: var(--mono); font-weight: 400; font-size: 11px; color: var(--muted); letter-spacing: 0.1em; text-transform: uppercase; margin-left: 10px; }
nav { display: flex; gap: 2px; border: 1px solid var(--line); }
nav a { font-family: var(--mono); font-size: 11px; letter-spacing: 0.1em; text-transform: uppercase; color: var(--muted); text-decoration: none; padding: 7px 12px; }
nav a.on { color: var(--ink); background: var(--panel-2); }
nav a:hover { color: var(--ember-2); }
.live { font-family: var(--mono); font-size: 12px; color: var(--muted); display: flex; align-items: center; gap: 8px; }
.live b { width: 8px; height: 8px; border-radius: 50%; background: var(--good); box-shadow: 0 0 0 0 rgba(95,208,138,.6); animation: ping 2.4s infinite; }
.live.stale b { background: var(--warn); animation: none; }
.live.down b { background: var(--bad); animation: none; }
@keyframes ping { 0% { box-shadow: 0 0 0 0 rgba(95,208,138,.55); } 70% { box-shadow: 0 0 0 9px rgba(95,208,138,0); } 100% { box-shadow: 0 0 0 0 rgba(95,208,138,0); } }
h2 { font-family: var(--display); font-weight: 500; font-size: 13px; letter-spacing: 0.08em; text-transform: uppercase; color: var(--muted); margin: 32px 0 12px; display: flex; align-items: baseline; gap: 12px; flex-wrap: wrap; }
h2 span { font-family: var(--mono); font-size: 11px; letter-spacing: 0; text-transform: none; color: var(--muted); }
.empty { padding: 18px; color: var(--muted); border: 1px dashed var(--line); font-family: var(--mono); font-size: 12px; line-height: 1.6; }
.pill { display: inline-block; font-family: var(--mono); font-size: 10px; letter-spacing: 0.06em; text-transform: uppercase; padding: 3px 7px; border: 1px solid var(--line); color: var(--muted); white-space: nowrap; }
.pill.running, .pill.done, .pill.ok { border-color: var(--good); color: var(--good); }
.pill.failed, .pill.timeout, .pill.aborted, .pill.error { border-color: var(--bad); color: var(--bad); }
.pill.waiting, .pill.queued, .pill.stale { border-color: var(--warn); color: var(--warn); }
.pill.kind { border-color: #3a2a22; color: var(--ember-2); }
time[title] { cursor: help; border-bottom: 1px dotted var(--line); }
/* the server */
.server { margin-top: 18px; background: var(--panel); border: 1px solid var(--line); position: relative; overflow: hidden; }
.server.hot::after { content: ""; position: absolute; inset: auto -30% -70% -30%; height: 90%; background: radial-gradient(ellipse at center, rgba(242,84,27,.18), transparent 60%); pointer-events: none; animation: breathe 3.2s ease-in-out infinite; }
@keyframes breathe { 0%, 100% { opacity: .5; } 50% { opacity: 1; } }
.server .head { display: flex; flex-wrap: wrap; justify-content: space-between; align-items: flex-end; gap: 10px 20px; padding: 16px 18px 0; }
.server .name { font-family: var(--display); font-weight: 700; font-size: 22px; letter-spacing: 0.01em; }
.server .name small { font-family: var(--mono); font-weight: 400; font-size: 11px; color: var(--muted); margin-left: 10px; letter-spacing: 0.08em; text-transform: uppercase; }
.server .facts { font-family: var(--mono); font-size: 11px; color: var(--muted); display: flex; flex-wrap: wrap; gap: 4px 16px; }
.server .facts b { color: var(--ink); font-weight: 500; }
.cores { display: grid; grid-template-columns: repeat(48, 1fr); gap: 2px; padding: 14px 18px 0; height: 78px; align-items: end; position: relative; z-index: 1; }
.cores i { display: block; height: 100%; background: var(--panel-2); position: relative; overflow: hidden; }
.cores i::after { content: ""; position: absolute; left: 0; right: 0; bottom: 0; height: var(--h, 2%); min-height: 2px; background: linear-gradient(180deg, var(--ember-2), var(--ember)); transition: height .9s cubic-bezier(.2,.8,.2,1); }
.cores i.hot::after { animation: flicker 1.3s ease-in-out infinite alternate; }
@keyframes flicker { from { filter: brightness(.85); } to { filter: brightness(1.15); } }
.corelabel { display: flex; justify-content: space-between; flex-wrap: wrap; gap: 2px 12px; font-family: var(--mono); font-size: 10px; color: var(--muted); padding: 6px 18px 0; letter-spacing: 0.08em; text-transform: uppercase; }
.gauges { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 1px; background: var(--line); border-top: 1px solid var(--line); margin-top: 14px; position: relative; z-index: 1; }
@media (min-width: 900px) { .gauges { grid-template-columns: repeat(5, 1fr); } }
.g { background: var(--panel); padding: 12px 14px; display: flex; gap: 12px; align-items: center; }
.g svg { width: 56px; height: 56px; flex: none; }
.g .ring { fill: none; stroke: var(--panel-2); stroke-width: 6; }
.g .arc { fill: none; stroke: var(--ember); stroke-width: 6; stroke-linecap: round; transform: rotate(-90deg); transform-origin: 50% 50%; transition: stroke-dasharray .9s ease; }
.g .arc.good { stroke: var(--good); } .g .arc.warn { stroke: var(--warn); } .g .arc.bad { stroke: var(--bad); } .g .arc.cool { stroke: var(--cool); }
.g .l { font-size: 10px; letter-spacing: 0.12em; text-transform: uppercase; color: var(--muted); }
.g .v { font-family: var(--display); font-weight: 500; font-size: 18px; margin-top: 2px; font-variant-numeric: tabular-nums; line-height: 1.1; }
.g .s { font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 3px; }
/* the other workers */
.crew { display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); gap: 12px; }
.w { background: var(--panel); border: 1px solid var(--line); padding: 12px 14px; }
.w .top { display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; }
.w .card { font-family: var(--display); font-weight: 500; font-size: 13px; }
.w .meta { font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 3px; }
.w .doing { font-size: 12px; margin-top: 8px; min-height: 2.6em; }
.w .foot { display: flex; justify-content: space-between; font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 8px; border-top: 1px solid var(--line); padding-top: 8px; }
.slots { display: flex; gap: 4px; margin-top: 8px; }
.slots i { height: 6px; flex: 1; background: var(--panel-2); }
.slots i.held { background: linear-gradient(90deg, var(--ember), var(--ember-2)); }
.slots i.run { background: var(--cool); }
.slots i.measure { background: var(--warn); }
/* lanes */
.lane { display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); gap: 12px; }
.job { background: var(--panel); border: 1px solid var(--line); padding: 12px 14px; position: relative; overflow: hidden; }
.job.running { border-color: #3a2a22; box-shadow: inset 0 0 0 1px rgba(242,84,27,.25); }
.job.running::before { content: ""; position: absolute; left: 0; top: 0; bottom: 0; width: 3px; background: linear-gradient(180deg, var(--ember-2), var(--ember)); }
.job .top { display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; }
.job .title { font-family: var(--display); font-weight: 500; font-size: 13px; line-height: 1.3; word-break: break-word; }
.job .where { font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 3px; }
.job .cmd { font-family: var(--mono); font-size: 11px; color: var(--ink); margin-top: 8px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
.job .foot { display: flex; justify-content: space-between; align-items: baseline; font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 10px; border-top: 1px solid var(--line); padding-top: 8px; gap: 8px; }
.job .elapsed { font-family: var(--display); font-weight: 500; font-size: 18px; color: var(--ember-2); font-variant-numeric: tabular-nums; }
.job .pulse { position: absolute; right: -40px; bottom: -40px; width: 120px; height: 120px; border-radius: 50%; background: radial-gradient(circle, rgba(242,84,27,.22), transparent 65%); animation: breathe 2.4s ease-in-out infinite; pointer-events: none; }
/* timeline */
.tl { background: var(--panel); border: 1px solid var(--line); }
.tl .row { display: grid; grid-template-columns: 64px 1fr 150px 92px; gap: 10px; align-items: center; padding: 9px 12px; border-bottom: 1px solid var(--line); cursor: pointer; }
.tl .row:hover { background: var(--panel-2); }
.tl .row:last-child, .tl .row.open { border-bottom-color: transparent; }
.tl .at { font-family: var(--mono); font-size: 11px; color: var(--muted); }
.tl .what { min-width: 0; }
.tl .what .t { font-size: 13px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
.tl .what .m { font-family: var(--mono); font-size: 11px; color: var(--muted); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
.tl .bar { height: 8px; background: var(--panel-2); position: relative; }
.tl .bar i { position: absolute; left: 0; top: 0; bottom: 0; background: linear-gradient(90deg, var(--good), #8ee3ad); }
.tl .bar i.failed { background: linear-gradient(90deg, var(--bad), #ff9b92); }
.tl .dur { font-family: var(--mono); font-size: 12px; text-align: right; font-variant-numeric: tabular-nums; }
.tl .lines { display: none; font-family: var(--mono); font-size: 11px; color: var(--muted); padding: 2px 12px 12px 86px; border-bottom: 1px solid var(--line); white-space: pre-wrap; word-break: break-all; line-height: 1.6; }
.tl .lines.show { display: block; }
.tl .lines b { color: var(--ink); font-weight: 500; }
@media (max-width: 640px) { .tl .row { grid-template-columns: 52px 1fr 70px; } .tl .bar { display: none; } .tl .lines { padding-left: 12px; } }
/* analytics */
.an { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 12px; }
.an .a { background: var(--panel); border: 1px solid var(--line); padding: 12px 14px; min-width: 0; }
.an .l { font-size: 10px; letter-spacing: 0.12em; text-transform: uppercase; color: var(--muted); display: flex; justify-content: space-between; gap: 8px; }
.an .l span { letter-spacing: 0; text-transform: none; font-family: var(--mono); }
.an svg { width: 100%; height: 96px; display: block; margin-top: 10px; }
.an table { border-collapse: collapse; width: 100%; font-size: 12px; margin-top: 8px; }
.an th, .an td { text-align: left; padding: 6px 6px; border-bottom: 1px solid var(--line); white-space: nowrap; }
.an th { font-size: 10px; text-transform: uppercase; letter-spacing: 0.1em; color: var(--muted); font-weight: 600; }
.an td.num, .an th.num { text-align: right; font-family: var(--mono); font-variant-numeric: tabular-nums; }
.an tr:last-child td { border-bottom: 0; }
.an .note { font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 8px; line-height: 1.5; }
.head3 { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 1px; background: var(--line); border: 1px solid var(--line); }
.h3 { background: var(--panel); padding: 14px 16px; }
.h3 .l { font-size: 10px; letter-spacing: 0.12em; text-transform: uppercase; color: var(--muted); }
.h3 .v { font-family: var(--display); font-weight: 500; font-size: 26px; margin-top: 6px; font-variant-numeric: tabular-nums; color: var(--ember-2); }
.h3 .v small { font-size: 12px; color: var(--muted); font-family: var(--mono); font-weight: 400; margin-left: 8px; }
.h3 .cmp { display: flex; gap: 14px; font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 6px; flex-wrap: wrap; }
.h3 .cmp b { color: var(--ink); font-weight: 500; }
.h3 .s { font-size: 11px; color: var(--muted); margin-top: 8px; line-height: 1.5; }
.h3 .meter { height: 4px; background: var(--panel-2); margin-top: 10px; position: relative; }
.h3 .meter i { display: block; height: 100%; background: linear-gradient(90deg, var(--ember), var(--ember-2)); transition: width .6s ease; }
.h3 .meter em { position: absolute; top: -3px; width: 2px; height: 10px; background: var(--muted); }
.sources { font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 32px; border-top: 1px solid var(--line); padding-top: 10px; display: flex; flex-wrap: wrap; gap: 6px 18px; }
.sources b { color: var(--good); font-weight: 500; } .sources b.no { color: var(--bad); } .sources b.warn { color: var(--warn); }
a:focus-visible, button:focus-visible { outline: 2px solid var(--ember-2); outline-offset: 2px; }
@media (prefers-reduced-motion: reduce) { .live b, .server.hot::after, .job .pulse, .cores i.hot::after { animation: none !important; } .cores i::after, .g .arc { transition: none; } }
</style></head><body>
<div class="wrap">
<header>
<div class="brand">
<svg viewBox="0 0 100 100" aria-hidden="true"><polygon points="50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34" fill="#F2541B"/><polygon points="50,42 59,58 50,82 41,58" fill="#0C0C0E"/></svg>
<h1>Igneum Workers<small>build shop</small></h1>
</div>
<nav aria-label="Pages"><a href="./">Fleet</a><a href="workers.html" class="on" aria-current="page">Workers</a></nav>
<div class="live" id="live"><b></b><span id="stamp">waiting for the first write</span></div>
</header>
<section class="server" id="server" aria-label="The build server">
<div class="head">
<div><div class="name" id="s-name">igneum-build-1<small>build server</small></div><div class="facts" id="s-facts"></div></div>
<div class="facts" id="s-right"></div>
</div>
<div class="cores" id="cores" aria-label="96 cores, busy share per core"></div>
<div class="corelabel"><span id="cores-l">cores</span><span id="cores-r"></span></div>
<div class="gauges" id="gauges"></div>
</section>
<h2>Workers <span>every machine that builds, tests or measures</span></h2>
<div class="crew" id="crew"></div>
<h2>Now building <span id="now-sub"></span></h2>
<div class="lane" id="now"></div>
<h2>Queue <span id="queue-sub"></span></h2>
<div class="lane" id="queue"></div>
<h2>Recently done <span>click a row for the closing lines</span></h2>
<div class="tl" id="done"></div>
<h2>Headline timings <span>what the box does against the Mac and PC 1 on the same tree</span></h2>
<div class="head3" id="headline"></div>
<h2>Analytics <span id="an-sub"></span></h2>
<div class="an" id="an"></div>
<div class="sources" id="sources"></div>
</div>
<script type="module">
import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, kindLabel, fmtDur, fmtDurShort, fmtBytes, fmtBps, fmtUptime, ukTime, ago, headline } from './shape.js?v=1';
const $ = id => document.getElementById(id);
const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));
const pill = (s, cls) => `<span class="pill ${esc(cls || String(s || '').toLowerCase())}">${esc(s || '?')}</span>`;
const t = (isoStr, withDate) => { const u = ukTime(isoStr, { withDate }); return u.text ? `<time title="${esc(u.utc)}" datetime="${esc(isoStr)}">${esc(u.text)}</time>` : ''; };
const base = location.pathname.replace(/\/workers\.html$/, '').replace(/\/$/, '');
const demo = new URLSearchParams(location.search).get('demo');
let D = null, open = new Set();
function arc(pct, cls) {
const r = 24, c = 2 * Math.PI * r, p = Math.max(0, Math.min(100, Number(pct) || 0));
return `<svg viewBox="0 0 60 60" aria-hidden="true"><circle class="ring" cx="30" cy="30" r="${r}"/><circle class="arc ${cls || ''}" cx="30" cy="30" r="${r}" stroke-dasharray="${(c * p / 100).toFixed(1)} ${c.toFixed(1)}"/></svg>`;
}
const tone = (pct, warn = 70, bad = 90) => pct >= bad ? 'bad' : pct >= warn ? 'warn' : '';
function renderServer(b, src) {
const el = $('server');
if (!b) {
$('s-facts').innerHTML = `<span>${esc(src && src.error || 'no box facts yet')}</span>`; $('s-right').innerHTML = '';
$('cores').innerHTML = Array.from({ length: 96 }, () => '<i style="--h:2%"></i>').join(''); $('cores-r').textContent = 'no sample'; $('gauges').innerHTML = ''; el.classList.remove('hot'); return;
}
$('s-name').innerHTML = `${esc(b.name)}<small>${esc(b.os || 'build server')}</small>`;
$('s-facts').innerHTML = [`<b>${b.cores}</b> threads`, `<b>${esc(fmtUptime(b.uptime_s))}</b> up`, `load <b>${(b.load || []).map(x => Number(x).toFixed(1)).join(' / ')}</b>`, b.kernel ? `kernel <b>${esc(b.kernel)}</b>` : ''].filter(Boolean).map(x => `<span>${x}</span>`).join('');
const temps = (b.temps || []).map(x => `${esc(x.name)} <b>${x.c.toFixed(0)}°</b>`);
$('s-right').innerHTML = [...temps, b.net ? `net ↓<b>${esc(fmtBps(b.net.rx_bps))}</b> ↑<b>${esc(fmtBps(b.net.tx_bps))}</b>` : '', `<span title="${esc(b.collected_at)}">read ${esc(ago(b.collected_at))}</span>`].filter(Boolean).map(x => `<span>${x}</span>`).join('');
const per = (b.cpu && b.cpu.per_core) || [];
const n = Math.max(per.length, b.cores || 96);
$('cores').style.gridTemplateColumns = `repeat(${n > 64 ? 48 : n > 32 ? 32 : n}, 1fr)`;
$('cores').innerHTML = Array.from({ length: n }, (_, i) => { const p = per[i] ?? 0; return `<i class="${p >= 50 ? 'hot' : ''}" style="--h:${Math.max(2, p)}%" title="core ${i}: ${p}%"></i>`; }).join('');
$('cores-l').textContent = `${n} cores, ${b.cpu ? b.cpu.cores_busy : 0} busy`;
$('cores-r').textContent = b.cpu ? `${b.cpu.busy_pct}% of the box over the last second, ${b.compilers || 0} compilers running` : '';
el.classList.toggle('hot', !!(b.running && b.running.length));
const m = b.mem || {}, d = b.disk || {}, sc = b.sccache;
const hit = sc && sc.hit_rate_pct !== null && sc.hit_rate_pct !== undefined ? sc.hit_rate_pct : null;
$('gauges').innerHTML = [
`<div class="g">${arc(b.cpu ? b.cpu.busy_pct : 0, tone(b.cpu ? b.cpu.busy_pct : 0, 101, 101))}<div><div class="l">CPU</div><div class="v">${b.cpu ? b.cpu.busy_pct : 0}%</div><div class="s">load 1 min ${b.load ? Number(b.load[0]).toFixed(1) : '?'} of ${b.cores}</div></div></div>`,
`<div class="g">${arc(m.used_pct, tone(m.used_pct))}<div><div class="l">Memory</div><div class="v">${m.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes((m.used_kb || 0) * 1024))} of ${esc(fmtBytes((m.total_kb || 0) * 1024))}</div></div></div>`,
`<div class="g">${arc(d.used_pct, tone(d.used_pct, 80, 92))}<div><div class="l">Disk ${esc(d.mount || '/srv')}</div><div class="v">${d.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes(d.used_bytes))} used, ${esc(fmtBytes(d.avail_bytes))} free</div></div></div>`,
`<div class="g">${arc(hit ?? 0, 'cool')}<div><div class="l">sccache hits</div><div class="v">${hit === null ? '–' : hit + '%'}</div><div class="s">${sc ? `${sc.hits ?? 0} hit, ${sc.misses ?? 0} miss, ${esc(sc.cache_size || '?')} of ${esc(sc.max_size || '?')}` : 'sccache not answering'}</div></div></div>`,
`<div class="g">${arc(b.slots ? (b.slots.held.length / Math.max(1, b.slots.count)) * 100 : 0, 'good')}<div><div class="l">Build slots</div><div class="v">${b.slots ? `${b.slots.held.length}/${b.slots.count}` : '–'}</div><div class="s">${b.queue && b.queue.length ? `${b.queue.length} waiting` : 'nobody waiting'}</div></div></div>`,
].join('');
}
function slotBar(slots, count) {
const held = new Map((slots && slots.held || []).map(h => [h.slot, h]));
const names = Array.from({ length: count }, (_, i) => `build-${i}`);
return `<div class="slots">${names.map(nm => `<i class="${held.has(nm) ? 'held' : ''}" title="${esc(nm)}${held.has(nm) ? ': ' + esc(held.get(nm).label) : ': free'}"></i>`).join('')}</div>`;
}
function renderCrew() {
const b = D.box, mac = D.mac, pcs = (D.pcs && D.pcs.machines) || [];
const cards = [];
if (b) cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(b.name)}</div><div class="meta">Hetzner, ${b.cores} threads, ${esc(fmtBytes((b.mem && b.mem.total_kb || 0) * 1024))}, RAID 1 NVMe</div></div>${pill(b.running && b.running.length ? 'building' : 'idle', b.running && b.running.length ? 'running' : '')}</div><div class="doing">${b.running && b.running.length ? b.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('<br>') : 'Slot free. The next build-remote.sh or cross-remote.sh from any worktree takes it.'}</div>${slotBar(b.slots, b.slots ? b.slots.count : 1)}<div class="foot"><span>${b.recent ? b.recent.length : 0} builds logged</span><span>${esc(ago(b.collected_at))}</span></div></div>`);
else cards.push(`<div class="w"><div class="top"><div><div class="card">igneum-build-1</div><div class="meta">build server</div></div>${pill('unreachable', 'error')}</div><div class="doing">${esc(D.sources && D.sources.box && D.sources.box.error || 'no facts from the box')}</div></div>`);
if (mac) {
const held = mac.slots.held || [], runs = held.filter(h => /^run-/.test(h.slot)), builds = held.filter(h => /^build-/.test(h.slot)), meas = held.filter(h => h.slot === 'measure');
const doing = held.length ? held.map(h => `${esc(h.slot)}: ${esc((h.worktree ? h.worktree + ' ' : '') + (h.command || h.label || ''))}`.slice(0, 140)).join('<br>') : 'Nothing under with-lock.sh. macOS binaries and the DMG build here; everything else goes to the box or the PCs.';
cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(mac.name)}</div><div class="meta">this Mac, ${mac.build_slots} build slot${mac.build_slots === 1 ? '' : 's'} open, 3 run slots</div></div>${pill(meas.length ? 'measuring' : builds.length ? 'building' : runs.length ? 'running' : 'idle', held.length ? 'running' : '')}</div><div class="doing">${doing}</div><div class="slots">${['build-0', 'build-1', 'build-2', 'run-0', 'run-1', 'run-2', 'measure'].map(nm => { const h = held.find(x => x.slot === nm); return `<i class="${h ? (nm === 'measure' ? 'measure' : nm.startsWith('run') ? 'run' : 'held') : ''}" title="${esc(nm)}${h ? ': ' + esc(h.label) : ': free'}"></i>`; }).join('')}</div><div class="foot"><span>${mac.queue.length ? mac.queue.length + ' waiting for a slot' : 'nobody waiting'}</span><span>${esc(ago(mac.collected_at))}</span></div></div>`);
} else cards.push(`<div class="w"><div class="top"><div><div class="card">MacBook-Pro</div><div class="meta">this Mac</div></div>${pill('no push', 'error')}</div><div class="doing">${esc(D.sources && D.sources.mac && D.sources.mac.error || 'the Mac pusher has not written yet (launchd com.igneum.workers-push, every 60 s)')}</div></div>`);
for (const pc of pcs) {
const r = pc.running;
cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(pc.name)}</div><div class="meta">${esc(pc.machine || pc.id)} · ${esc(pc.role)}</div></div>${pill(r ? 'running' : pc.queue.length ? 'queued' : 'idle', r ? 'running' : pc.queue.length ? 'queued' : '')}</div><div class="doing">${r ? `${esc(r.kind)} ${esc(r.job)}${r.title ? '<br>' + esc(r.title) : ''}${r.stage ? `<br><span class="pill">stage ${esc(r.stage)}</span>` : ''}` : pc.queue.length ? `Next: ${esc(pc.queue[0].kind)} ${esc(pc.queue[0].job)}${pc.queue[0].title ? ', ' + esc(pc.queue[0].title) : ''}` : esc(pc.note || 'idle on jobs; the relay polls every 10 min')}</div><div class="foot"><span>${pc.recent.length} reports kept</span><span>${pc.last_report_at ? 'last report ' + esc(ago(pc.last_report_at)) : 'no report in 7 days'}</span></div></div>`);
}
if (!pcs.length) cards.push(`<div class="w"><div class="top"><div><div class="card">PC 1 and PC 2</div><div class="meta">relay jobs</div></div>${pill('no data', 'error')}</div><div class="doing">${esc(D.sources && D.sources.pcs && D.sources.pcs.error || 'the Mac pusher reads the relay intake; nothing has arrived')}</div></div>`);
$('crew').innerHTML = cards.join('');
}
function nowItems() {
const out = [];
for (const r of (D.box && D.box.running) || []) out.push({ where: D.box.name, slot: r.slot, title: `${kindLabel(r.kind)}`, sub: `${r.worktree || '?'}/${r.crate || ''}${r.agent ? ' · ' + r.agent : ''}`, cmd: r.command, started_at: r.started_at, kind: r.kind, extra: r.target || '' });
for (const h of (D.mac && D.mac.slots.held) || []) out.push({ where: D.mac.name, slot: h.slot, title: h.slot === 'measure' ? 'measurement' : h.slot.startsWith('run') ? 'functional run' : kindLabel(h.kind), sub: h.worktree || 'this Mac', cmd: h.command || h.label, started_at: h.since, kind: h.kind, extra: h.waited_s ? `waited ${fmtDur(h.waited_s)}` : '' });
for (const pc of (D.pcs && D.pcs.machines) || []) if (pc.running) out.push({ where: pc.name, slot: 'relay job', title: `${pc.running.kind} ${pc.running.job}`, sub: pc.running.title || pc.machine || '', cmd: pc.running.stage ? `stage ${pc.running.stage}` : 'reports every 5 min while it runs', started_at: pc.running.started_at, kind: pc.running.kind, extra: pc.running.last_report_at ? `last report ${ago(pc.running.last_report_at)}` : '' });
return out;
}
function renderNow() {
const items = nowItems();
$('now-sub').textContent = items.length ? `${items.length} in flight` : '';
if (!items.length) { $('now').innerHTML = `<div class="empty">Nothing is building right now. The box's slot is free, the Mac holds no lock and neither PC reports a running job. A build appears here within 30 s of its slot being taken (the box) or 60 s (the Mac and the PCs).</div>`; return; }
$('now').innerHTML = items.map(i => `<div class="job running" data-start="${esc(i.started_at || '')}"><div class="pulse"></div><div class="top"><div><div class="title">${esc(i.title)}</div><div class="where">${esc(i.where)} · ${esc(i.slot || '')}${i.sub ? ' · ' + esc(i.sub) : ''}</div></div>${pill(i.kind ? kindLabel(i.kind) : 'job', 'kind')}</div><div class="cmd" title="${esc(i.cmd)}">${esc(i.cmd || '')}</div><div class="foot"><span>since ${t(i.started_at)}${i.extra ? ' · ' + esc(i.extra) : ''}</span><span class="elapsed">${i.started_at ? fmtDurShort((Date.now() - Date.parse(i.started_at)) / 1000) : '–'}</span></div></div>`).join('');
}
function tick() { for (const el of document.querySelectorAll('.job.running[data-start]')) { const s = el.dataset.start; if (!s) continue; const e = el.querySelector('.elapsed'); if (e) e.textContent = fmtDurShort((Date.now() - Date.parse(s)) / 1000); } }
function renderQueue() {
const items = [];
for (const w of (D.box && D.box.queue) || []) items.push({ where: D.box.name, title: w.label ? kindLabel(w.kind) : 'a build', sub: w.label || 'label unknown: the waiter writes none until its slot is taken', since: w.since, wait: w.waiting_s });
for (const w of (D.mac && D.mac.queue) || []) items.push({ where: D.mac.name, title: `${w.mode} slot`, sub: w.command, since: null, wait: w.waiting_s });
for (const pc of (D.pcs && D.pcs.machines) || []) for (const q of pc.queue) items.push({ where: pc.name, title: `${q.kind} ${q.job}`, sub: q.title || '', since: null, wait: null, until: q.expires_at });
$('queue-sub').textContent = items.length ? `${items.length} waiting` : '';
if (!items.length) { $('queue').innerHTML = `<div class="empty">The queue is empty. On the box a second build waits on the flock of build-0 and shows here; on the Mac a with-lock.sh wait; on a PC a signed job it has not reported on yet.</div>`; return; }
$('queue').innerHTML = items.map(i => `<div class="job"><div class="top"><div><div class="title">${esc(i.title)}</div><div class="where">${esc(i.where)}</div></div>${pill('waiting')}</div><div class="cmd" title="${esc(i.sub)}">${esc(i.sub)}</div><div class="foot"><span>${i.since ? 'since ' + t(i.since) : i.until ? 'signed, expires ' + t(i.until, true) : ''}</span><span>${i.wait !== null && i.wait !== undefined ? 'waiting ' + esc(fmtDur(i.wait)) : ''}</span></div></div>`).join('');
}
function doneRows() {
const rows = [];
for (const b of (D.box && D.box.recent) || []) rows.push({ key: b.id, where: D.box.name, at: b.end || b.start, title: `${kindLabel(b.kind)} · ${b.worktree || '?'}${b.agent ? ' · ' + b.agent : ''}`, meta: b.command || '', ok: b.ok, secs: b.secs, status: b.ok ? 'ok' : `exit ${b.exit}`, lines: b.closing || null, raw: b });
for (const pc of (D.pcs && D.pcs.machines) || []) for (const r of pc.recent) rows.push({ key: `${pc.id}-${r.job}`, where: pc.name, at: r.finished_at, title: `${r.kind} ${r.job}`, meta: r.title || r.summary || '', ok: r.ok, secs: r.secs, status: r.status, lines: [r.summary, ...(r.errors || [])].filter(Boolean), raw: r });
rows.sort((a, b) => String(b.at || '').localeCompare(String(a.at || '')));
return rows;
}
function renderDone() {
const rows = doneRows().slice(0, 40);
const boxRows = timeline((D.box && D.box.recent) || [], { limit: 200 }); const closing = new Map(boxRows.map(r => [r.id, r.closing]));
if (!rows.length) { $('done').innerHTML = `<div class="empty">No finished work yet. The box appends one line per build to /srv/builds/_log/builds.jsonl when bs_remote_run ends${D.box && D.box.log && !D.box.log.present ? ' (the file does not exist yet)' : ''}; the PCs' closing reports come from the relay intake.</div>`; return; }
const max = Math.max(60, ...rows.map(r => Number(r.secs) || 0));
$('done').innerHTML = rows.map(r => {
const w = Math.max(1.5, ((Number(r.secs) || 0) / max) * 100).toFixed(1);
const lines = r.lines || closing.get(r.key) || [];
const isOpen = open.has(r.key);
return `<div class="row ${isOpen ? 'open' : ''}" data-key="${esc(r.key)}" role="button" tabindex="0"><div class="at">${t(r.at)}<br><span style="opacity:.7">${esc(r.where)}</span></div><div class="what"><div class="t">${esc(r.title)}</div><div class="m" title="${esc(r.meta)}">${esc(r.meta)}</div></div><div class="bar"><i class="${r.ok ? '' : 'failed'}" style="width:${w}%"></i></div><div class="dur" style="color:${r.ok ? 'var(--good)' : 'var(--bad)'}">${esc(fmtDurShort(r.secs))}<br><span style="color:var(--muted);font-size:10px">${esc(r.status)}</span></div></div><div class="lines ${isOpen ? 'show' : ''}">${lines.length ? lines.map(l => esc(l)).join('\n') : 'no closing lines kept for this entry'}</div>`;
}).join('');
for (const el of $('done').querySelectorAll('.row')) { const f = () => { const k = el.dataset.key; open.has(k) ? open.delete(k) : open.add(k); renderDone(); }; el.onclick = f; el.onkeydown = e => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); f(); } }; }
}
function renderHeadline() {
const rows = headline(D.headline);
$('headline').innerHTML = rows.map(r => {
const max = Math.max(r.box_s || 0, r.mac_s || 0, r.pc_s || 0, 1);
return `<div class="h3"><div class="l">${esc(r.label)}</div><div class="v">${r.box_s === null ? '<span style="color:var(--muted);font-size:14px">not measured yet</span>' : esc(fmtDur(r.box_s)) + '<small>box</small>'}</div><div class="cmp">${r.mac_s !== null ? `Mac <b>${esc(fmtDur(r.mac_s))}</b>${r.speedup ? ` (${r.speedup}× slower)` : ''}` : 'Mac <b>not measured on this tree</b>'}${r.pc_s !== null ? ` · PC 1 <b>${esc(fmtDur(r.pc_s))}</b>` : ''}</div>${r.box_s !== null ? `<div class="meter"><i style="width:${(r.box_s / max * 100).toFixed(1)}%"></i>${r.mac_s !== null ? `<em style="left:${(r.mac_s / max * 100).toFixed(1)}%" title="Mac"></em>` : ''}${r.pc_s !== null ? `<em style="left:${(r.pc_s / max * 100).toFixed(1)}%;background:var(--cool)" title="PC 1"></em>` : ''}</div>` : ''}<div class="s">${esc(r.note || r.what)}${r.measured_at ? ` · measured ${t(r.measured_at, true)}` : ''}</div></div>`;
}).join('');
}
function spark(points, { max = 100, label = v => v } = {}) {
if (!points.length) return '';
const W = 300, H = 90, L = 30, R = 6, T = 8, B = 20;
const X = i => L + (points.length === 1 ? (W - L - R) / 2 : (i / (points.length - 1)) * (W - L - R));
const Y = v => T + (1 - Math.max(0, Math.min(max, v)) / max) * (H - T - B);
const path = points.map((p, i) => `${i ? 'L' : 'M'}${X(i).toFixed(1)} ${Y(p.v).toFixed(1)}`).join(' ');
const area = `${path} L${X(points.length - 1).toFixed(1)} ${(H - B).toFixed(1)} L${X(0).toFixed(1)} ${(H - B).toFixed(1)} Z`;
return `<svg viewBox="0 0 ${W} ${H}" aria-label="${esc(label(max))} at the top"><line x1="${L}" y1="${H - B}" x2="${W - R}" y2="${H - B}" stroke="#26262d"/><line x1="${L}" y1="${T}" x2="${L}" y2="${H - B}" stroke="#26262d"/><path d="${area}" fill="rgba(106,169,255,.16)"/><path d="${path}" fill="none" stroke="#6aa9ff" stroke-width="2"/>${points.map((p, i) => `<circle cx="${X(i).toFixed(1)}" cy="${Y(p.v).toFixed(1)}" r="2.5" fill="#6aa9ff"><title>${esc(p.t)}</title></circle>`).join('')}<text x="4" y="${T + 4}" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${esc(label(max))}</text><text x="4" y="${H - B}" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">0</text><text x="${L}" y="${H - 6}" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${esc(points[0].x)}</text><text x="${W - R}" y="${H - 6}" text-anchor="end" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${esc(points[points.length - 1].x)}</text></svg>`;
}
function bars(buckets) {
const W = 300, H = 90, T = 8, B = 18, max = Math.max(1, ...buckets.map(b => b.n));
const bw = (W - 8) / 24;
return `<svg viewBox="0 0 ${W} ${H}" aria-label="builds per hour today"><line x1="4" y1="${H - B}" x2="${W - 4}" y2="${H - B}" stroke="#26262d"/>${buckets.map((b, i) => { const h = (b.n / max) * (H - T - B); const ok = b.n ? (b.ok / b.n) * h : 0; const x = 4 + i * bw; return `<g><title>${String(i).padStart(2, '0')}:00 UTC: ${b.n} build${b.n === 1 ? '' : 's'}, ${b.failed} failed</title><rect x="${(x + 1).toFixed(1)}" y="${(H - B - h).toFixed(1)}" width="${(bw - 2).toFixed(1)}" height="${h.toFixed(1)}" fill="#ff6b5e"/><rect x="${(x + 1).toFixed(1)}" y="${(H - B - ok).toFixed(1)}" width="${(bw - 2).toFixed(1)}" height="${ok.toFixed(1)}" fill="#5fd08a"/></g>`; }).join('')}${[0, 6, 12, 18, 23].map(h => `<text x="${(4 + h * bw + bw / 2).toFixed(1)}" y="${H - 5}" text-anchor="middle" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${String(h).padStart(2, '0')}Z</text>`).join('')}<text x="4" y="${T + 4}" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${max}</text></svg>`;
}
function renderAnalytics() {
const builds = (D.box && D.box.recent) || [];
const day = new Date().toISOString().slice(0, 10);
const todays = builds.filter(b => String(b.end || b.start || '').startsWith(day));
$('an-sub').textContent = builds.length ? `${builds.length} logged builds, ${todays.length} today, ${todays.filter(b => b.ok).length} passed` : '';
if (!builds.length) { $('an').innerHTML = `<div class="empty">Analytics start with the first logged build on the box. The log is /srv/builds/_log/builds.jsonl${D.box && D.box.log && D.box.log.present ? ` (${D.box.log.total} lines, ${D.box.log.bad_lines} unreadable)` : ' (not written yet)'}.</div>`; return; }
const ph = buildsPerHour(builds, day);
const dk = durationByKind(builds);
const hr = hitRateSeries(builds);
const cmp = compareKinds(builds, D.baselines || {});
const ukNow = ukTime(new Date().toISOString());
$('an').innerHTML = [
`<div class="a"><div class="l">Builds per hour today<span>${todays.length} today, hours in UTC</span></div>${bars(ph)}<div class="note">Green passed, red failed. Local time now ${esc(ukNow.text)} (${esc(ukNow.utc.slice(11))}).</div></div>`,
`<div class="a"><div class="l">Duration by kind<span>median and p90, all logged</span></div><table><thead><tr><th>Kind</th><th class="num">n</th><th class="num">median</th><th class="num">p90</th><th class="num">max</th></tr></thead><tbody>${dk.map(d => `<tr><td>${esc(d.label)}</td><td class="num">${d.n}</td><td class="num">${esc(fmtDurShort(d.median_s))}</td><td class="num">${esc(fmtDurShort(d.p90_s))}</td><td class="num">${esc(fmtDurShort(d.max_s))}</td></tr>`).join('')}</tbody></table></div>`,
`<div class="a"><div class="l">sccache hit rate over time<span>per build</span></div>${hr.length ? spark(hr.map(p => ({ v: p.pct, t: `${ukTime(p.at).utc}: ${p.pct}% (${kindLabel(p.kind)})`, x: ukTime(p.at).text })), { max: 100, label: v => v + '%' }) : '<div class="empty">no sccache figures in the log yet</div>'}<div class="note">A cold tree misses everything; the second build of the same tree should climb towards 100%.</div></div>`,
`<div class="a"><div class="l">Box against Mac<span>same kind, median</span></div>${cmp.length ? `<table><thead><tr><th>Kind</th><th class="num">box</th><th class="num">Mac</th><th class="num">×</th></tr></thead><tbody>${cmp.map(c => `<tr><td>${esc(c.label)}</td><td class="num">${c.box_s === null ? '–' : esc(fmtDurShort(c.box_s)) + ` <span style="color:var(--muted)">(${c.box_n})</span>`}</td><td class="num" title="${esc(c.mac_source || '')}">${c.mac_s === null ? '–' : esc(fmtDurShort(c.mac_s))}</td><td class="num">${c.speedup === null ? '' : c.speedup + '×'}</td></tr>`).join('')}</tbody></table><div class="note">Mac figures marked approximate come from the docs that measured them, not from this tree; a Mac row goes exact when the same tree is built here under with-lock.sh and logged.</div>` : '<div class="empty">no kinds to compare yet</div>'}</div>`,
].join('');
}
function renderSources() {
const s = D.sources || {};
const one = (name, v, what) => `<span>${esc(name)} <b class="${v && v.ok ? '' : v && v.at ? 'warn' : 'no'}">${v && v.ok ? 'live' : v && v.at ? 'stale' : 'missing'}</b>${v && v.at ? ' ' + esc(ago(v.at)) : ''}${v && v.error ? ': ' + esc(v.error) : ''}${!v || v.ok ? '' : ' · ' + esc(what)}</span>`;
$('sources').innerHTML = [one('box', s.box, 'collect.mjs on the box, systemd timer every 30 s'), one('mac', s.mac, 'push.mjs on the Mac, launchd every 60 s'), one('pcs', s.pcs, 'relay intake read by push.mjs'), `<span>${esc(D._feed || '')}: fetched ${esc(ago(D._fetched))}, written ${t(D.generated_at)} by ${esc(D.generated_by || '?')}</span>`, D.box && D.box.log ? `<span>log ${D.box.log.present ? `${D.box.log.total} lines${D.box.log.bad_lines ? `, ${D.box.log.bad_lines} unreadable` : ''}` : 'not written yet'}</span>` : ''].join('');
}
function apply(d) {
D = d; D._fetched = new Date().toISOString();
const age = (Date.now() - Date.parse(d.generated_at)) / 1000;
const live = $('live'); live.className = 'live' + (age > 900 ? ' down' : age > 420 ? ' stale' : '');
const u = ukTime(d.generated_at);
$('stamp').innerHTML = `${age > 420 ? 'stale: ' : ''}written ${t(d.generated_at)} (${esc(ago(d.generated_at))})${d._feed ? ' · ' + esc(d._feed) : ''}`;
renderServer(d.box, d.sources && d.sources.box); renderCrew(); renderNow(); renderQueue(); renderDone(); renderHeadline(); renderAnalytics(); renderSources();
}
const LIVE_URL = 'https://build.igneum.network/workers.json'; // the box itself, 30 s fresh (Caddy, read-only); the folder copy is up to 5 min behind
let skipLive = 0, feed = '';
async function load() {
let d = null, err = null;
if (!demo && skipLive <= 0) {
try { const r = await fetch(`${LIVE_URL}?t=${Date.now()}`, { cache: 'no-store', signal: AbortSignal.timeout(4000) }); if (!r.ok) throw new Error(`http ${r.status}`); d = await r.json(); feed = 'live from the box'; }
catch (e) { skipLive = 4; err = e; } // the box not answering: use the edge copy for the next minute before trying again
} else skipLive--;
if (!d) {
try { const file = demo ? 'workers.demo.json' : 'workers.json'; const r = await fetch(`${base}/${file}?t=${Date.now()}`, { cache: 'no-store' }); if (!r.ok) throw new Error(`http ${r.status}`); d = await r.json(); feed = demo ? 'demo document' : 'edge copy, up to 5 min behind the box'; }
catch (e) { $('live').className = 'live down'; $('stamp').textContent = `workers.json not reachable (${e && e.message || e})`; if (!D) { $('now').innerHTML = $('queue').innerHTML = `<div class="empty">No workers.json next to this page yet and the box does not answer. The Mac pusher writes the copy every 60 s and the fleet deploy carries it within 5 min.</div>`; } return; }
}
d._feed = feed; apply(d);
}
await load(); setInterval(load, 15000); setInterval(tick, 1000); setInterval(() => { if (D) { renderSources(); } }, 10000);
</script>
</body></html>

35
tools/workers/publish-workers.sh Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Put the worker dashboard page next to the fleet page and deploy the folder through publish-fleet.sh's mechanism.
# tools/workers/publish-workers.sh copy workers.html, shape.js, workers.demo.json; one push.mjs pass; deploy
# tools/workers/publish-workers.sh --no-deploy copy only (the fleet orchestrator's next deploy carries it)
# The folder is $DLSITE/<fleet-path>/ (dlsite-dir and fleet-path in ~/.config/igneum). The fleet index.html gets the
# Workers nav link if it does not carry one yet (the fleet page has no repo source; this is the only edit made to it).
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../.." && pwd)"
DLSITE="$(cat "$HOME/.config/igneum/dlsite-dir")"; P="$(cat "$HOME/.config/igneum/fleet-path")"
DIR="$DLSITE/$P"; [ -d "$DIR" ] || { echo "no $DIR" >&2; exit 1; }
cp "$HERE/page/workers.html" "$DIR/workers.html"
cp "$HERE/shape.js" "$DIR/shape.js"
cp "$HERE/fixtures/workers.demo.json" "$DIR/workers.demo.json"
if ! grep -q 'workers.html' "$DIR/index.html"; then
python3 - "$DIR/index.html" <<'PY'
import sys, re
p = sys.argv[1]; s = open(p).read()
css = " nav { display: flex; gap: 2px; border: 1px solid var(--line); }\n nav a { font-family: var(--mono); font-size: 11px; letter-spacing: 0.1em; text-transform: uppercase; color: var(--muted); text-decoration: none; padding: 7px 12px; }\n nav a.on { color: var(--ink); background: var(--panel-2); }\n nav a:hover { color: var(--ember-2); }\n"
s = s.replace(" .live {", css + " .live {", 1)
s = s.replace(' <div class="live"><b></b><span id="stamp">', ' <nav aria-label="Pages"><a href="./" class="on" aria-current="page">Fleet</a><a href="workers.html">Workers</a></nav>\n <div class="live"><b></b><span id="stamp">', 1)
open(p, 'w').write(s)
print("nav link added to the fleet index.html")
PY
fi
node "$HERE/push.mjs" --no-deploy
if [ "${1:-}" != --no-deploy ]; then
tmp="$(mktemp)"; cp "$DIR/fleet.json" "$tmp"
"$ROOT/tools/fleet/publish-fleet.sh" "$tmp" >/dev/null || "/Users/joshm/Projects/igneum/tools/fleet/publish-fleet.sh" "$tmp"
rm -f "$tmp"
for i in $(seq 1 12); do
if curl -fsS --max-time 10 "https://dl.igneum.network/$P/workers.html?t=$(date +%s)" | grep -c "Igneum Workers" >/dev/null; then echo "live: https://dl.igneum.network/$P/workers.html"; exit 0; fi
sleep 5
done
echo "deployed but the edge does not serve workers.html yet" >&2; exit 2
fi

205
tools/workers/push.mjs Normal file
View file

@ -0,0 +1,205 @@
#!/usr/bin/env node
// The worker dashboard's Mac side, run by launchd every 60 s (tools/workers/launchd/com.igneum.workers-push.plist).
// Gathers what only this Mac can see, merges it with the box's file and publishes:
// 1. the Mac's with-lock slots (/tmp/igneum-locks: build-0..2, run-0..2, measure, build) and their waiters
// 2. the two PCs' relay job states from the log intake (Neon, the same reader as tools/jobs.mjs): the running job,
// the queue (signed jobs for that machine with no report yet), the last 20 closing reports with durations
// 3. scp of those two facts to the box (/srv/workers/sources/{mac,pcs}.json) so the box's own file is whole too
// 4. the box's /srv/workers/workers.json pulled back, the Mac and PC facts overlaid, headline.json and the Mac baselines
// added, written to $DLSITE/<fleet-path>/workers.json
// 5. a Vercel deploy ONLY when the live copy is older than 6 minutes: the fleet orchestrator deploys the same folder every
// 5 minutes (tools/fleet/autorun.py), so the file normally rides along with it; one deploy per publish is the
// publish-fleet.sh mechanism and a deploy a minute is not wanted.
//
// node tools/workers/push.mjs one pass (what launchd runs)
// node tools/workers/push.mjs --stdout print the merged document, publish nothing
// node tools/workers/push.mjs --deploy force the deploy this pass
// node tools/workers/push.mjs --no-deploy never deploy this pass (write the folder copy only)
// Reads ~/.config/igneum/{build-server,env,dl-token,dlsite-dir,fleet-path} and ~/.ssh/igneum_ed25519. Prints nothing
// secret. Log: ~/Library/Logs/igneum-workers-push.log (launchd) or stderr.
import { readFileSync, readdirSync, writeFileSync, renameSync, mkdirSync, existsSync, statSync } from 'node:fs';
import { spawnSync, execFileSync } from 'node:child_process';
import { homedir, hostname, tmpdir } from 'node:os';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseLockDir, summaryOf, deriveKind, BASELINES } from './lib.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const argv = process.argv.slice(2);
const has = f => argv.includes(`--${f}`);
const cfg = n => { try { return readFileSync(join(homedir(), '.config', 'igneum', n), 'utf8').trim(); } catch { return ''; } };
const iso = ms => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
const log = (...a) => console.error(`${iso(Date.now())} push: ${a.join(' ')}`);
const KEY = process.env.IGNEUM_BUILD_KEY || join(homedir(), '.ssh', 'igneum_ed25519');
const HOST = (cfg('build-server') || '').split('\n')[0].trim(); // build@<ip>
const DLSITE = cfg('dlsite-dir'), FLEET = cfg('fleet-path');
const STALE_LIVE_S = 360;
const SSH_OPTS = ['-i', KEY, '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=accept-new', '-o', 'ConnectTimeout=10', '-o', 'ControlMaster=auto', '-o', `ControlPath=${join(homedir(), '.ssh', 'cm', 'igneum-workers-%r@%h:%p')}`, '-o', 'ControlPersist=600'];
mkdirSync(join(homedir(), '.ssh', 'cm'), { recursive: true });
// ---- 1. the Mac's slots ----------------------------------------------------------------------------------------------------
const alive = pid => { try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; } };
export function macSlots(now = Date.now(), dir = '/tmp/igneum-locks') {
let files = [];
try { files = readdirSync(dir).map(name => ({ name, text: readFileSync(join(dir, name), 'utf8') })); } catch { files = []; }
const r = parseLockDir(files, { now, aliveFn: alive });
let cap = 3; try { cap = Math.max(1, Math.min(3, parseInt(cfg('build-slots'), 10) || 3)); } catch { cap = 3; }
// waiters: the python with-lock.sh exec'd into, still in its wait loop (`python3 - build /tmp/igneum-locks <cmd>`); the
// holder exec'd on into its command, so a process still carrying that argv is waiting. Our own argv never matches.
const waiters = [];
const ps = spawnSync('ps', ['-eo', 'pid=,etimes=,args='], { encoding: 'utf8' }).stdout || '';
for (const line of ps.split('\n')) {
const m = /^\s*(\d+)\s+(\d+)\s+(\S*python3?) - (build|measure|run) \/tmp\/igneum-locks (.*)$/.exec(line);
if (!m || Number(m[1]) === process.pid) continue;
const held = r.held.find(h => h.pid === Number(m[1]));
if (held) continue; // measure mode keeps python in front? no: it execs too; a listed one is a waiter
waiters.push({ pid: Number(m[1]), mode: m[4], waiting_s: Number(m[2]), command: m[5].slice(0, 160), kind: deriveKind({ command: m[5] }) });
}
for (const h of r.held) h.worktree = h.worktree || worktreeOfPid(h.pid);
return { name: hostname().replace(/\.local$/, ''), collected_at: iso(now), build_slots: cap, slots: { count: cap, held: r.held, free: r.free }, queue: waiters };
}
// the holder's cwd names the worktree it builds in (lsof is slow; /proc does not exist here, so `lsof -p` only when needed)
function worktreeOfPid(pid) {
const out = spawnSync('lsof', ['-a', '-p', String(pid), '-d', 'cwd', '-Fn'], { encoding: 'utf8', timeout: 4000 }).stdout || '';
const m = /\n n(\/Users\/[^\n]*)/.exec('\n' + out.replace(/^n/m, ' n'));
const p = m ? m[1] : (out.split('\n').find(l => l.startsWith('n/')) || '').slice(1);
const w = /\/Projects\/(igneum(?:-wt-[\w.-]+)?)/.exec(p);
return w ? w[1] : null;
}
// ---- 2. the PCs through the intake ---------------------------------------------------------------------------------------
const MACHINES = [
{ id: 'ae432dc7', name: 'PC 1', role: 'builds', note: 'Linux and Windows binaries, Josh\'s desk' },
{ id: '1ccfe586', name: 'PC 2', role: 'test suites', note: 'cargo test suites and shard runs' },
];
function db() {
const m = /^DATABASE_URL=(.*)$/m.exec(cfg('env'));
if (!m) return null;
const url = m[1].trim().replace(/^['"]|['"]$/g, '');
const host = new URL(url).hostname.replace('-pooler', '');
return async (query, params = []) => {
const r = await fetch(`https://${host}/sql`, { method: 'POST', headers: { 'Neon-Connection-String': url, 'Content-Type': 'application/json' }, body: JSON.stringify({ query, params }), signal: AbortSignal.timeout(15000) });
const j = await r.json();
if (!r.ok) throw new Error(j.message || `http ${r.status}`);
return j.rows;
};
}
async function signedJobs() {
const tok = cfg('dl-token'); if (!tok) return [];
try {
const r = await fetch(`https://dl.igneum.network/dl/${tok}/igneum-jobs.signed.json?t=${Date.now()}`, { headers: { 'Cache-Control': 'no-cache' }, signal: AbortSignal.timeout(15000) });
if (!r.ok) return [];
const env = await r.json(); const f = JSON.parse(env.file);
return Array.isArray(f.jobs) ? f.jobs : [];
} catch (e) { log('jobs file:', e.message); return []; }
}
export function shapePcs(rows, jobs, now = Date.now()) {
// rows: the newest upload per run_id with its machine; a run_id is job-<id>-<machine id>
const byMachine = new Map(MACHINES.map(m => [m.id, { ...m, machine: null, running: null, queue: [], recent: [], last_report_at: null }]));
const seenJob = new Set();
for (const r of rows) {
const mid = (/-([0-9a-f]{8})$/.exec(r.machine || '') || [])[1] || (/-([0-9a-f]{8})$/.exec(r.run_id || '') || [])[1];
const pc = byMachine.get(mid); if (!pc) continue;
pc.machine = pc.machine || r.machine;
const s = summaryOf(r.lines); if (!s) continue;
const jobId = s.job || (r.run_id || '').replace(/^job-/, '').replace(new RegExp(`-${mid}$`), '');
seenJob.add(jobId);
const kind = s.kind || (jobId.split('-')[0]) || 'job';
const at = Date.parse(String(r.received_at).replace(' ', 'T').replace(/([+-]\d\d)$/, '$1:00'));
if (!pc.last_report_at || at > Date.parse(pc.last_report_at)) pc.last_report_at = iso(at);
if (s.status === 'running') {
// a progress report under 15 minutes old is a live job; older is a job whose closing report never came
const fresh = now - at < 15 * 60_000;
const entry = { job: jobId, kind, title: titleOf(jobs, jobId), started_at: s.started_at || null, stage: s.stage, elapsed_s: s.started_at ? Math.max(0, Math.round((now - Date.parse(s.started_at)) / 1000)) : null, last_report_at: iso(at), stale: !fresh };
if (!pc.running || Date.parse(entry.last_report_at) > Date.parse(pc.running.last_report_at)) pc.running = entry;
} else {
pc.recent.push({ job: jobId, kind, title: titleOf(jobs, jobId), status: s.status, ok: s.status === 'done' && Number(s.exit) === 0, exit: s.exit, secs: s.duration_s, started_at: s.started_at || null, finished_at: s.finished_at && s.finished_at !== 'closed' ? s.finished_at : iso(at), summary: String(s.summary || '').slice(0, 240), errors: (s.errors || []).map(e => e.slice(0, 200)), stage: s.stage });
}
}
for (const pc of byMachine.values()) {
pc.recent.sort((a, b) => String(b.finished_at).localeCompare(String(a.finished_at))); pc.recent = pc.recent.slice(0, 20);
if (pc.running && pc.running.stale) { pc.running = null; }
// the queue: signed, unexpired jobs aimed at this machine that no report has mentioned yet
for (const j of jobs) {
const t = j.target || {}; const ids = Array.isArray(t.machine_ids) ? t.machine_ids : t.machine_ids ? [t.machine_ids] : [];
if (!ids.includes(pc.id)) continue;
if (Date.parse(j.expires_at) < now) continue;
if (seenJob.has(j.id)) continue;
pc.queue.push({ job: j.id, kind: j.kind, title: j.title || '', expires_at: j.expires_at });
}
pc.queue.sort((a, b) => String(a.job).localeCompare(String(b.job)));
}
return { collected_at: iso(now), machines: [...byMachine.values()] };
}
const titleOf = (jobs, id) => { const j = jobs.find(j => j.id === id); return j ? (j.title || '') : ''; };
async function pcs() {
const sql = db(); if (!sql) return { error: 'no DATABASE_URL in ~/.config/igneum/env', collected_at: iso(Date.now()), machines: [] };
const rows = await sql(`SELECT DISTINCT ON (run_id) run_id, machine, received_at, lines FROM miner_logs WHERE run_id LIKE 'job-%' AND label LIKE 'job-%' AND received_at > now() - interval '7 days' ORDER BY run_id, received_at DESC`);
const jobs = await signedJobs();
return shapePcs(rows, jobs);
}
// ---- 3 and 4. the box -------------------------------------------------------------------------------------------------------
function ssh(args, input) { return spawnSync('ssh', [...SSH_OPTS, HOST, ...args], { encoding: 'utf8', input, timeout: 30000, maxBuffer: 16 * 1024 * 1024 }); }
function dropToBox(name, doc) {
const r = ssh([`cat > /srv/workers/sources/.${name}.tmp && mv /srv/workers/sources/.${name}.tmp /srv/workers/sources/${name}.json`], JSON.stringify(doc));
if (r.status !== 0) throw new Error(`scp ${name}: ${(r.stderr || '').trim().slice(0, 200) || 'ssh failed'}`);
}
function pullFromBox() {
const r = ssh(['cat /srv/workers/workers.json; echo; echo __SEP__; cat /srv/workers/headline.json 2>/dev/null || echo null']);
if (r.status !== 0) throw new Error((r.stderr || '').trim().slice(0, 200) || 'ssh failed');
const [w, h] = r.stdout.split('__SEP__');
return { workers: JSON.parse(w), headline: JSON.parse((h || 'null').trim() || 'null') };
}
// ---- 5. publish ----------------------------------------------------------------------------------------------------------------
async function liveAge() {
try {
const r = await fetch(`https://dl.igneum.network/${FLEET}/workers.json?t=${Date.now()}`, { cache: 'no-store', signal: AbortSignal.timeout(10000) });
if (!r.ok) return Infinity;
const j = await r.json(); return (Date.now() - Date.parse(j.generated_at)) / 1000;
} catch { return Infinity; }
}
function deploy() {
const r = spawnSync('npx', ['--yes', 'vercel@latest', '--global-config', join(homedir(), '.config', 'igneum', 'vercel'), 'deploy', '--prod', '--yes'], { cwd: DLSITE, encoding: 'utf8', timeout: 240000, env: { ...process.env, PATH: `/opt/homebrew/bin:/usr/local/bin:${process.env.PATH || ''}` } });
return r.status === 0;
}
async function main() {
const now = Date.now();
const doc = { v: 1, generated_at: iso(now), generated_by: `push.mjs on ${hostname().replace(/\.local$/, '')}`, box: null, mac: null, pcs: null, sources: {}, baselines: BASELINES, headline: null };
// the Mac
try { doc.mac = macSlots(now); doc.sources.mac = { ok: true, at: doc.mac.collected_at }; } catch (e) { doc.sources.mac = { ok: false, error: String(e.message || e) }; }
// the PCs
try { doc.pcs = await pcs(); doc.sources.pcs = doc.pcs.error ? { ok: false, error: doc.pcs.error } : { ok: true, at: doc.pcs.collected_at }; } catch (e) { doc.sources.pcs = { ok: false, error: `intake: ${String(e.message || e).slice(0, 160)}` }; }
// the box: drop ours, pull theirs
if (!HOST) doc.sources.box = { ok: false, error: 'no ~/.config/igneum/build-server' };
else {
try {
if (doc.mac) dropToBox('mac', doc.mac);
if (doc.pcs && !doc.pcs.error) dropToBox('pcs', doc.pcs);
const { workers, headline } = pullFromBox();
doc.box = workers.box; doc.headline = headline;
const age = (now - Date.parse(workers.generated_at)) / 1000;
doc.sources.box = { ok: age < 120, at: workers.generated_at, age_s: Math.round(age), ...(age >= 120 ? { error: `the box's collector last wrote ${Math.round(age)} s ago (timer igneum-workers.timer)` } : {}) };
} catch (e) { doc.sources.box = { ok: false, error: `ssh to the box failed: ${String(e.message || e).slice(0, 160)}` }; }
}
if (!doc.headline) doc.headline = null;
const text = JSON.stringify(doc);
if (has('stdout')) { process.stdout.write(text + '\n'); return; }
if (!DLSITE || !FLEET) { log('no dlsite-dir or fleet-path in ~/.config/igneum; nothing published'); return; }
const dir = join(DLSITE, FLEET); mkdirSync(dir, { recursive: true });
const tmp = join(dir, '.workers.json.tmp'); writeFileSync(tmp, text); renameSync(tmp, join(dir, 'workers.json'));
let why = 'folder copy written; the fleet orchestrator deploys the folder within 5 min';
if (!has('no-deploy')) {
const age = has('deploy') ? Infinity : await liveAge();
if (age > STALE_LIVE_S) { const ok = deploy(); why = ok ? `deployed (live copy was ${age === Infinity ? 'missing' : Math.round(age) + ' s old'})` : 'DEPLOY FAILED'; }
else why = `live copy ${Math.round(age)} s old, no deploy`;
}
log(`box ${doc.sources.box.ok ? 'ok' : 'NO: ' + doc.sources.box.error}; mac ${doc.sources.mac.ok ? `${doc.mac.slots.held.length} held, ${doc.mac.queue.length} waiting` : 'NO'}; pcs ${doc.sources.pcs.ok ? doc.pcs.machines.map(m => `${m.name} ${m.running ? 'running ' + m.running.job : 'idle'} q${m.queue.length}`).join(', ') : 'NO: ' + doc.sources.pcs.error}; ${why}`);
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
main().catch(e => { log('failed:', e.message || e); process.exit(1); });
}

99
tools/workers/shape.js Normal file
View file

@ -0,0 +1,99 @@
// Data shaping for the worker dashboard page (workers.html loads it as a module; node --test imports the same file).
// Pure functions over workers.json: the analytics strip, the timeline bars, the time formatting. No DOM here.
export const KIND_LABEL = { 'node-linux': 'node Linux', 'node-windows': 'node Windows', app: 'app', 'app-windows': 'app Windows', suite: 'suite', check: 'check', prove: 'prove', pow: 'pow', other: 'other' };
export const kindLabel = k => KIND_LABEL[k] || k || 'other';
export function quantile(sorted, q) {
if (!sorted.length) return null;
const pos = (sorted.length - 1) * q, lo = Math.floor(pos), hi = Math.ceil(pos);
return lo === hi ? sorted[lo] : sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo);
}
/// Median and p90 duration per kind over finished builds (failed ones included: a failed build still cost its time).
export function durationByKind(builds) {
const by = new Map();
for (const b of builds || []) { if (!Number.isFinite(b.secs)) continue; const k = b.kind || 'other'; if (!by.has(k)) by.set(k, []); by.get(k).push(b.secs); }
const out = [];
for (const [kind, arr] of by) { arr.sort((a, b) => a - b); out.push({ kind, label: kindLabel(kind), n: arr.length, median_s: quantile(arr, 0.5), p90_s: quantile(arr, 0.9), min_s: arr[0], max_s: arr[arr.length - 1] }); }
return out.sort((a, b) => b.n - a.n || a.kind.localeCompare(b.kind));
}
/// Builds that ENDED in each hour of `day` (UTC date string), 24 buckets of { hour, n, ok, failed }.
export function buildsPerHour(builds, day) {
const buckets = Array.from({ length: 24 }, (_, hour) => ({ hour, n: 0, ok: 0, failed: 0 }));
for (const b of builds || []) {
const t = b.end || b.start; if (!t || !String(t).startsWith(day)) continue;
const h = Number(String(t).slice(11, 13)); if (!(h >= 0 && h < 24)) continue;
buckets[h].n++; if (b.ok) buckets[h].ok++; else buckets[h].failed++;
}
return buckets;
}
/// sccache hit rate per build in time order, for the sparkline: [{ at, pct, kind }].
export function hitRateSeries(builds) {
return (builds || []).filter(b => b.sccache && Number.isFinite(b.sccache.hit_rate_pct)).map(b => ({ at: b.end || b.start, pct: b.sccache.hit_rate_pct, kind: b.kind })).sort((a, b) => String(a.at).localeCompare(String(b.at)));
}
/// Box against Mac for the same kind: the box's median from the log, the Mac's from `baselines` ({ kind: { secs, source } }).
export function compareKinds(builds, baselines) {
const box = new Map(durationByKind(builds).map(d => [d.kind, d]));
const kinds = new Set([...box.keys(), ...Object.keys(baselines || {})]);
return [...kinds].map(kind => {
const b = box.get(kind); const m = baselines && baselines[kind];
const ratio = b && m && m.secs ? Math.round((m.secs / b.median_s) * 10) / 10 : null;
return { kind, label: kindLabel(kind), box_s: b ? b.median_s : null, box_n: b ? b.n : 0, mac_s: m ? m.secs : null, mac_source: m ? m.source || null : null, speedup: ratio };
}).filter(r => r.box_s !== null || r.mac_s !== null).sort((a, b) => (b.box_n - a.box_n) || a.kind.localeCompare(b.kind));
}
/// The timeline: newest first, each with a bar width as a share of the longest, and the lines to show on click.
export function timeline(builds, { limit = 40 } = {}) {
const rows = (builds || []).slice().sort((a, b) => String(b.end || b.start || '').localeCompare(String(a.end || a.start || ''))).slice(0, limit);
const max = Math.max(60, ...rows.map(r => Number(r.secs) || 0));
return rows.map(r => ({
...r,
bar_pct: Math.max(1.5, Math.round(((Number(r.secs) || 0) / max) * 1000) / 10),
closing: closingLines(r),
}));
}
/// What a click on a done build shows: the facts the log carries, one per line.
export function closingLines(b) {
const out = [];
if (b.command) out.push(`$ ${b.command}`);
if (b.worktree) out.push(`${b.worktree}/${b.crate || ''}${b.branch ? ` (${b.branch}${b.sha ? ' ' + String(b.sha).slice(0, 8) : ''})` : ''}`);
out.push(`exit ${b.exit ?? '?'} after ${fmtDur(b.secs)}${b.wait_s ? `, waited ${fmtDur(b.wait_s)} for a slot` : ''}${b.slot !== undefined && b.slot !== null ? ` on build-${b.slot}` : ''}`);
if (b.compiles !== undefined) out.push(`${b.compiles} compiles, sccache ${b.sccache && b.sccache.hit_rate_pct !== null ? b.sccache.hit_rate_pct + '% hits' : 'no stats'} (${b.sccache ? `${b.sccache.hits} hit, ${b.sccache.misses} miss` : ''})`);
if (Array.isArray(b.load_end) && b.load_end.length) out.push(`load at the end ${b.load_end.join(' ')}`);
for (const a of b.artefacts || []) out.push(`${a.path} ${fmtBytes(a.bytes)} sha256 ${String(a.sha256 || '').slice(0, 16)}…`);
if (b.agent) out.push(`agent ${b.agent}`);
return out;
}
export const fmtDur = s => { s = Math.max(0, Math.round(Number(s) || 0)); const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), r = s % 60; return h ? `${h} h ${String(m).padStart(2, '0')} min` : m ? `${m} min ${String(r).padStart(2, '0')} s` : `${r} s`; };
export const fmtDurShort = s => { s = Math.max(0, Math.round(Number(s) || 0)); const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), r = s % 60; return h ? `${h}:${String(m).padStart(2, '0')}:${String(r).padStart(2, '0')}` : `${m}:${String(r).padStart(2, '0')}`; };
export const fmtBytes = n => { n = Number(n) || 0; if (n < 1024) return `${n} B`; if (n < 1048576) return `${(n / 1024).toFixed(1)} KB`; if (n < 1073741824) return `${(n / 1048576).toFixed(1)} MB`; return `${(n / 1073741824).toFixed(2)} GB`; };
export const fmtBps = n => { n = Number(n) || 0; if (n < 1e6) return `${(n / 1e3).toFixed(0)} kb/s`; if (n < 1e9) return `${(n / 1e6).toFixed(1)} Mb/s`; return `${(n / 1e9).toFixed(2)} Gb/s`; };
export const fmtUptime = s => { s = Number(s) || 0; const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60); return d ? `${d} d ${h} h` : h ? `${h} h ${m} min` : `${m} min`; };
/// UK local time for display with the UTC in the title; `tz` defaults to Europe/London.
export function ukTime(isoStr, { tz = 'Europe/London', withDate = false } = {}) {
if (!isoStr) return { text: '', utc: '' };
const d = new Date(isoStr); if (Number.isNaN(d.getTime())) return { text: String(isoStr), utc: '' };
const opts = { timeZone: tz, hour: '2-digit', minute: '2-digit', hour12: false, ...(withDate ? { day: '2-digit', month: 'short' } : {}) };
let text; try { text = new Intl.DateTimeFormat('en-GB', opts).format(d); } catch { text = d.toISOString().slice(11, 16); }
return { text, utc: d.toISOString().replace('T', ' ').slice(0, 19) + ' UTC' };
}
/// "x min ago" for stamps.
export const ago = (isoStr, now = Date.now()) => { if (!isoStr) return ''; const s = Math.max(0, Math.round((now - Date.parse(isoStr)) / 1000)); return s < 60 ? `${s} s ago` : s < 3600 ? `${Math.floor(s / 60)} min ago` : `${Math.floor(s / 3600)} h ${Math.floor((s % 3600) / 60)} min ago`; };
/// The headline timings: the three the build-server agent measures with the Mac baseline beside each.
export function headline(h) {
const rows = [
{ key: 'clean_node', label: 'Clean node build', what: 'cold target dir, Linux node' },
{ key: 'incremental_under_load', label: 'Incremental under load', what: 'one file changed while another build runs' },
{ key: 'windows_cross', label: 'Windows cross', what: 'igneumd.exe and igneum-miner.exe' },
];
return rows.map(r => { const v = (h && h[r.key]) || {}; return { ...r, box_s: Number.isFinite(v.box_s) ? v.box_s : null, mac_s: Number.isFinite(v.mac_s) ? v.mac_s : null, pc_s: Number.isFinite(v.pc_s) ? v.pc_s : null, note: v.note || null, measured_at: v.measured_at || null, speedup: Number.isFinite(v.box_s) && Number.isFinite(v.mac_s) && v.box_s > 0 ? Math.round((v.mac_s / v.box_s) * 10) / 10 : null }; });
}

154
tools/workers/test/parsers.test.mjs Normal file
View file

@ -0,0 +1,154 @@
// node --test tools/workers/test : the collector's parsers against the fixtures, and the page's data shaping.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync, readdirSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseProcStat, cpuBusy, parseMeminfo, parseNetDev, parseLoadavg, parseSlotLine, splitLabel, deriveKind, parseLockDir, parseBuildsJsonl, parseSccacheStats, pickTemps, cargoProcess, summaryOf } from '../lib.mjs';
import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, closingLines, ukTime, headline, quantile } from '../shape.js';
const FX = join(dirname(fileURLToPath(import.meta.url)), '..', 'fixtures');
const fx = n => readFileSync(join(FX, n), 'utf8');
test('/proc/stat: cores and busy deltas', () => {
const a = parseProcStat(fx('proc-stat-0.txt')), b = parseProcStat(fx('proc-stat-1.txt'));
assert.equal(a.cores.length, 4);
const busy = cpuBusy(a, b);
assert.equal(busy.per_core.length, 4);
assert.equal(busy.per_core[0], 100); // 100 user jiffies, 0 idle
assert.equal(busy.per_core[1], 0); // idle only
assert.equal(busy.per_core[2], 100); // 50 user + 50 system
assert.equal(busy.per_core[3], 25); // 25 busy of 100
assert.equal(busy.cores_busy, 2);
// whole box: 1000 user + 200 system busy of 2000 -> 60 %
assert.equal(busy.busy_pct, 60);
// a zero delta never divides by zero
assert.equal(cpuBusy(a, a).busy_pct, 0);
});
test('/proc/meminfo, /proc/net/dev, /proc/loadavg', () => {
const m = parseMeminfo('MemTotal: 131556268 kB\nMemFree: 88262160 kB\nMemAvailable: 97647588 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n');
assert.equal(m.total_kb, 131556268); assert.equal(m.used_kb, 131556268 - 97647588); assert.equal(m.used_pct, 25.8);
const n = parseNetDev('Inter-| Receive | Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n lo: 17515759 1 0 0 0 0 0 0 17515759 1 0 0 0 0 0 0\nenp193s0f0np0: 994079034 2 0 0 0 0 0 0 80040958 3 0 0 0 0 0 0\n');
assert.deepEqual(n, [{ iface: 'enp193s0f0np0', rx_bytes: 994079034, tx_bytes: 80040958 }]);
assert.deepEqual(parseLoadavg('13.86 9.61 4.32 1/1235 36952\n'), { load: [13.86, 9.61, 4.32], running: 1, threads: 1235 });
});
test('slot line: pid, since, waited, label, agent, worktree, crate, command, target', () => {
const l = parseSlotLine('pid 36992 since 17:34:00Z waited 0 s: igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server', { day: '2026-10-06', now: '2026-10-06T18:00:00Z' });
assert.equal(l.pid, 36992); assert.equal(l.since, '2026-10-06T17:34:00Z'); assert.equal(l.waited_s, 0);
assert.equal(l.worktree, 'igneum-wt-build-server'); assert.equal(l.crate, 'vendor/igneum-node-bs');
assert.equal(l.target, 'x86_64-pc-windows-gnu'); assert.equal(l.agent, 'build-server');
assert.equal(deriveKind(l), 'node-windows');
// a clock time later than now is yesterday
const y = parseSlotLine('pid 5 since 23:50:00Z waited 2 s: x', { day: '2026-10-06', now: '2026-10-06T00:10:00Z' });
assert.equal(y.since, '2026-10-05T23:50:00Z');
assert.equal(parseSlotLine(''), null); assert.equal(parseSlotLine('garbage'), null);
// a Mac with-lock label is a bare command
const m = splitLabel('cargo build --release -p igneum-miner');
assert.equal(m.worktree, null); assert.equal(m.target, 'x86_64-unknown-linux-gnu');
});
test('kind rule matches the JSONL proposal', () => {
assert.equal(deriveKind({ command: 'cargo test --release -p kaspa-consensus-core', crate: 'vendor/igneum-node-v4' }), 'suite');
assert.equal(deriveKind({ command: 'cargo clippy --all-targets', crate: 'app/igneum-app' }), 'check');
assert.equal(deriveKind({ command: 'cargo build --release -p kaspad', crate: 'vendor/igneum-node-bs' }), 'node-linux');
assert.equal(deriveKind({ command: 'cargo build --release -p kaspad --target x86_64-pc-windows-gnu', crate: 'vendor/igneum-node-bs' }), 'node-windows');
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'app/igneum-app' }), 'app');
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'app/igneum-app', tool: 'cross-remote' }), 'app-windows');
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'proving/igneum-prove' }), 'prove');
assert.equal(deriveKind({ kind: 'suite', command: 'cargo build' }), 'suite'); // an explicit kind wins
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'tools/x' }), 'other');
});
test('lock dir fixture: one held slot, one stale line, one empty, count from slots', () => {
const dir = join(FX, 'locks');
const files = readdirSync(dir).map(name => ({ name, text: readFileSync(join(dir, name), 'utf8') }));
const alive = pid => pid === 36992; // only the real holder is alive; pid 1's line is stale
const r = parseLockDir(files, { day: '2026-10-06', now: '2026-10-06T18:00:00Z', aliveFn: alive });
assert.equal(r.count, 1);
assert.equal(r.held.length, 1); assert.equal(r.held[0].slot, 'build-0'); assert.equal(r.held[0].kind, 'node-windows'); assert.equal(r.held[0].agent, 'build-server');
assert.deepEqual(r.free, ['build-1', 'build-2']);
// an explicit flock verdict overrides the pid check
const r2 = parseLockDir(files.map(f => ({ ...f, held: f.name === 'build-1' })), { day: '2026-10-06', now: '2026-10-06T18:00:00Z' });
assert.deepEqual(r2.held.map(h => h.slot), ['build-1']);
});
test('builds JSONL fixture: five rows, one bad line, newest first, derived fields', () => {
const r = parseBuildsJsonl(fx('builds.jsonl'));
assert.equal(r.total, 5); assert.equal(r.bad, 1);
assert.equal(r.rows[0].id, 'igneum-build-1-1759774200-37300');
const win = r.rows.find(b => b.tool === 'cross-remote');
assert.equal(win.kind, 'node-windows'); assert.equal(win.ok, true); assert.equal(win.sccache.hit_rate_pct, 0); assert.equal(win.artefacts.length, 2);
const suite = r.rows.find(b => b.exit === 101);
assert.equal(suite.kind, 'suite'); assert.equal(suite.ok, false);
const lin = r.rows.find(b => b.id.endsWith('37001'));
assert.equal(lin.kind, 'node-linux'); assert.equal(lin.target, 'x86_64-unknown-linux-gnu'); assert.equal(lin.sccache.hit_rate_pct, 71.7);
// secs derived from start and end when missing
const d = parseBuildsJsonl('{"start":"2026-10-06T10:00:00Z","end":"2026-10-06T10:05:30Z","exit":0,"command":"cargo build"}');
assert.equal(d.rows[0].secs, 330);
});
test('sccache --show-stats', () => {
const s = parseSccacheStats(fx('sccache-stats.txt'));
assert.equal(s.requests, 2294); assert.equal(s.hits, 790); assert.equal(s.misses, 1886); assert.equal(s.hit_rate_pct, 29.5);
assert.equal(s.cache_size, '12 GiB'); assert.equal(s.max_size, '100 GiB');
assert.equal(parseSccacheStats('Cache hits 0\nCache misses 10\n').hit_rate_pct, 0);
});
test('hwmon pick: hottest per class, cpu first, nothing absurd', () => {
const t = pickTemps([{ name: 'bnxt_en', label: null, millic: 55000 }, { name: 'nvme', label: 'Composite', millic: 36850 }, { name: 'nvme', label: 'Composite', millic: 34850 }, { name: 'k10temp', label: 'Tctl', millic: 75125 }, { name: 'k10temp', label: 'Tccd1', millic: 0 }, { name: 'acpitz', label: null, millic: 999000 }]);
assert.deepEqual(t.map(x => [x.name, x.c]), [['cpu', 75.1], ['nvme', 36.9], ['nic', 55]]);
});
test('a cargo process becomes a running build with its worktree, target, kind and start', () => {
const stat = '37210 (cargo) S 37209 37210 36992 0 -1 4194304 1 0 0 0 1 2 0 0 20 0 2 0 62341 1 2 3 4 5 6 7 8 9';
const p = cargoProcess({ pid: 37210, cwd: '/srv/builds/igneum-wt-build-server/vendor/igneum-node-bs', cmdline: ['/home/build/.rustup/toolchains/1.99.0-x86_64-unknown-linux-gnu/bin/cargo','build','--release','-p','kaspad','--target','x86_64-pc-windows-gnu','-j','90',''].join('\0'), stat, bootSec: 1759772000, clk: 100, now: (1759772000 + 1000) * 1000 });
assert.equal(p.worktree, 'igneum-wt-build-server'); assert.equal(p.crate, 'vendor/igneum-node-bs');
assert.equal(p.target, 'x86_64-pc-windows-gnu'); assert.equal(p.kind, 'node-windows');
assert.equal(p.started_at, new Date((1759772000 + 623.41) * 1000).toISOString().replace(/\.\d{3}Z$/, 'Z'));
assert.equal(p.elapsed_s, 377);
assert.equal(cargoProcess({ pid: 1, cmdline: ['rustc','--crate-name','x'].join('\0') }), null);
});
test('relay job report: SUMMARY and closing line, running without finished_at', () => {
const done = summaryOf('IGNEUM-APP 0.3.14\nSUMMARY {"job":"build-20261006-171423","kind":"build","status":"done","exit":0,"duration_s":622,"started_at":"2026-10-06T17:15:09Z","finished_at":"2026-10-06T17:25:31Z","summary":"ok"}\nSTAGE linux start 2026-10-06T17:16:00Z cap 20 min\njob build-20261006-171423: done (exit 0) after 622 s: every stage ok\n');
assert.equal(done.status, 'done'); assert.equal(done.duration_s, 622); assert.equal(done.stage, 'linux');
const run = summaryOf('SUMMARY {"job":"j1","kind":"build","status":"running","started_at":"2026-10-06T17:15:09Z"}\nSTAGE windows start 2026-10-06T17:30:00Z cap 20 min\n');
assert.equal(run.status, 'running'); assert.equal(run.stage, 'windows');
assert.equal(summaryOf('nothing here'), null);
});
// ---- page shaping ----------------------------------------------------------------------------------------------------------
const builds = parseBuildsJsonl(fx('builds.jsonl')).rows;
test('shape: median and p90 by kind, builds per hour, hit-rate series', () => {
const d = durationByKind(builds);
const nl = d.find(x => x.kind === 'node-linux'); assert.equal(nl.n, 1); assert.equal(nl.median_s, 801); assert.equal(nl.p90_s, 801);
assert.equal(quantile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 0.9), 9.1);
const h = buildsPerHour(builds, '2026-10-06');
assert.equal(h.length, 24); assert.equal(h[17].n, 1); assert.equal(h[18].n, 4); assert.equal(h[18].failed, 1); assert.equal(h[3].n, 0);
const s = hitRateSeries(builds);
assert.deepEqual(s.map(x => x.pct), [0, 71.7, 77.6, 85.7, 93.8]);
});
test('shape: box against Mac, timeline bars, closing lines, headline', () => {
const c = compareKinds(builds, { 'node-linux': { secs: 720, source: 'infra/cross/build-linux.sh, 4 Oct' }, 'app-windows': { secs: 900, source: 'runner' } });
const nl = c.find(x => x.kind === 'node-linux'); assert.equal(nl.mac_s, 720); assert.equal(nl.speedup, 0.9);
const aw = c.find(x => x.kind === 'app-windows'); assert.equal(aw.box_s, null); assert.equal(aw.mac_s, 900);
const t = timeline(builds, { limit: 3 });
assert.equal(t.length, 3); assert.equal(t[0].id, 'igneum-build-1-1759774200-37300');
assert.equal(Math.max(...t.map(r => r.bar_pct)), 100);
const lines = closingLines(builds.find(b => b.tool === 'cross-remote'));
assert.ok(lines[0].startsWith('$ cargo build')); assert.ok(lines.some(l => /igneumd\.exe\s+50\.5 MB\s+sha256 0f3a9c2e1d8b7a6f/.test(l)));
const hl = headline({ clean_node: { box_s: 600, mac_s: 2400 } });
assert.equal(hl[0].speedup, 4); assert.equal(hl[1].box_s, null); assert.equal(hl.length, 3);
});
test('shape: UK local time with the UTC in the tooltip', () => {
const t = ukTime('2026-10-06T17:34:00Z');
assert.equal(t.text, '18:34'); // BST on 6 October
assert.equal(t.utc, '2026-10-06 17:34:00 UTC');
assert.equal(ukTime('2026-12-06T17:34:00Z').text, '17:34'); // GMT in December
assert.deepEqual(ukTime(null), { text: '', utc: '' });
});