igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

Merge workers-dash: the worker dashboard on the fleet URL (collector on igneum-build-1, Mac pusher, workers.html)

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-06 17:59:54 +00:00
commit dbdfda0d21
21 changed files with 1376 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
infra/build-server/workers/igneum-workers.service Normal file
View file

@ -0,0 +1,21 @@
# The worker dashboard's collector on igneum-build-1: one pass, written by igneum-workers.timer every 30 s.
# Reads /proc, /sys, /srv/builds/_locks, the process table and /srv/builds/_log/builds.jsonl; writes /srv/workers/workers.json.
# Installed by infra/build-server/workers/install.sh (copies tools/workers/{lib,collect}.mjs to /srv/workers/bin).
[Unit]
Description=Igneum worker dashboard collector (one pass)
After=local-fs.target
[Service]
Type=oneshot
User=build
Group=build
Environment=HOME=/home/build
Environment=PATH=/home/build/.cargo/bin:/usr/local/bin:/usr/bin:/bin
Environment=SCCACHE_DIR=/srv/sccache
Environment=IGNEUM_WORKERS_DIR=/srv/workers
Environment=IGNEUM_BUILD_SLOTS_DIR=/srv/builds/_locks
Environment=IGNEUM_BUILD_ROOT=/srv/builds
WorkingDirectory=/srv/workers/bin
ExecStart=/usr/local/bin/node /srv/workers/bin/collect.mjs
TimeoutStartSec=25
Nice=10

13
infra/build-server/workers/igneum-workers.timer Normal file
View file

@ -0,0 +1,13 @@
# Fires the collector every 30 s from boot. `systemctl list-timers igneum-workers.timer` shows the next pass;
# `journalctl -u igneum-workers.service -n 20` the last errors.
[Unit]
Description=Igneum worker dashboard collector, every 30 s
[Timer]
OnBootSec=20s
OnUnitActiveSec=30s
AccuracySec=1s
Unit=igneum-workers.service
[Install]
WantedBy=timers.target

14
infra/build-server/workers/install.sh Executable file
View file

@ -0,0 +1,14 @@
#!/usr/bin/env bash
# Install or refresh the worker dashboard collector on igneum-build-1 from this Mac.
# infra/build-server/workers/install.sh copies tools/workers/{lib,collect}.mjs and the two units, enables the timer
# Needs root over ssh (root@<ip> with ~/.ssh/igneum_ed25519); the host ip comes from ~/.config/igneum/build-server (build@<ip>).
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../../.." && pwd)"
KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')"
IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server" >&2; exit 1; }
SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10)
"${SSH[@]}" "root@$IP" 'install -d -o build -g build /srv/workers /srv/workers/bin /srv/workers/sources /srv/builds/_log; chown build:build /srv/builds/_log'
scp -q -i "$KEY" "$ROOT/tools/workers/lib.mjs" "$ROOT/tools/workers/collect.mjs" "build@$IP:/srv/workers/bin/"
scp -q -i "$KEY" "$HERE/igneum-workers.service" "$HERE/igneum-workers.timer" "root@$IP:/etc/systemd/system/"
"${SSH[@]}" "root@$IP" 'systemctl daemon-reload && systemctl enable --now igneum-workers.timer >/dev/null 2>&1; systemctl start igneum-workers.service; systemctl is-active igneum-workers.timer; systemctl list-timers igneum-workers.timer --no-pager | sed -n 2p; ls -la /srv/workers/workers.json'

160
tools/workers/collect.mjs Normal file
View file

@ -0,0 +1,160 @@
#!/usr/bin/env node
// The worker dashboard's collector. Runs on igneum-build-1 every 30 s (infra/build-server/workers/igneum-workers.timer)
// as the build user and writes /srv/workers/workers.json: what the MACHINE says (uptime, load, /proc/stat deltas per core,
// memory, the /srv disk, network bytes, hwmon temperatures), the slot and lock state under /srv/builds/_locks, the cargo
// processes with their worktree and target, the slot waiters, the recent builds from /srv/builds/_log/builds.jsonl and
// sccache's own counters. Nothing here is a reported number: every field is read from /proc, /sys, the lock files, the
// process table or the log the build itself appended.
//
// node tools/workers/collect.mjs write /srv/workers/workers.json (IGNEUM_WORKERS_DIR overrides)
// node tools/workers/collect.mjs --stdout print instead of writing
// node tools/workers/collect.mjs --sample-ms 1000 the /proc/stat and network delta window (default 1000)
//
// The Mac side (tools/workers/push.mjs) drops its own facts at /srv/workers/sources/mac.json (the Mac's with-lock slots) and
// /srv/workers/sources/pcs.json (the two PCs' relay job states); when present and under 10 minutes old they are merged
// into the same workers.json, so one file answers for the whole shop whichever side serves it. Needs Node 22, no deps.
import { readFileSync, readdirSync, writeFileSync, renameSync, mkdirSync, existsSync, statSync, readlinkSync, openSync, closeSync } from 'node:fs';
import { spawnSync } from 'node:child_process';
import { hostname } from 'node:os';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseProcStat, cpuBusy, parseMeminfo, parseNetDev, parseLoadavg, parseLockDir, parseBuildsJsonl, parseSccacheStats, pickTemps, cargoProcess, deriveKind, BASELINES } from './lib.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const argv = process.argv.slice(2);
const flag = (n, d) => { const i = argv.indexOf(`--${n}`); return i >= 0 ? (argv[i + 1] ?? true) : d; };
const OUT_DIR = process.env.IGNEUM_WORKERS_DIR || '/srv/workers';
const LOCKS = process.env.IGNEUM_BUILD_SLOTS_DIR || '/srv/builds/_locks';
const BUILDS = process.env.IGNEUM_BUILD_ROOT || '/srv/builds';
const LOG = process.env.IGNEUM_BUILDS_LOG || join(BUILDS, '_log', 'builds.jsonl');
const SAMPLE_MS = Number(flag('sample-ms', 1000));
const iso = ms => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
const read = p => { try { return readFileSync(p, 'utf8'); } catch { return ''; } };
const sh = (cmd, args, opts = {}) => { const r = spawnSync(cmd, args, { encoding: 'utf8', timeout: 8000, ...opts }); return r.status === 0 ? r.stdout : ''; };
// ---- machine facts ----------------------------------------------------------------------------------------------------
function machine() {
const stat0 = parseProcStat(read('/proc/stat')); const net0 = parseNetDev(read('/proc/net/dev')); const t0 = Date.now();
const end = t0 + SAMPLE_MS; while (Date.now() < end) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, end - Date.now());
const stat1 = parseProcStat(read('/proc/stat')); const net1 = parseNetDev(read('/proc/net/dev')); const dt = (Date.now() - t0) / 1000;
const cpu = cpuBusy(stat0, stat1);
const up = Number(read('/proc/uptime').split(' ')[0]) || 0;
const la = parseLoadavg(read('/proc/loadavg'));
const mem = parseMeminfo(read('/proc/meminfo'));
const df = sh('df', ['-B1', '--output=target,size,used,avail', BUILDS]).trim().split('\n').pop() || '';
const dfp = df.trim().split(/\s+/);
const disk = dfp.length >= 4 ? { mount: dfp[0], total_bytes: Number(dfp[1]), used_bytes: Number(dfp[2]), avail_bytes: Number(dfp[3]), used_pct: Number(dfp[1]) ? Math.round((Number(dfp[2]) / Number(dfp[1])) * 1000) / 10 : 0 } : null;
// the busiest interface carries the build traffic (rsync in, artefacts out)
const nets = net1.map(n => { const b = net0.find(x => x.iface === n.iface) || n; return { ...n, rx_bps: Math.max(0, Math.round((n.rx_bytes - b.rx_bytes) * 8 / dt)), tx_bps: Math.max(0, Math.round((n.tx_bytes - b.tx_bytes) * 8 / dt)) }; }).filter(n => n.rx_bytes + n.tx_bytes > 0).sort((a, b) => (b.rx_bytes + b.tx_bytes) - (a.rx_bytes + a.tx_bytes));
const sensors = [];
for (const h of safeList('/sys/class/hwmon')) {
const dir = join('/sys/class/hwmon', h); const name = read(join(dir, 'name')).trim();
for (const f of safeList(dir)) { const m = /^(temp\d+)_input$/.exec(f); if (!m) continue; sensors.push({ name, label: read(join(dir, `${m[1]}_label`)).trim() || null, millic: Number(read(join(dir, f))) }); }
}
const osr = read('/etc/os-release'); const pretty = (/^PRETTY_NAME="?([^"\n]+)"?/m.exec(osr) || [])[1] || null;
const clk = Number(sh('getconf', ['CLK_TCK']).trim()) || 100;
return { name: hostname(), os: pretty, cores: stat1.cores.length, uptime_s: Math.round(up), boot_at: iso(Date.now() - up * 1000), ...la, cpu, mem, disk, net: nets[0] || null, temps: pickTemps(sensors), clk, kernel: read('/proc/sys/kernel/osrelease').trim() || null };
}
const safeList = d => { try { return readdirSync(d); } catch { return []; } };
const alive = pid => { try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; } };
// flock probe without python: `flock -n <file> true` fails while the slot is held (the slot is held by another process's fd)
const flockHeld = p => { const r = spawnSync('flock', ['-n', p, 'true'], { timeout: 3000 }); return r.status !== 0; };
// ---- slots, waiters, running builds ---------------------------------------------------------------------------------------
function slots(now) {
const files = safeList(LOCKS).map(n => ({ name: n, text: read(join(LOCKS, n)), held: /^(build-\d+|run-\d+|build|measure)$/.test(n) ? flockHeld(join(LOCKS, n)) : undefined }));
return parseLockDir(files, { now, aliveFn: alive });
}
function procs(bootSec, clk, now) {
const running = [], waiters = []; let compilers = 0;
for (const d of safeList('/proc')) {
if (!/^\d+$/.test(d)) continue;
const pid = Number(d); const comm = read(`/proc/${d}/comm`).trim();
if (['rustc', 'cc1plus', 'cc1', 'ld', 'ld.lld', 'x86_64-w64-mingw32-ld'].includes(comm)) compilers++;
if (comm === 'cargo') {
let cwd = null; try { cwd = readlinkSync(`/proc/${d}/cwd`); } catch { cwd = null; }
const p = cargoProcess({ pid, cwd, cmdline: read(`/proc/${d}/cmdline`), stat: read(`/proc/${d}/stat`), bootSec, clk, now, root: BUILDS });
if (p) running.push(p);
}
if (comm === 'flock') {
const args = read(`/proc/${d}/cmdline`).split('\0').filter(Boolean);
if (args[1] === '-w' && /^\d+$/.test(args[2] || '')) {
const stat = read(`/proc/${d}/stat`); const rest = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
const ppid = Number(rest[1]); const startTicks = Number(rest[19]);
const since = Number.isFinite(startTicks) ? iso((bootSec + startTicks / clk) * 1000) : null;
const wait = read(join(LOCKS, `wait-${ppid}`)).trim(); // a waiter's own line, when the appender writes one
waiters.push({ pid: ppid, since, waiting_s: since ? Math.max(0, Math.round((now - Date.parse(since)) / 1000)) : null, label: wait || null, ...(wait ? { kind: deriveKind(parseLabelSafe(wait)) } : {}) });
}
}
}
// the innermost cargo per worktree: a `cargo build` spawns no nested cargo, but a `cargo test` runs test binaries, not cargo
running.sort((a, b) => String(a.started_at).localeCompare(String(b.started_at)));
return { running, waiters, compilers };
}
function parseLabelSafe(line) { const r = /: (.*)$/.exec(line); return { command: r ? r[1] : line }; }
function sccache() {
const bin = existsSync('/home/build/.cargo/bin/sccache') ? '/home/build/.cargo/bin/sccache' : 'sccache';
const out = sh(bin, ['--show-stats'], { env: { ...process.env, SCCACHE_DIR: process.env.SCCACHE_DIR || '/srv/sccache' } });
return out ? parseSccacheStats(out) : null;
}
// ---- sources dropped by the Mac ------------------------------------------------------------------------------------------
function source(name, now) {
const p = join(OUT_DIR, 'sources', `${name}.json`);
try {
const st = statSync(p); const age = Math.round((now - st.mtimeMs) / 1000);
const j = JSON.parse(readFileSync(p, 'utf8'));
return { data: j, meta: { at: iso(st.mtimeMs), age_s: age, stale: age > 600, ok: true } };
} catch (e) { return { data: null, meta: { ok: false, error: e.code === 'ENOENT' ? 'the Mac has not pushed yet' : String(e.message || e) } }; }
}
// ---- main -------------------------------------------------------------------------------------------------------------------
export function collect() {
const now = Date.now();
const m = machine();
const bootSec = Math.round((now - m.uptime_s * 1000) / 1000);
const s = slots(now);
const p = procs(bootSec, m.clk, now);
// attach the slot holder to its build (the label names the worktree and crate the cargo runs in)
for (const r of p.running) {
const h = s.held.find(h => h.worktree === r.worktree && (!h.crate || h.crate === r.crate || !r.crate));
r.slot = h ? h.slot : null; r.agent = h ? h.agent : null; r.queued_at = h ? h.since : null; r.waited_s = h ? h.waited_s : null;
}
const logText = read(LOG);
const recent = parseBuildsJsonl(logText, { limit: 200 });
const mac = source('mac', now), pcs = source('pcs', now);
let headline = null; try { headline = JSON.parse(readFileSync(join(OUT_DIR, 'headline.json'), 'utf8')); } catch { headline = null; }
const doc = {
v: 1,
generated_at: iso(now),
generated_by: 'collect.mjs on ' + m.name,
box: {
...m, clk: undefined,
collected_at: iso(now),
sccache: sccache(),
slots: { count: s.count, held: s.held, free: s.free },
running: p.running,
queue: p.waiters,
compilers: p.compilers,
recent: recent.rows,
log: { path: LOG, present: !!logText, bad_lines: recent.bad, total: recent.total, note: logText ? null : 'builds.jsonl is not written yet: the build-server agent adds the append to bs_remote_run; until then this lane fills from nothing' },
},
mac: mac.data, pcs: pcs.data,
headline, baselines: BASELINES,
sources: { box: { ok: true, at: iso(now) }, mac: mac.meta, pcs: pcs.meta },
};
return doc;
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
const doc = collect();
const text = JSON.stringify(doc);
if (flag('stdout', false)) { process.stdout.write(text + '\n'); }
else {
mkdirSync(OUT_DIR, { recursive: true }); mkdirSync(join(OUT_DIR, 'sources'), { recursive: true });
const tmp = join(OUT_DIR, '.workers.json.tmp'); writeFileSync(tmp, text); renameSync(tmp, join(OUT_DIR, 'workers.json'));
const fd = openSync(join(OUT_DIR, 'last-run'), 'w'); closeSync(fd);
}
}

6
tools/workers/fixtures/builds.jsonl Normal file
View file

@ -0,0 +1,6 @@
{"v":1,"id":"igneum-build-1-1759770840-36992","host":"igneum-build-1","tool":"cross-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu -j 90","target":"x86_64-pc-windows-gnu","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:33:58Z","start":"2026-10-06T17:34:00Z","end":"2026-10-06T17:51:12Z","secs":1032,"exit":0,"compiles":1954,"sccache":{"hits":0,"misses":1886,"hits_total":0,"misses_total":1886},"load_end":[13.86,9.61,4.32],"artefacts":[{"path":"target/x86_64-pc-windows-gnu/release/igneumd.exe","bytes":52911104,"sha256":"0f3a9c2e1d8b7a6f5e4d3c2b1a0f9e8d7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f"},{"path":"target/x86_64-pc-windows-gnu/release/igneum-miner.exe","bytes":9811456,"sha256":"1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b"}]}
{"v":1,"id":"igneum-build-1-1759772000-37001","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow -j 90","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","slot":0,"wait_s":0,"start":"2026-10-06T17:53:20Z","end":"2026-10-06T18:06:41Z","secs":801,"exit":0,"compiles":1102,"sccache":{"hits":790,"misses":312,"hits_total":790,"misses_total":2198},"load_end":[61.2,40.1,22.0],"artefacts":[{"path":"target/release/igneumd","bytes":61120512,"sha256":"2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c"},{"path":"target/release/igneum-miner","bytes":11010048,"sha256":"3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d"}]}
{"v":1,"id":"igneum-build-1-1759773000-37100","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ship0314","crate":"app/igneum-app","command":"cargo build --release -j 90","branch":"release-0.3.14","sha":"952ce8f","label":"igneum-wt-ship0314/app/igneum-app cargo build --release","slot":0,"wait_s":412,"queued_at":"2026-10-06T18:03:10Z","start":"2026-10-06T18:10:02Z","end":"2026-10-06T18:13:48Z","secs":226,"exit":0,"compiles":388,"sccache":{"hits":301,"misses":87,"hits_total":1091,"misses_total":2285},"load_end":[70.3,55.8,30.2],"artefacts":[{"path":"target/release/igneum-app","bytes":30801920,"sha256":"4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e"}]}
{"v":1,"id":"igneum-build-1-1759773600-37200","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ca3-v4-node","crate":"vendor/igneum-node-v4","command":"cargo test --release -p kaspa-consensus-core --lib -j 90","branch":"ca3-v4-node","sha":"f39a8eb","label":"igneum-wt-ca3-v4-node/vendor/igneum-node-v4 cargo test --release -p kaspa-consensus-core --lib","slot":0,"wait_s":0,"start":"2026-10-06T18:14:00Z","end":"2026-10-06T18:19:33Z","secs":333,"exit":101,"compiles":210,"sccache":{"hits":180,"misses":30,"hits_total":1271,"misses_total":2315},"load_end":[55.0,50.2,33.1],"artefacts":[]}
not json at all
{"v":1,"id":"igneum-build-1-1759774200-37300","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-proving-v2","crate":"proving/igneum-prove","command":"cargo build --release -j 90","branch":"proving-v2","sha":"621f258","label":"igneum-wt-proving-v2/proving/igneum-prove cargo build --release","slot":0,"start":"2026-10-06T18:21:00Z","end":"2026-10-06T18:29:10Z","secs":490,"exit":0,"compiles":640,"sccache":{"hits":600,"misses":40,"hits_total":1871,"misses_total":2355},"load_end":[60.1,52.0,38.4],"artefacts":[{"path":"target/release/igneum-prove-host","bytes":80000000,"sha256":"5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f"}]}

1
tools/workers/fixtures/locks/build-0 Normal file
View file

@ -0,0 +1 @@
pid 36992 since 17:34:00Z waited 0 s: igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server

1
tools/workers/fixtures/locks/build-1 Normal file
View file

@ -0,0 +1 @@
pid 1 since 09:00:00Z waited 3 s: igneum-wt-old/app/igneum-app cargo build --release

0
tools/workers/fixtures/locks/build-2 Normal file
View file

1
tools/workers/fixtures/locks/slots Normal file
View file

@ -0,0 +1 @@
1

8
tools/workers/fixtures/proc-stat-0.txt Normal file
View file

@ -0,0 +1,8 @@
cpu 513633 175 77715 5794769 1804 0 128 0 0 0
cpu0 10000 0 1000 50000 10 0 1 0 0 0
cpu1 20000 0 2000 40000 10 0 1 0 0 0
cpu2 100 0 10 60000 10 0 1 0 0 0
cpu3 30000 0 500 30000 10 0 1 0 0 0
intr 1 2 3
ctxt 99
btime 1759772000

8
tools/workers/fixtures/proc-stat-1.txt Normal file
View file

@ -0,0 +1,8 @@
cpu 514633 175 77915 5795569 1804 0 128 0 0 0
cpu0 10100 0 1000 50000 10 0 1 0 0 0
cpu1 20000 0 2000 40100 10 0 1 0 0 0
cpu2 150 0 60 60000 10 0 1 0 0 0
cpu3 30025 0 500 30075 10 0 1 0 0 0
intr 1 2 3
ctxt 99
btime 1759772000

27
tools/workers/fixtures/sccache-stats.txt Normal file
View file

@ -0,0 +1,27 @@
Compile requests 2294
Compile requests executed 1954
Cache hits 790
Cache misses 1886
Cache misses (Assembler) 27
Cache misses (C/C++) 777
Cache misses (Rust) 1082
Cache hits rate 29.52 %
Cache timeouts 0
Cache read errors 0
Forced recaches 0
Cache write errors 0
Compilation failures 0
Cache errors 0
Non-cacheable compilations 0
Non-cacheable calls 340
Non-compilation calls 0
Unsupported compiler calls 0
Average cache write 0.002 s
Average compiler 1.624 s
Average cache read hit 0.001 s
Failed distributed compilations 0
Cache location Local disk: "/srv/sccache"
Use direct/preprocessor mode? yes
Version (client) 0.10.0
Cache size 12 GiB
Max cache size 100 GiB

1
tools/workers/fixtures/workers.demo.json Normal file
View file

File diff suppressed because one or more lines are too long

16
tools/workers/launchd/com.igneum.workers-push.plist Normal file
View file

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<!-- The worker dashboard's Mac pusher: tools/workers/push.mjs every 60 s. Installed by tools/workers/launchd/install.sh
into ~/Library/LaunchAgents (the __ROOT__ marker becomes the checkout path). Log: ~/Library/Logs/igneum-workers-push.log -->
<plist version="1.0"><dict>
<key>Label</key><string>com.igneum.workers-push</string>
<key>ProgramArguments</key><array><string>/opt/homebrew/bin/node</string><string>__ROOT__/tools/workers/push.mjs</string></array>
<key>StartInterval</key><integer>60</integer>
<key>RunAtLoad</key><true/>
<key>WorkingDirectory</key><string>__ROOT__</string>
<key>EnvironmentVariables</key><dict><key>PATH</key><string>/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin</string><key>HOME</key><string>__HOME__</string></dict>
<key>StandardOutPath</key><string>__HOME__/Library/Logs/igneum-workers-push.log</string>
<key>StandardErrorPath</key><string>__HOME__/Library/Logs/igneum-workers-push.log</string>
<key>Nice</key><integer>10</integer>
<key>ProcessType</key><string>Background</string>
</dict></plist>

19
tools/workers/launchd/install.sh Executable file
View file

@ -0,0 +1,19 @@
#!/usr/bin/env bash
# Install or refresh the Mac pusher's launchd agent (every 60 s). Run from any checkout: the plist points at the MAIN
# checkout's tools/workers/push.mjs (/Users/joshm/Projects/igneum) unless IGNEUM_WORKERS_ROOT names another, so the agent
# keeps working after this worktree is removed.
# tools/workers/launchd/install.sh install and (re)load
# tools/workers/launchd/install.sh --remove unload and delete
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT="${IGNEUM_WORKERS_ROOT:-/Users/joshm/Projects/igneum}"
LABEL=com.igneum.workers-push; DEST="$HOME/Library/LaunchAgents/$LABEL.plist"
if [ "${1:-}" = --remove ]; then launchctl bootout "gui/$(id -u)/$LABEL" 2>/dev/null || true; rm -f "$DEST"; echo "removed $LABEL"; exit 0; fi
[ -f "$ROOT/tools/workers/push.mjs" ] || { echo "no $ROOT/tools/workers/push.mjs (merge workers-dash to master first, or set IGNEUM_WORKERS_ROOT)" >&2; exit 1; }
mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs"
sed -e "s#__ROOT__#$ROOT#g" -e "s#__HOME__#$HOME#g" "$HERE/$LABEL.plist" > "$DEST"
plutil -lint "$DEST" >/dev/null
launchctl bootout "gui/$(id -u)/$LABEL" 2>/dev/null || true
launchctl bootstrap "gui/$(id -u)" "$DEST"
launchctl kickstart -k "gui/$(id -u)/$LABEL"
sleep 3; launchctl print "gui/$(id -u)/$LABEL" | grep -E 'state|last exit|runs' | head -4; tail -2 "$HOME/Library/Logs/igneum-workers-push.log" | sed 's/dl\/[A-Za-z0-9_-]\{8,\}/dl\/<token>/g'

221
tools/workers/lib.mjs Normal file
View file

@ -0,0 +1,221 @@
// Parsers shared by tools/workers/collect.mjs (on igneum-build-1) and tools/workers/push.mjs (on the Mac). Pure functions
// over text the machine gives us (/proc files, lock files, the builds JSONL), so node --test can run them against the
// fixtures in tools/workers/fixtures without a box. Every reader here takes the machine's own facts, never a reported number.
/// /proc/stat -> { total: [{user, nice, system, idle, iowait, irq, softirq, steal}], cores: [...] } (jiffies).
export function parseProcStat(text) {
const cores = []; let total = null;
for (const line of String(text).split('\n')) {
if (!line.startsWith('cpu')) continue;
const f = line.trim().split(/\s+/);
const n = f.slice(1, 9).map(Number);
const row = { user: n[0], nice: n[1], system: n[2], idle: n[3], iowait: n[4] || 0, irq: n[5] || 0, softirq: n[6] || 0, steal: n[7] || 0 };
if (f[0] === 'cpu') total = row; else cores.push(row);
}
return { total, cores };
}
/// Busy percentage between two /proc/stat samples: 0 to 100 for the whole box and per core, in core order.
export function cpuBusy(before, after) {
const pct = (a, b) => {
if (!a || !b) return 0;
const sum = r => r.user + r.nice + r.system + r.idle + r.iowait + r.irq + r.softirq + r.steal;
const idle = r => r.idle + r.iowait;
const dt = sum(b) - sum(a); const di = idle(b) - idle(a);
if (dt <= 0) return 0;
return Math.max(0, Math.min(100, Math.round((1 - di / dt) * 1000) / 10));
};
const n = Math.min(before.cores.length, after.cores.length);
const per = []; for (let i = 0; i < n; i++) per.push(pct(before.cores[i], after.cores[i]));
return { busy_pct: pct(before.total, after.total), per_core: per, cores_busy: per.filter(p => p >= 50).length };
}
/// /proc/meminfo -> kB numbers and a used percentage (used = total minus available, what a build can still take).
export function parseMeminfo(text) {
const m = {}; for (const line of String(text).split('\n')) { const r = /^(\w+):\s+(\d+)/.exec(line); if (r) m[r[1]] = Number(r[2]); }
const total = m.MemTotal || 0, avail = m.MemAvailable ?? m.MemFree ?? 0;
return { total_kb: total, available_kb: avail, used_kb: Math.max(0, total - avail), used_pct: total ? Math.round(((total - avail) / total) * 1000) / 10 : 0, swap_total_kb: m.SwapTotal || 0, swap_free_kb: m.SwapFree || 0 };
}
/// /proc/net/dev -> [{ iface, rx_bytes, tx_bytes }] without lo.
export function parseNetDev(text) {
const out = [];
for (const line of String(text).split('\n').slice(2)) {
const r = /^\s*([^:\s]+):\s*(.*)$/.exec(line); if (!r) continue;
const f = r[2].trim().split(/\s+/).map(Number);
if (r[1] === 'lo') continue;
out.push({ iface: r[1], rx_bytes: f[0] || 0, tx_bytes: f[8] || 0 });
}
return out;
}
/// /proc/loadavg -> { load: [1, 5, 15], running, threads }.
export function parseLoadavg(text) {
const f = String(text).trim().split(/\s+/);
const rq = (f[3] || '0/0').split('/');
return { load: [Number(f[0]) || 0, Number(f[1]) || 0, Number(f[2]) || 0], running: Number(rq[0]) || 0, threads: Number(rq[1]) || 0 };
}
/// The slot-file line both with-lock.sh and bs_remote_run write: "pid N since HH:MM:SSZ waited S s: <label>". Returns null
/// for an empty or foreign line. `day` (YYYY-MM-DD, UTC) anchors the clock time; a time later than `now` is yesterday.
export function parseSlotLine(line, { day, now } = {}) {
const r = /^pid (\d+) since (\d\d):(\d\d):(\d\d)Z waited (\d+) s: ?(.*)$/.exec(String(line || '').trim());
if (!r) return null;
const nowMs = now ? new Date(now).getTime() : Date.now();
const d = day || new Date(nowMs).toISOString().slice(0, 10);
let since = Date.parse(`${d}T${r[2]}:${r[3]}:${r[4]}Z`);
if (since > nowMs + 60_000) since -= 86_400_000;
const label = r[6].trim();
const agent = (/;\s*agent=([^;]+)\s*$/.exec(label) || [])[1] || null;
return { pid: Number(r[1]), since: new Date(since).toISOString().replace(/\.\d{3}Z$/, 'Z'), waited_s: Number(r[5]), label: agent ? label.replace(/;\s*agent=[^;]+\s*$/, '').trim() : label, agent, ...splitLabel(label) };
}
/// "igneum-wt-x/vendor/igneum-node-bs cross x86_64-pc-windows-gnu" or "igneum-wt-x/app/igneum-app cargo build --release ..."
/// -> { worktree, crate, command, target }. A Mac with-lock label is a plain command line: worktree and crate are null.
export function splitLabel(label) {
const s = String(label || '').replace(/;\s*agent=[^;]+\s*$/, '').trim();
const r = /^(igneum(?:-wt-[\w.-]+)?)\/(\S+) (cargo .*|cross .*)$/.exec(s) || /^(igneum(?:-wt-[\w.-]+)?)\/(\S+) (.*)$/.exec(s);
if (!r) return { worktree: null, crate: null, command: s || null, target: targetOf(s) };
let command = r[3];
if (command.startsWith('cross ')) command = `cargo build --release --target ${command.slice(6).trim()}`;
return { worktree: r[1], crate: r[2], command, target: targetOf(command) };
}
export function targetOf(command) {
const r = /--target[= ](\S+)/.exec(String(command || ''));
if (r) return r[1];
return /\bcargo\b/.test(String(command || '')) ? 'x86_64-unknown-linux-gnu' : null;
}
/// The build's kind, from the builds-log fields or a slot label: node-linux, node-windows, app, app-windows, prove, suite,
/// check, other. The same rule as the JSONL proposal, so both sides agree.
export function deriveKind({ kind, command, crate, tool, target } = {}) {
if (kind) return kind;
const cmd = String(command || ''); const cr = String(crate || '');
const sub = (/\bcargo\s+(\S+)/.exec(cmd) || [])[1] || cmd.split(/\s+/)[0] || '';
const tgt = target || targetOf(cmd);
const win = tgt === 'x86_64-pc-windows-gnu' || tool === 'cross-remote';
if (sub === 'test' || sub === 'nextest') return 'suite';
if (sub === 'check' || sub === 'clippy') return 'check';
if (/^vendor\//.test(cr) || /\bkaspad\b/.test(cmd)) return win ? 'node-windows' : 'node-linux';
if (/igneum-app/.test(cr)) return win ? 'app-windows' : 'app';
if (/igneum-prove/.test(cr) || /\bprove\b/.test(cr)) return 'prove';
if (/igneum-pow/.test(cr)) return 'pow';
return 'other';
}
/// A lock directory listing -> { count, held: [...], free: [...] }. `files` is [{ name, text, held }] where `held` says whether
/// the flock is taken (the caller knows: flock probe on the box, pid alive on the Mac). Stale lines without a live lock are free.
export function parseLockDir(files, { day, now, aliveFn } = {}) {
const held = [], free = [];
let count = 1;
for (const f of files) {
if (f.name === 'slots') { const n = parseInt(f.text, 10); if (n >= 1) count = n; continue; }
if (!/^(build-\d+|run-\d+|build|measure)$/.test(f.name)) continue;
const line = parseSlotLine((f.text || '').split('\n')[0], { day, now });
const alive = line && aliveFn ? aliveFn(line.pid) : null;
const isHeld = f.held ?? (line ? alive !== false : false);
if (line && isHeld) held.push({ slot: f.name, ...line, kind: deriveKind(line) });
else free.push(f.name);
}
held.sort((a, b) => a.slot.localeCompare(b.slot));
return { count, held, free: free.sort() };
}
/// The builds JSONL (one object per line, the format sent to the build-server agent). Bad lines are skipped and counted.
export function parseBuildsJsonl(text, { limit = 200 } = {}) {
const rows = []; let bad = 0;
for (const line of String(text).split('\n')) {
const t = line.trim(); if (!t) continue;
try { const o = JSON.parse(t); if (o && typeof o === 'object') rows.push(normaliseBuild(o)); else bad++; } catch { bad++; }
}
rows.sort((a, b) => String(b.end || b.start || '').localeCompare(String(a.end || a.start || '')));
return { rows: rows.slice(0, limit), bad, total: rows.length };
}
export function normaliseBuild(o) {
const start = o.start || null, end = o.end || null;
const secs = Number.isFinite(o.secs) ? o.secs : (start && end ? Math.round((Date.parse(end) - Date.parse(start)) / 1000) : null);
const sc = o.sccache || {};
const hits = Number(sc.hits ?? 0), misses = Number(sc.misses ?? 0);
return {
...o,
id: o.id || `${o.host || 'box'}-${start || ''}-${o.pid || ''}`,
kind: deriveKind(o),
target: o.target || targetOf(o.command),
secs,
ok: Number(o.exit) === 0,
sccache: { hits, misses, hit_rate_pct: hits + misses ? Math.round((hits / (hits + misses)) * 1000) / 10 : null, hits_total: sc.hits_total ?? null, misses_total: sc.misses_total ?? null },
artefacts: Array.isArray(o.artefacts) ? o.artefacts : [],
};
}
/// `sccache --show-stats` text -> { requests, executed, hits, misses, hit_rate_pct, cache_size, max_size }.
export function parseSccacheStats(text) {
const num = re => { const r = re.exec(String(text)); return r ? Number(r[1].replace(/,/g, '')) : null; };
const str = re => { const r = re.exec(String(text)); return r ? r[1].trim() : null; };
const hits = num(/^Cache hits\s{2,}(\d[\d,]*)/m), misses = num(/^Cache misses\s{2,}(\d[\d,]*)/m);
return {
requests: num(/^Compile requests\s+(\d[\d,]*)/m), executed: num(/^Compile requests executed\s+(\d[\d,]*)/m),
hits, misses, hit_rate_pct: hits !== null && misses !== null && hits + misses ? Math.round((hits / (hits + misses)) * 1000) / 10 : (hits === 0 && misses ? 0 : null),
cache_size: str(/^Cache size\s+(.+)$/m), max_size: str(/^Max cache size\s+(.+)$/m),
};
}
/// /sys/class/hwmon readings -> [{ name, label, c }] for the sensors worth a tile: the CPU die, the NVMe drives, the NIC.
export function pickTemps(sensors) {
const out = [];
for (const s of sensors) {
const c = Number(s.millic) / 1000;
if (!Number.isFinite(c) || c <= 0 || c > 150) continue;
const name = s.name === 'k10temp' || s.name === 'coretemp' ? 'cpu' : s.name === 'nvme' ? 'nvme' : s.name.startsWith('bnxt') || s.name.includes('eth') ? 'nic' : s.name;
out.push({ name, label: s.label || s.name, c: Math.round(c * 10) / 10 });
}
// one per name: the hottest (two NVMe drives in RAID 1, two NIC ports)
const best = new Map();
for (const t of out) { const cur = best.get(t.name); if (!cur || t.c > cur.c) best.set(t.name, t); }
const order = ['cpu', 'nvme', 'nic'];
return [...best.values()].sort((a, b) => (order.indexOf(a.name) + 99) % 99 - (order.indexOf(b.name) + 99) % 99 || a.name.localeCompare(b.name));
}
/// A cargo process -> a running build: { pid, cwd, worktree, crate, command, target, kind, started_at, elapsed_s }.
/// `stat` is /proc/<pid>/stat text, `bootSec` the boot time in unix seconds, `clk` CLK_TCK.
export function cargoProcess({ pid, cwd, cmdline, stat, bootSec, clk = 100, now = Date.now(), root = '/srv/builds' }) {
const args = String(cmdline || '').split('\0').filter(Boolean);
if (!args.length || !/(^|\/)cargo$/.test(args[0])) return null;
const command = ['cargo', ...args.slice(1)].join(' ');
const rest = stat ? stat.slice(stat.lastIndexOf(')') + 2).split(' ') : [];
const startTicks = Number(rest[19]); // field 22 of /proc/<pid>/stat, after pid and comm
const started = Number.isFinite(startTicks) && bootSec ? (bootSec + startTicks / clk) * 1000 : null;
const rel = String(cwd || '').startsWith(root + '/') ? cwd.slice(root.length + 1) : null;
const worktree = rel ? rel.split('/')[0] : null;
const crate = rel ? rel.split('/').slice(1).join('/') || '.' : null;
const target = targetOf(command);
return { pid, cwd, worktree, crate, command, target, kind: deriveKind({ command, crate, target }), started_at: started ? new Date(started).toISOString().replace(/\.\d{3}Z$/, 'Z') : null, elapsed_s: started ? Math.max(0, Math.round((now - started) / 1000)) : null };
}
/// The relay job SUMMARY line and closing line of one report (as tools/jobs.mjs reads them) -> { job, kind, status, exit,
/// duration_s, started_at, finished_at, summary, errors[] } or null.
export function summaryOf(lines) {
const ls = String(lines || '').split('\n');
let s = null;
for (const l of ls) { if (l.startsWith('SUMMARY ')) { try { s = JSON.parse(l.slice(8)); } catch { s = null; } break; } }
const close = ls.slice(-40).map(l => /^job (\S+): (done|failed|timeout|aborted) \(exit (-?\d+)\) after (\d+) s: (.*)$/.exec(l)).filter(Boolean).pop();
if (close) { s = s || {}; s.job = s.job || close[1]; s.status = close[2]; s.exit = Number(close[3]); s.duration_s = Number(close[4]); s.summary = close[5]; s.finished_at = s.finished_at || 'closed'; }
else if (s && !s.finished_at) s.status = 'running';
if (s) {
s.errors = ls.filter(l => /^(Error:|BUILD FAILED|GATE FAILED)|\bFAILED\b|panicked at/.test(l) && !l.startsWith('SUMMARY ')).slice(0, 5);
s.stage = (ls.map(l => /^STAGE (\S+) start/.exec(l)).filter(Boolean).pop() || [])[1] || null;
}
return s;
}
/// Duration bars want the same scale: the longest of the lot, at least a minute.
export const fmtSecs = s => { s = Math.max(0, Math.round(Number(s) || 0)); const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), r = s % 60; return h ? `${h} h ${String(m).padStart(2, '0')} min` : m ? `${m} min ${String(r).padStart(2, '0')} s` : `${r} s`; };
/// Mac baselines for the kind comparison: approximate, from the docs that measured them; a measurement on the same tree
/// replaces them when one is logged. Shared by collect.mjs (the box's file) and push.mjs (the edge copy).
export const BASELINES = {
'node-linux': { secs: 690, source: 'approximate: infra/cross/build-linux.sh on this Mac, 8 to 15 min per target at nice 19 with 4 to 6 jobs (docs/plans/build-job.md, 4 Oct 2026)' },
'node-windows': { secs: 690, source: 'approximate: proto-cuda/windows-node/cross-build.sh on this Mac, the same 8 to 15 min (docs/plans/build-job.md, 4 Oct 2026)' },
};

366
tools/workers/page/workers.html Normal file
View file

@ -0,0 +1,366 @@
<!doctype html>
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover"><meta name="robots" content="noindex">
<title>Igneum Workers</title>
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Unbounded:wght@500;700&family=IBM+Plex+Mono:wght@400;500;600&family=IBM+Plex+Sans:wght@400;500;600&display=swap">
<style>
:root {
--bg: #0c0c0e; --panel: #141418; --panel-2: #1b1b21; --ink: #f1ede4; --muted: #8e8b86; --line: #26262d;
--ember: #f2541b; --ember-2: #ffb15a; --good: #5fd08a; --warn: #f2c14e; --bad: #ff6b5e; --cool: #6aa9ff;
--display: "Unbounded", "IBM Plex Sans", -apple-system, sans-serif;
--mono: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace;
--sans: "IBM Plex Sans", -apple-system, "Segoe UI", Helvetica, Arial, sans-serif;
color-scheme: dark;
}
* { box-sizing: border-box; }
body { background: var(--bg); color: var(--ink); font-family: var(--sans); font-size: 14px; line-height: 1.45; margin: 0; }
.wrap { max-width: 1180px; margin: 0 auto; padding-inline: 16px; padding-block: 18px 48px; }
header { display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 10px 20px; }
.brand { display: flex; align-items: center; gap: 12px; }
.brand svg { width: 34px; height: 34px; }
.brand h1 { font-family: var(--display); font-weight: 700; font-size: 20px; letter-spacing: 0.02em; margin: 0; }
.brand h1 small { font-family: var(--mono); font-weight: 400; font-size: 11px; color: var(--muted); letter-spacing: 0.1em; text-transform: uppercase; margin-left: 10px; }
nav { display: flex; gap: 2px; border: 1px solid var(--line); }
nav a { font-family: var(--mono); font-size: 11px; letter-spacing: 0.1em; text-transform: uppercase; color: var(--muted); text-decoration: none; padding: 7px 12px; }
nav a.on { color: var(--ink); background: var(--panel-2); }
nav a:hover { color: var(--ember-2); }
.live { font-family: var(--mono); font-size: 12px; color: var(--muted); display: flex; align-items: center; gap: 8px; }
.live b { width: 8px; height: 8px; border-radius: 50%; background: var(--good); box-shadow: 0 0 0 0 rgba(95,208,138,.6); animation: ping 2.4s infinite; }
.live.stale b { background: var(--warn); animation: none; }
.live.down b { background: var(--bad); animation: none; }
@keyframes ping { 0% { box-shadow: 0 0 0 0 rgba(95,208,138,.55); } 70% { box-shadow: 0 0 0 9px rgba(95,208,138,0); } 100% { box-shadow: 0 0 0 0 rgba(95,208,138,0); } }
h2 { font-family: var(--display); font-weight: 500; font-size: 13px; letter-spacing: 0.08em; text-transform: uppercase; color: var(--muted); margin: 32px 0 12px; display: flex; align-items: baseline; gap: 12px; flex-wrap: wrap; }
h2 span { font-family: var(--mono); font-size: 11px; letter-spacing: 0; text-transform: none; color: var(--muted); }
.empty { padding: 18px; color: var(--muted); border: 1px dashed var(--line); font-family: var(--mono); font-size: 12px; line-height: 1.6; }
.pill { display: inline-block; font-family: var(--mono); font-size: 10px; letter-spacing: 0.06em; text-transform: uppercase; padding: 3px 7px; border: 1px solid var(--line); color: var(--muted); white-space: nowrap; }
.pill.running, .pill.done, .pill.ok { border-color: var(--good); color: var(--good); }
.pill.failed, .pill.timeout, .pill.aborted, .pill.error { border-color: var(--bad); color: var(--bad); }
.pill.waiting, .pill.queued, .pill.stale { border-color: var(--warn); color: var(--warn); }
.pill.kind { border-color: #3a2a22; color: var(--ember-2); }
time[title] { cursor: help; border-bottom: 1px dotted var(--line); }
/* the server */
.server { margin-top: 18px; background: var(--panel); border: 1px solid var(--line); position: relative; overflow: hidden; }
.server.hot::after { content: ""; position: absolute; inset: auto -30% -70% -30%; height: 90%; background: radial-gradient(ellipse at center, rgba(242,84,27,.18), transparent 60%); pointer-events: none; animation: breathe 3.2s ease-in-out infinite; }
@keyframes breathe { 0%, 100% { opacity: .5; } 50% { opacity: 1; } }
.server .head { display: flex; flex-wrap: wrap; justify-content: space-between; align-items: flex-end; gap: 10px 20px; padding: 16px 18px 0; }
.server .name { font-family: var(--display); font-weight: 700; font-size: 22px; letter-spacing: 0.01em; }
.server .name small { font-family: var(--mono); font-weight: 400; font-size: 11px; color: var(--muted); margin-left: 10px; letter-spacing: 0.08em; text-transform: uppercase; }
.server .facts { font-family: var(--mono); font-size: 11px; color: var(--muted); display: flex; flex-wrap: wrap; gap: 4px 16px; }
.server .facts b { color: var(--ink); font-weight: 500; }
.cores { display: grid; grid-template-columns: repeat(48, 1fr); gap: 2px; padding: 14px 18px 0; height: 78px; align-items: end; position: relative; z-index: 1; }
.cores i { display: block; height: 100%; background: var(--panel-2); position: relative; overflow: hidden; }
.cores i::after { content: ""; position: absolute; left: 0; right: 0; bottom: 0; height: var(--h, 2%); min-height: 2px; background: linear-gradient(180deg, var(--ember-2), var(--ember)); transition: height .9s cubic-bezier(.2,.8,.2,1); }
.cores i.hot::after { animation: flicker 1.3s ease-in-out infinite alternate; }
@keyframes flicker { from { filter: brightness(.85); } to { filter: brightness(1.15); } }
.corelabel { display: flex; justify-content: space-between; flex-wrap: wrap; gap: 2px 12px; font-family: var(--mono); font-size: 10px; color: var(--muted); padding: 6px 18px 0; letter-spacing: 0.08em; text-transform: uppercase; }
.gauges { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 1px; background: var(--line); border-top: 1px solid var(--line); margin-top: 14px; position: relative; z-index: 1; }
@media (min-width: 900px) { .gauges { grid-template-columns: repeat(5, 1fr); } }
.g { background: var(--panel); padding: 12px 14px; display: flex; gap: 12px; align-items: center; }
.g svg { width: 56px; height: 56px; flex: none; }
.g .ring { fill: none; stroke: var(--panel-2); stroke-width: 6; }
.g .arc { fill: none; stroke: var(--ember); stroke-width: 6; stroke-linecap: round; transform: rotate(-90deg); transform-origin: 50% 50%; transition: stroke-dasharray .9s ease; }
.g .arc.good { stroke: var(--good); } .g .arc.warn { stroke: var(--warn); } .g .arc.bad { stroke: var(--bad); } .g .arc.cool { stroke: var(--cool); }
.g .l { font-size: 10px; letter-spacing: 0.12em; text-transform: uppercase; color: var(--muted); }
.g .v { font-family: var(--display); font-weight: 500; font-size: 18px; margin-top: 2px; font-variant-numeric: tabular-nums; line-height: 1.1; }
.g .s { font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 3px; }
/* the other workers */
.crew { display: grid; grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); gap: 12px; }
.w { background: var(--panel); border: 1px solid var(--line); padding: 12px 14px; }
.w .top { display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; }
.w .card { font-family: var(--display); font-weight: 500; font-size: 13px; }
.w .meta { font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 3px; }
.w .doing { font-size: 12px; margin-top: 8px; min-height: 2.6em; }
.w .foot { display: flex; justify-content: space-between; font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 8px; border-top: 1px solid var(--line); padding-top: 8px; }
.slots { display: flex; gap: 4px; margin-top: 8px; }
.slots i { height: 6px; flex: 1; background: var(--panel-2); }
.slots i.held { background: linear-gradient(90deg, var(--ember), var(--ember-2)); }
.slots i.run { background: var(--cool); }
.slots i.measure { background: var(--warn); }
/* lanes */
.lane { display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); gap: 12px; }
.job { background: var(--panel); border: 1px solid var(--line); padding: 12px 14px; position: relative; overflow: hidden; }
.job.running { border-color: #3a2a22; box-shadow: inset 0 0 0 1px rgba(242,84,27,.25); }
.job.running::before { content: ""; position: absolute; left: 0; top: 0; bottom: 0; width: 3px; background: linear-gradient(180deg, var(--ember-2), var(--ember)); }
.job .top { display: flex; justify-content: space-between; align-items: flex-start; gap: 8px; }
.job .title { font-family: var(--display); font-weight: 500; font-size: 13px; line-height: 1.3; word-break: break-word; }
.job .where { font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 3px; }
.job .cmd { font-family: var(--mono); font-size: 11px; color: var(--ink); margin-top: 8px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
.job .foot { display: flex; justify-content: space-between; align-items: baseline; font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 10px; border-top: 1px solid var(--line); padding-top: 8px; gap: 8px; }
.job .elapsed { font-family: var(--display); font-weight: 500; font-size: 18px; color: var(--ember-2); font-variant-numeric: tabular-nums; }
.job .pulse { position: absolute; right: -40px; bottom: -40px; width: 120px; height: 120px; border-radius: 50%; background: radial-gradient(circle, rgba(242,84,27,.22), transparent 65%); animation: breathe 2.4s ease-in-out infinite; pointer-events: none; }
/* timeline */
.tl { background: var(--panel); border: 1px solid var(--line); }
.tl .row { display: grid; grid-template-columns: 64px 1fr 150px 92px; gap: 10px; align-items: center; padding: 9px 12px; border-bottom: 1px solid var(--line); cursor: pointer; }
.tl .row:hover { background: var(--panel-2); }
.tl .row:last-child, .tl .row.open { border-bottom-color: transparent; }
.tl .at { font-family: var(--mono); font-size: 11px; color: var(--muted); }
.tl .what { min-width: 0; }
.tl .what .t { font-size: 13px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
.tl .what .m { font-family: var(--mono); font-size: 11px; color: var(--muted); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
.tl .bar { height: 8px; background: var(--panel-2); position: relative; }
.tl .bar i { position: absolute; left: 0; top: 0; bottom: 0; background: linear-gradient(90deg, var(--good), #8ee3ad); }
.tl .bar i.failed { background: linear-gradient(90deg, var(--bad), #ff9b92); }
.tl .dur { font-family: var(--mono); font-size: 12px; text-align: right; font-variant-numeric: tabular-nums; }
.tl .lines { display: none; font-family: var(--mono); font-size: 11px; color: var(--muted); padding: 2px 12px 12px 86px; border-bottom: 1px solid var(--line); white-space: pre-wrap; word-break: break-all; line-height: 1.6; }
.tl .lines.show { display: block; }
.tl .lines b { color: var(--ink); font-weight: 500; }
@media (max-width: 640px) { .tl .row { grid-template-columns: 52px 1fr 70px; } .tl .bar { display: none; } .tl .lines { padding-left: 12px; } }
/* analytics */
.an { display: grid; grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); gap: 12px; }
.an .a { background: var(--panel); border: 1px solid var(--line); padding: 12px 14px; min-width: 0; }
.an .l { font-size: 10px; letter-spacing: 0.12em; text-transform: uppercase; color: var(--muted); display: flex; justify-content: space-between; gap: 8px; }
.an .l span { letter-spacing: 0; text-transform: none; font-family: var(--mono); }
.an svg { width: 100%; height: 96px; display: block; margin-top: 10px; }
.an table { border-collapse: collapse; width: 100%; font-size: 12px; margin-top: 8px; }
.an th, .an td { text-align: left; padding: 6px 6px; border-bottom: 1px solid var(--line); white-space: nowrap; }
.an th { font-size: 10px; text-transform: uppercase; letter-spacing: 0.1em; color: var(--muted); font-weight: 600; }
.an td.num, .an th.num { text-align: right; font-family: var(--mono); font-variant-numeric: tabular-nums; }
.an tr:last-child td { border-bottom: 0; }
.an .note { font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 8px; line-height: 1.5; }
.head3 { display: grid; grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); gap: 1px; background: var(--line); border: 1px solid var(--line); }
.h3 { background: var(--panel); padding: 14px 16px; }
.h3 .l { font-size: 10px; letter-spacing: 0.12em; text-transform: uppercase; color: var(--muted); }
.h3 .v { font-family: var(--display); font-weight: 500; font-size: 26px; margin-top: 6px; font-variant-numeric: tabular-nums; color: var(--ember-2); }
.h3 .v small { font-size: 12px; color: var(--muted); font-family: var(--mono); font-weight: 400; margin-left: 8px; }
.h3 .cmp { display: flex; gap: 14px; font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 6px; flex-wrap: wrap; }
.h3 .cmp b { color: var(--ink); font-weight: 500; }
.h3 .s { font-size: 11px; color: var(--muted); margin-top: 8px; line-height: 1.5; }
.h3 .meter { height: 4px; background: var(--panel-2); margin-top: 10px; position: relative; }
.h3 .meter i { display: block; height: 100%; background: linear-gradient(90deg, var(--ember), var(--ember-2)); transition: width .6s ease; }
.h3 .meter em { position: absolute; top: -3px; width: 2px; height: 10px; background: var(--muted); }
.sources { font-family: var(--mono); font-size: 11px; color: var(--muted); margin-top: 32px; border-top: 1px solid var(--line); padding-top: 10px; display: flex; flex-wrap: wrap; gap: 6px 18px; }
.sources b { color: var(--good); font-weight: 500; } .sources b.no { color: var(--bad); } .sources b.warn { color: var(--warn); }
a:focus-visible, button:focus-visible { outline: 2px solid var(--ember-2); outline-offset: 2px; }
@media (prefers-reduced-motion: reduce) { .live b, .server.hot::after, .job .pulse, .cores i.hot::after { animation: none !important; } .cores i::after, .g .arc { transition: none; } }
</style></head><body>
<div class="wrap">
<header>
<div class="brand">
<svg viewBox="0 0 100 100" aria-hidden="true"><polygon points="50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34" fill="#F2541B"/><polygon points="50,42 59,58 50,82 41,58" fill="#0C0C0E"/></svg>
<h1>Igneum Workers<small>build shop</small></h1>
</div>
<nav aria-label="Pages"><a href="./">Fleet</a><a href="workers.html" class="on" aria-current="page">Workers</a></nav>
<div class="live" id="live"><b></b><span id="stamp">waiting for the first write</span></div>
</header>
<section class="server" id="server" aria-label="The build server">
<div class="head">
<div><div class="name" id="s-name">igneum-build-1<small>build server</small></div><div class="facts" id="s-facts"></div></div>
<div class="facts" id="s-right"></div>
</div>
<div class="cores" id="cores" aria-label="96 cores, busy share per core"></div>
<div class="corelabel"><span id="cores-l">cores</span><span id="cores-r"></span></div>
<div class="gauges" id="gauges"></div>
</section>
<h2>Workers <span>every machine that builds, tests or measures</span></h2>
<div class="crew" id="crew"></div>
<h2>Now building <span id="now-sub"></span></h2>
<div class="lane" id="now"></div>
<h2>Queue <span id="queue-sub"></span></h2>
<div class="lane" id="queue"></div>
<h2>Recently done <span>click a row for the closing lines</span></h2>
<div class="tl" id="done"></div>
<h2>Headline timings <span>what the box does against the Mac and PC 1 on the same tree</span></h2>
<div class="head3" id="headline"></div>
<h2>Analytics <span id="an-sub"></span></h2>
<div class="an" id="an"></div>
<div class="sources" id="sources"></div>
</div>
<script type="module">
import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, kindLabel, fmtDur, fmtDurShort, fmtBytes, fmtBps, fmtUptime, ukTime, ago, headline } from './shape.js?v=1';
const $ = id => document.getElementById(id);
const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));
const pill = (s, cls) => `<span class="pill ${esc(cls || String(s || '').toLowerCase())}">${esc(s || '?')}</span>`;
const t = (isoStr, withDate) => { const u = ukTime(isoStr, { withDate }); return u.text ? `<time title="${esc(u.utc)}" datetime="${esc(isoStr)}">${esc(u.text)}</time>` : ''; };
const base = location.pathname.replace(/\/workers\.html$/, '').replace(/\/$/, '');
const demo = new URLSearchParams(location.search).get('demo');
let D = null, open = new Set();
function arc(pct, cls) {
const r = 24, c = 2 * Math.PI * r, p = Math.max(0, Math.min(100, Number(pct) || 0));
return `<svg viewBox="0 0 60 60" aria-hidden="true"><circle class="ring" cx="30" cy="30" r="${r}"/><circle class="arc ${cls || ''}" cx="30" cy="30" r="${r}" stroke-dasharray="${(c * p / 100).toFixed(1)} ${c.toFixed(1)}"/></svg>`;
}
const tone = (pct, warn = 70, bad = 90) => pct >= bad ? 'bad' : pct >= warn ? 'warn' : '';
function renderServer(b, src) {
const el = $('server');
if (!b) {
$('s-facts').innerHTML = `<span>${esc(src && src.error || 'no box facts yet')}</span>`; $('s-right').innerHTML = '';
$('cores').innerHTML = Array.from({ length: 96 }, () => '<i style="--h:2%"></i>').join(''); $('cores-r').textContent = 'no sample'; $('gauges').innerHTML = ''; el.classList.remove('hot'); return;
}
$('s-name').innerHTML = `${esc(b.name)}<small>${esc(b.os || 'build server')}</small>`;
$('s-facts').innerHTML = [`<b>${b.cores}</b> threads`, `<b>${esc(fmtUptime(b.uptime_s))}</b> up`, `load <b>${(b.load || []).map(x => Number(x).toFixed(1)).join(' / ')}</b>`, b.kernel ? `kernel <b>${esc(b.kernel)}</b>` : ''].filter(Boolean).map(x => `<span>${x}</span>`).join('');
const temps = (b.temps || []).map(x => `${esc(x.name)} <b>${x.c.toFixed(0)}°</b>`);
$('s-right').innerHTML = [...temps, b.net ? `net ↓<b>${esc(fmtBps(b.net.rx_bps))}</b> ↑<b>${esc(fmtBps(b.net.tx_bps))}</b>` : '', `<span title="${esc(b.collected_at)}">read ${esc(ago(b.collected_at))}</span>`].filter(Boolean).map(x => `<span>${x}</span>`).join('');
const per = (b.cpu && b.cpu.per_core) || [];
const n = Math.max(per.length, b.cores || 96);
$('cores').style.gridTemplateColumns = `repeat(${n > 64 ? 48 : n > 32 ? 32 : n}, 1fr)`;
$('cores').innerHTML = Array.from({ length: n }, (_, i) => { const p = per[i] ?? 0; return `<i class="${p >= 50 ? 'hot' : ''}" style="--h:${Math.max(2, p)}%" title="core ${i}: ${p}%"></i>`; }).join('');
$('cores-l').textContent = `${n} cores, ${b.cpu ? b.cpu.cores_busy : 0} busy`;
$('cores-r').textContent = b.cpu ? `${b.cpu.busy_pct}% of the box over the last second, ${b.compilers || 0} compilers running` : '';
el.classList.toggle('hot', !!(b.running && b.running.length));
const m = b.mem || {}, d = b.disk || {}, sc = b.sccache;
const hit = sc && sc.hit_rate_pct !== null && sc.hit_rate_pct !== undefined ? sc.hit_rate_pct : null;
$('gauges').innerHTML = [
`<div class="g">${arc(b.cpu ? b.cpu.busy_pct : 0, tone(b.cpu ? b.cpu.busy_pct : 0, 101, 101))}<div><div class="l">CPU</div><div class="v">${b.cpu ? b.cpu.busy_pct : 0}%</div><div class="s">load 1 min ${b.load ? Number(b.load[0]).toFixed(1) : '?'} of ${b.cores}</div></div></div>`,
`<div class="g">${arc(m.used_pct, tone(m.used_pct))}<div><div class="l">Memory</div><div class="v">${m.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes((m.used_kb || 0) * 1024))} of ${esc(fmtBytes((m.total_kb || 0) * 1024))}</div></div></div>`,
`<div class="g">${arc(d.used_pct, tone(d.used_pct, 80, 92))}<div><div class="l">Disk ${esc(d.mount || '/srv')}</div><div class="v">${d.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes(d.used_bytes))} used, ${esc(fmtBytes(d.avail_bytes))} free</div></div></div>`,
`<div class="g">${arc(hit ?? 0, 'cool')}<div><div class="l">sccache hits</div><div class="v">${hit === null ? '–' : hit + '%'}</div><div class="s">${sc ? `${sc.hits ?? 0} hit, ${sc.misses ?? 0} miss, ${esc(sc.cache_size || '?')} of ${esc(sc.max_size || '?')}` : 'sccache not answering'}</div></div></div>`,
`<div class="g">${arc(b.slots ? (b.slots.held.length / Math.max(1, b.slots.count)) * 100 : 0, 'good')}<div><div class="l">Build slots</div><div class="v">${b.slots ? `${b.slots.held.length}/${b.slots.count}` : '–'}</div><div class="s">${b.queue && b.queue.length ? `${b.queue.length} waiting` : 'nobody waiting'}</div></div></div>`,
].join('');
}
function slotBar(slots, count) {
const held = new Map((slots && slots.held || []).map(h => [h.slot, h]));
const names = Array.from({ length: count }, (_, i) => `build-${i}`);
return `<div class="slots">${names.map(nm => `<i class="${held.has(nm) ? 'held' : ''}" title="${esc(nm)}${held.has(nm) ? ': ' + esc(held.get(nm).label) : ': free'}"></i>`).join('')}</div>`;
}
function renderCrew() {
const b = D.box, mac = D.mac, pcs = (D.pcs && D.pcs.machines) || [];
const cards = [];
if (b) cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(b.name)}</div><div class="meta">Hetzner, ${b.cores} threads, ${esc(fmtBytes((b.mem && b.mem.total_kb || 0) * 1024))}, RAID 1 NVMe</div></div>${pill(b.running && b.running.length ? 'building' : 'idle', b.running && b.running.length ? 'running' : '')}</div><div class="doing">${b.running && b.running.length ? b.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('<br>') : 'Slot free. The next build-remote.sh or cross-remote.sh from any worktree takes it.'}</div>${slotBar(b.slots, b.slots ? b.slots.count : 1)}<div class="foot"><span>${b.recent ? b.recent.length : 0} builds logged</span><span>${esc(ago(b.collected_at))}</span></div></div>`);
else cards.push(`<div class="w"><div class="top"><div><div class="card">igneum-build-1</div><div class="meta">build server</div></div>${pill('unreachable', 'error')}</div><div class="doing">${esc(D.sources && D.sources.box && D.sources.box.error || 'no facts from the box')}</div></div>`);
if (mac) {
const held = mac.slots.held || [], runs = held.filter(h => /^run-/.test(h.slot)), builds = held.filter(h => /^build-/.test(h.slot)), meas = held.filter(h => h.slot === 'measure');
const doing = held.length ? held.map(h => `${esc(h.slot)}: ${esc((h.worktree ? h.worktree + ' ' : '') + (h.command || h.label || ''))}`.slice(0, 140)).join('<br>') : 'Nothing under with-lock.sh. macOS binaries and the DMG build here; everything else goes to the box or the PCs.';
cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(mac.name)}</div><div class="meta">this Mac, ${mac.build_slots} build slot${mac.build_slots === 1 ? '' : 's'} open, 3 run slots</div></div>${pill(meas.length ? 'measuring' : builds.length ? 'building' : runs.length ? 'running' : 'idle', held.length ? 'running' : '')}</div><div class="doing">${doing}</div><div class="slots">${['build-0', 'build-1', 'build-2', 'run-0', 'run-1', 'run-2', 'measure'].map(nm => { const h = held.find(x => x.slot === nm); return `<i class="${h ? (nm === 'measure' ? 'measure' : nm.startsWith('run') ? 'run' : 'held') : ''}" title="${esc(nm)}${h ? ': ' + esc(h.label) : ': free'}"></i>`; }).join('')}</div><div class="foot"><span>${mac.queue.length ? mac.queue.length + ' waiting for a slot' : 'nobody waiting'}</span><span>${esc(ago(mac.collected_at))}</span></div></div>`);
} else cards.push(`<div class="w"><div class="top"><div><div class="card">MacBook-Pro</div><div class="meta">this Mac</div></div>${pill('no push', 'error')}</div><div class="doing">${esc(D.sources && D.sources.mac && D.sources.mac.error || 'the Mac pusher has not written yet (launchd com.igneum.workers-push, every 60 s)')}</div></div>`);
for (const pc of pcs) {
const r = pc.running;
cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(pc.name)}</div><div class="meta">${esc(pc.machine || pc.id)} · ${esc(pc.role)}</div></div>${pill(r ? 'running' : pc.queue.length ? 'queued' : 'idle', r ? 'running' : pc.queue.length ? 'queued' : '')}</div><div class="doing">${r ? `${esc(r.kind)} ${esc(r.job)}${r.title ? '<br>' + esc(r.title) : ''}${r.stage ? `<br><span class="pill">stage ${esc(r.stage)}</span>` : ''}` : pc.queue.length ? `Next: ${esc(pc.queue[0].kind)} ${esc(pc.queue[0].job)}${pc.queue[0].title ? ', ' + esc(pc.queue[0].title) : ''}` : esc(pc.note || 'idle on jobs; the relay polls every 10 min')}</div><div class="foot"><span>${pc.recent.length} reports kept</span><span>${pc.last_report_at ? 'last report ' + esc(ago(pc.last_report_at)) : 'no report in 7 days'}</span></div></div>`);
}
if (!pcs.length) cards.push(`<div class="w"><div class="top"><div><div class="card">PC 1 and PC 2</div><div class="meta">relay jobs</div></div>${pill('no data', 'error')}</div><div class="doing">${esc(D.sources && D.sources.pcs && D.sources.pcs.error || 'the Mac pusher reads the relay intake; nothing has arrived')}</div></div>`);
$('crew').innerHTML = cards.join('');
}
function nowItems() {
const out = [];
for (const r of (D.box && D.box.running) || []) out.push({ where: D.box.name, slot: r.slot, title: `${kindLabel(r.kind)}`, sub: `${r.worktree || '?'}/${r.crate || ''}${r.agent ? ' · ' + r.agent : ''}`, cmd: r.command, started_at: r.started_at, kind: r.kind, extra: r.target || '' });
for (const h of (D.mac && D.mac.slots.held) || []) out.push({ where: D.mac.name, slot: h.slot, title: h.slot === 'measure' ? 'measurement' : h.slot.startsWith('run') ? 'functional run' : kindLabel(h.kind), sub: h.worktree || 'this Mac', cmd: h.command || h.label, started_at: h.since, kind: h.kind, extra: h.waited_s ? `waited ${fmtDur(h.waited_s)}` : '' });
for (const pc of (D.pcs && D.pcs.machines) || []) if (pc.running) out.push({ where: pc.name, slot: 'relay job', title: `${pc.running.kind} ${pc.running.job}`, sub: pc.running.title || pc.machine || '', cmd: pc.running.stage ? `stage ${pc.running.stage}` : 'reports every 5 min while it runs', started_at: pc.running.started_at, kind: pc.running.kind, extra: pc.running.last_report_at ? `last report ${ago(pc.running.last_report_at)}` : '' });
return out;
}
function renderNow() {
const items = nowItems();
$('now-sub').textContent = items.length ? `${items.length} in flight` : '';
if (!items.length) { $('now').innerHTML = `<div class="empty">Nothing is building right now. The box's slot is free, the Mac holds no lock and neither PC reports a running job. A build appears here within 30 s of its slot being taken (the box) or 60 s (the Mac and the PCs).</div>`; return; }
$('now').innerHTML = items.map(i => `<div class="job running" data-start="${esc(i.started_at || '')}"><div class="pulse"></div><div class="top"><div><div class="title">${esc(i.title)}</div><div class="where">${esc(i.where)} · ${esc(i.slot || '')}${i.sub ? ' · ' + esc(i.sub) : ''}</div></div>${pill(i.kind ? kindLabel(i.kind) : 'job', 'kind')}</div><div class="cmd" title="${esc(i.cmd)}">${esc(i.cmd || '')}</div><div class="foot"><span>since ${t(i.started_at)}${i.extra ? ' · ' + esc(i.extra) : ''}</span><span class="elapsed">${i.started_at ? fmtDurShort((Date.now() - Date.parse(i.started_at)) / 1000) : '–'}</span></div></div>`).join('');
}
function tick() { for (const el of document.querySelectorAll('.job.running[data-start]')) { const s = el.dataset.start; if (!s) continue; const e = el.querySelector('.elapsed'); if (e) e.textContent = fmtDurShort((Date.now() - Date.parse(s)) / 1000); } }
function renderQueue() {
const items = [];
for (const w of (D.box && D.box.queue) || []) items.push({ where: D.box.name, title: w.label ? kindLabel(w.kind) : 'a build', sub: w.label || 'label unknown: the waiter writes none until its slot is taken', since: w.since, wait: w.waiting_s });
for (const w of (D.mac && D.mac.queue) || []) items.push({ where: D.mac.name, title: `${w.mode} slot`, sub: w.command, since: null, wait: w.waiting_s });
for (const pc of (D.pcs && D.pcs.machines) || []) for (const q of pc.queue) items.push({ where: pc.name, title: `${q.kind} ${q.job}`, sub: q.title || '', since: null, wait: null, until: q.expires_at });
$('queue-sub').textContent = items.length ? `${items.length} waiting` : '';
if (!items.length) { $('queue').innerHTML = `<div class="empty">The queue is empty. On the box a second build waits on the flock of build-0 and shows here; on the Mac a with-lock.sh wait; on a PC a signed job it has not reported on yet.</div>`; return; }
$('queue').innerHTML = items.map(i => `<div class="job"><div class="top"><div><div class="title">${esc(i.title)}</div><div class="where">${esc(i.where)}</div></div>${pill('waiting')}</div><div class="cmd" title="${esc(i.sub)}">${esc(i.sub)}</div><div class="foot"><span>${i.since ? 'since ' + t(i.since) : i.until ? 'signed, expires ' + t(i.until, true) : ''}</span><span>${i.wait !== null && i.wait !== undefined ? 'waiting ' + esc(fmtDur(i.wait)) : ''}</span></div></div>`).join('');
}
function doneRows() {
const rows = [];
for (const b of (D.box && D.box.recent) || []) rows.push({ key: b.id, where: D.box.name, at: b.end || b.start, title: `${kindLabel(b.kind)} · ${b.worktree || '?'}${b.agent ? ' · ' + b.agent : ''}`, meta: b.command || '', ok: b.ok, secs: b.secs, status: b.ok ? 'ok' : `exit ${b.exit}`, lines: b.closing || null, raw: b });
for (const pc of (D.pcs && D.pcs.machines) || []) for (const r of pc.recent) rows.push({ key: `${pc.id}-${r.job}`, where: pc.name, at: r.finished_at, title: `${r.kind} ${r.job}`, meta: r.title || r.summary || '', ok: r.ok, secs: r.secs, status: r.status, lines: [r.summary, ...(r.errors || [])].filter(Boolean), raw: r });
rows.sort((a, b) => String(b.at || '').localeCompare(String(a.at || '')));
return rows;
}
function renderDone() {
const rows = doneRows().slice(0, 40);
const boxRows = timeline((D.box && D.box.recent) || [], { limit: 200 }); const closing = new Map(boxRows.map(r => [r.id, r.closing]));
if (!rows.length) { $('done').innerHTML = `<div class="empty">No finished work yet. The box appends one line per build to /srv/builds/_log/builds.jsonl when bs_remote_run ends${D.box && D.box.log && !D.box.log.present ? ' (the file does not exist yet)' : ''}; the PCs' closing reports come from the relay intake.</div>`; return; }
const max = Math.max(60, ...rows.map(r => Number(r.secs) || 0));
$('done').innerHTML = rows.map(r => {
const w = Math.max(1.5, ((Number(r.secs) || 0) / max) * 100).toFixed(1);
const lines = r.lines || closing.get(r.key) || [];
const isOpen = open.has(r.key);
return `<div class="row ${isOpen ? 'open' : ''}" data-key="${esc(r.key)}" role="button" tabindex="0"><div class="at">${t(r.at)}<br><span style="opacity:.7">${esc(r.where)}</span></div><div class="what"><div class="t">${esc(r.title)}</div><div class="m" title="${esc(r.meta)}">${esc(r.meta)}</div></div><div class="bar"><i class="${r.ok ? '' : 'failed'}" style="width:${w}%"></i></div><div class="dur" style="color:${r.ok ? 'var(--good)' : 'var(--bad)'}">${esc(fmtDurShort(r.secs))}<br><span style="color:var(--muted);font-size:10px">${esc(r.status)}</span></div></div><div class="lines ${isOpen ? 'show' : ''}">${lines.length ? lines.map(l => esc(l)).join('\n') : 'no closing lines kept for this entry'}</div>`;
}).join('');
for (const el of $('done').querySelectorAll('.row')) { const f = () => { const k = el.dataset.key; open.has(k) ? open.delete(k) : open.add(k); renderDone(); }; el.onclick = f; el.onkeydown = e => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); f(); } }; }
}
function renderHeadline() {
const rows = headline(D.headline);
$('headline').innerHTML = rows.map(r => {
const max = Math.max(r.box_s || 0, r.mac_s || 0, r.pc_s || 0, 1);
return `<div class="h3"><div class="l">${esc(r.label)}</div><div class="v">${r.box_s === null ? '<span style="color:var(--muted);font-size:14px">not measured yet</span>' : esc(fmtDur(r.box_s)) + '<small>box</small>'}</div><div class="cmp">${r.mac_s !== null ? `Mac <b>${esc(fmtDur(r.mac_s))}</b>${r.speedup ? ` (${r.speedup}× slower)` : ''}` : 'Mac <b>not measured on this tree</b>'}${r.pc_s !== null ? ` · PC 1 <b>${esc(fmtDur(r.pc_s))}</b>` : ''}</div>${r.box_s !== null ? `<div class="meter"><i style="width:${(r.box_s / max * 100).toFixed(1)}%"></i>${r.mac_s !== null ? `<em style="left:${(r.mac_s / max * 100).toFixed(1)}%" title="Mac"></em>` : ''}${r.pc_s !== null ? `<em style="left:${(r.pc_s / max * 100).toFixed(1)}%;background:var(--cool)" title="PC 1"></em>` : ''}</div>` : ''}<div class="s">${esc(r.note || r.what)}${r.measured_at ? ` · measured ${t(r.measured_at, true)}` : ''}</div></div>`;
}).join('');
}
function spark(points, { max = 100, label = v => v } = {}) {
if (!points.length) return '';
const W = 300, H = 90, L = 30, R = 6, T = 8, B = 20;
const X = i => L + (points.length === 1 ? (W - L - R) / 2 : (i / (points.length - 1)) * (W - L - R));
const Y = v => T + (1 - Math.max(0, Math.min(max, v)) / max) * (H - T - B);
const path = points.map((p, i) => `${i ? 'L' : 'M'}${X(i).toFixed(1)} ${Y(p.v).toFixed(1)}`).join(' ');
const area = `${path} L${X(points.length - 1).toFixed(1)} ${(H - B).toFixed(1)} L${X(0).toFixed(1)} ${(H - B).toFixed(1)} Z`;
return `<svg viewBox="0 0 ${W} ${H}" aria-label="${esc(label(max))} at the top"><line x1="${L}" y1="${H - B}" x2="${W - R}" y2="${H - B}" stroke="#26262d"/><line x1="${L}" y1="${T}" x2="${L}" y2="${H - B}" stroke="#26262d"/><path d="${area}" fill="rgba(106,169,255,.16)"/><path d="${path}" fill="none" stroke="#6aa9ff" stroke-width="2"/>${points.map((p, i) => `<circle cx="${X(i).toFixed(1)}" cy="${Y(p.v).toFixed(1)}" r="2.5" fill="#6aa9ff"><title>${esc(p.t)}</title></circle>`).join('')}<text x="4" y="${T + 4}" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${esc(label(max))}</text><text x="4" y="${H - B}" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">0</text><text x="${L}" y="${H - 6}" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${esc(points[0].x)}</text><text x="${W - R}" y="${H - 6}" text-anchor="end" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${esc(points[points.length - 1].x)}</text></svg>`;
}
function bars(buckets) {
const W = 300, H = 90, T = 8, B = 18, max = Math.max(1, ...buckets.map(b => b.n));
const bw = (W - 8) / 24;
return `<svg viewBox="0 0 ${W} ${H}" aria-label="builds per hour today"><line x1="4" y1="${H - B}" x2="${W - 4}" y2="${H - B}" stroke="#26262d"/>${buckets.map((b, i) => { const h = (b.n / max) * (H - T - B); const ok = b.n ? (b.ok / b.n) * h : 0; const x = 4 + i * bw; return `<g><title>${String(i).padStart(2, '0')}:00 UTC: ${b.n} build${b.n === 1 ? '' : 's'}, ${b.failed} failed</title><rect x="${(x + 1).toFixed(1)}" y="${(H - B - h).toFixed(1)}" width="${(bw - 2).toFixed(1)}" height="${h.toFixed(1)}" fill="#ff6b5e"/><rect x="${(x + 1).toFixed(1)}" y="${(H - B - ok).toFixed(1)}" width="${(bw - 2).toFixed(1)}" height="${ok.toFixed(1)}" fill="#5fd08a"/></g>`; }).join('')}${[0, 6, 12, 18, 23].map(h => `<text x="${(4 + h * bw + bw / 2).toFixed(1)}" y="${H - 5}" text-anchor="middle" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${String(h).padStart(2, '0')}Z</text>`).join('')}<text x="4" y="${T + 4}" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${max}</text></svg>`;
}
function renderAnalytics() {
const builds = (D.box && D.box.recent) || [];
const day = new Date().toISOString().slice(0, 10);
const todays = builds.filter(b => String(b.end || b.start || '').startsWith(day));
$('an-sub').textContent = builds.length ? `${builds.length} logged builds, ${todays.length} today, ${todays.filter(b => b.ok).length} passed` : '';
if (!builds.length) { $('an').innerHTML = `<div class="empty">Analytics start with the first logged build on the box. The log is /srv/builds/_log/builds.jsonl${D.box && D.box.log && D.box.log.present ? ` (${D.box.log.total} lines, ${D.box.log.bad_lines} unreadable)` : ' (not written yet)'}.</div>`; return; }
const ph = buildsPerHour(builds, day);
const dk = durationByKind(builds);
const hr = hitRateSeries(builds);
const cmp = compareKinds(builds, D.baselines || {});
const ukNow = ukTime(new Date().toISOString());
$('an').innerHTML = [
`<div class="a"><div class="l">Builds per hour today<span>${todays.length} today, hours in UTC</span></div>${bars(ph)}<div class="note">Green passed, red failed. Local time now ${esc(ukNow.text)} (${esc(ukNow.utc.slice(11))}).</div></div>`,
`<div class="a"><div class="l">Duration by kind<span>median and p90, all logged</span></div><table><thead><tr><th>Kind</th><th class="num">n</th><th class="num">median</th><th class="num">p90</th><th class="num">max</th></tr></thead><tbody>${dk.map(d => `<tr><td>${esc(d.label)}</td><td class="num">${d.n}</td><td class="num">${esc(fmtDurShort(d.median_s))}</td><td class="num">${esc(fmtDurShort(d.p90_s))}</td><td class="num">${esc(fmtDurShort(d.max_s))}</td></tr>`).join('')}</tbody></table></div>`,
`<div class="a"><div class="l">sccache hit rate over time<span>per build</span></div>${hr.length ? spark(hr.map(p => ({ v: p.pct, t: `${ukTime(p.at).utc}: ${p.pct}% (${kindLabel(p.kind)})`, x: ukTime(p.at).text })), { max: 100, label: v => v + '%' }) : '<div class="empty">no sccache figures in the log yet</div>'}<div class="note">A cold tree misses everything; the second build of the same tree should climb towards 100%.</div></div>`,
`<div class="a"><div class="l">Box against Mac<span>same kind, median</span></div>${cmp.length ? `<table><thead><tr><th>Kind</th><th class="num">box</th><th class="num">Mac</th><th class="num">×</th></tr></thead><tbody>${cmp.map(c => `<tr><td>${esc(c.label)}</td><td class="num">${c.box_s === null ? '–' : esc(fmtDurShort(c.box_s)) + ` <span style="color:var(--muted)">(${c.box_n})</span>`}</td><td class="num" title="${esc(c.mac_source || '')}">${c.mac_s === null ? '–' : esc(fmtDurShort(c.mac_s))}</td><td class="num">${c.speedup === null ? '' : c.speedup + '×'}</td></tr>`).join('')}</tbody></table><div class="note">Mac figures marked approximate come from the docs that measured them, not from this tree; a Mac row goes exact when the same tree is built here under with-lock.sh and logged.</div>` : '<div class="empty">no kinds to compare yet</div>'}</div>`,
].join('');
}
function renderSources() {
const s = D.sources || {};
const one = (name, v, what) => `<span>${esc(name)} <b class="${v && v.ok ? '' : v && v.at ? 'warn' : 'no'}">${v && v.ok ? 'live' : v && v.at ? 'stale' : 'missing'}</b>${v && v.at ? ' ' + esc(ago(v.at)) : ''}${v && v.error ? ': ' + esc(v.error) : ''}${!v || v.ok ? '' : ' · ' + esc(what)}</span>`;
$('sources').innerHTML = [one('box', s.box, 'collect.mjs on the box, systemd timer every 30 s'), one('mac', s.mac, 'push.mjs on the Mac, launchd every 60 s'), one('pcs', s.pcs, 'relay intake read by push.mjs'), `<span>${esc(D._feed || '')}: fetched ${esc(ago(D._fetched))}, written ${t(D.generated_at)} by ${esc(D.generated_by || '?')}</span>`, D.box && D.box.log ? `<span>log ${D.box.log.present ? `${D.box.log.total} lines${D.box.log.bad_lines ? `, ${D.box.log.bad_lines} unreadable` : ''}` : 'not written yet'}</span>` : ''].join('');
}
function apply(d) {
D = d; D._fetched = new Date().toISOString();
const age = (Date.now() - Date.parse(d.generated_at)) / 1000;
const live = $('live'); live.className = 'live' + (age > 900 ? ' down' : age > 420 ? ' stale' : '');
const u = ukTime(d.generated_at);
$('stamp').innerHTML = `${age > 420 ? 'stale: ' : ''}written ${t(d.generated_at)} (${esc(ago(d.generated_at))})${d._feed ? ' · ' + esc(d._feed) : ''}`;
renderServer(d.box, d.sources && d.sources.box); renderCrew(); renderNow(); renderQueue(); renderDone(); renderHeadline(); renderAnalytics(); renderSources();
}
const LIVE_URL = 'https://build.igneum.network/workers.json'; // the box itself, 30 s fresh (Caddy, read-only); the folder copy is up to 5 min behind
let skipLive = 0, feed = '';
async function load() {
let d = null, err = null;
if (!demo && skipLive <= 0) {
try { const r = await fetch(`${LIVE_URL}?t=${Date.now()}`, { cache: 'no-store', signal: AbortSignal.timeout(4000) }); if (!r.ok) throw new Error(`http ${r.status}`); d = await r.json(); feed = 'live from the box'; }
catch (e) { skipLive = 4; err = e; } // the box not answering: use the edge copy for the next minute before trying again
} else skipLive--;
if (!d) {
try { const file = demo ? 'workers.demo.json' : 'workers.json'; const r = await fetch(`${base}/${file}?t=${Date.now()}`, { cache: 'no-store' }); if (!r.ok) throw new Error(`http ${r.status}`); d = await r.json(); feed = demo ? 'demo document' : 'edge copy, up to 5 min behind the box'; }
catch (e) { $('live').className = 'live down'; $('stamp').textContent = `workers.json not reachable (${e && e.message || e})`; if (!D) { $('now').innerHTML = $('queue').innerHTML = `<div class="empty">No workers.json next to this page yet and the box does not answer. The Mac pusher writes the copy every 60 s and the fleet deploy carries it within 5 min.</div>`; } return; }
}
d._feed = feed; apply(d);
}
await load(); setInterval(load, 15000); setInterval(tick, 1000); setInterval(() => { if (D) { renderSources(); } }, 10000);
</script>
</body></html>

35
tools/workers/publish-workers.sh Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env bash
# Put the worker dashboard page next to the fleet page and deploy the folder through publish-fleet.sh's mechanism.
# tools/workers/publish-workers.sh copy workers.html, shape.js, workers.demo.json; one push.mjs pass; deploy
# tools/workers/publish-workers.sh --no-deploy copy only (the fleet orchestrator's next deploy carries it)
# The folder is $DLSITE/<fleet-path>/ (dlsite-dir and fleet-path in ~/.config/igneum). The fleet index.html gets the
# Workers nav link if it does not carry one yet (the fleet page has no repo source; this is the only edit made to it).
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../.." && pwd)"
DLSITE="$(cat "$HOME/.config/igneum/dlsite-dir")"; P="$(cat "$HOME/.config/igneum/fleet-path")"
DIR="$DLSITE/$P"; [ -d "$DIR" ] || { echo "no $DIR" >&2; exit 1; }
cp "$HERE/page/workers.html" "$DIR/workers.html"
cp "$HERE/shape.js" "$DIR/shape.js"
cp "$HERE/fixtures/workers.demo.json" "$DIR/workers.demo.json"
if ! grep -q 'workers.html' "$DIR/index.html"; then
python3 - "$DIR/index.html" <<'PY'
import sys, re
p = sys.argv[1]; s = open(p).read()
css = " nav { display: flex; gap: 2px; border: 1px solid var(--line); }\n nav a { font-family: var(--mono); font-size: 11px; letter-spacing: 0.1em; text-transform: uppercase; color: var(--muted); text-decoration: none; padding: 7px 12px; }\n nav a.on { color: var(--ink); background: var(--panel-2); }\n nav a:hover { color: var(--ember-2); }\n"
s = s.replace(" .live {", css + " .live {", 1)
s = s.replace(' <div class="live"><b></b><span id="stamp">', ' <nav aria-label="Pages"><a href="./" class="on" aria-current="page">Fleet</a><a href="workers.html">Workers</a></nav>\n <div class="live"><b></b><span id="stamp">', 1)
open(p, 'w').write(s)
print("nav link added to the fleet index.html")
PY
fi
node "$HERE/push.mjs" --no-deploy
if [ "${1:-}" != --no-deploy ]; then
tmp="$(mktemp)"; cp "$DIR/fleet.json" "$tmp"
"$ROOT/tools/fleet/publish-fleet.sh" "$tmp" >/dev/null || "/Users/joshm/Projects/igneum/tools/fleet/publish-fleet.sh" "$tmp"
rm -f "$tmp"
for i in $(seq 1 12); do
if curl -fsS --max-time 10 "https://dl.igneum.network/$P/workers.html?t=$(date +%s)" | grep -c "Igneum Workers" >/dev/null; then echo "live: https://dl.igneum.network/$P/workers.html"; exit 0; fi
sleep 5
done
echo "deployed but the edge does not serve workers.html yet" >&2; exit 2
fi

205
tools/workers/push.mjs Normal file
View file

@ -0,0 +1,205 @@
#!/usr/bin/env node
// The worker dashboard's Mac side, run by launchd every 60 s (tools/workers/launchd/com.igneum.workers-push.plist).
// Gathers what only this Mac can see, merges it with the box's file and publishes:
// 1. the Mac's with-lock slots (/tmp/igneum-locks: build-0..2, run-0..2, measure, build) and their waiters
// 2. the two PCs' relay job states from the log intake (Neon, the same reader as tools/jobs.mjs): the running job,
// the queue (signed jobs for that machine with no report yet), the last 20 closing reports with durations
// 3. scp of those two facts to the box (/srv/workers/sources/{mac,pcs}.json) so the box's own file is whole too
// 4. the box's /srv/workers/workers.json pulled back, the Mac and PC facts overlaid, headline.json and the Mac baselines
// added, written to $DLSITE/<fleet-path>/workers.json
// 5. a Vercel deploy ONLY when the live copy is older than 6 minutes: the fleet orchestrator deploys the same folder every
// 5 minutes (tools/fleet/autorun.py), so the file normally rides along with it; one deploy per publish is the
// publish-fleet.sh mechanism and a deploy a minute is not wanted.
//
// node tools/workers/push.mjs one pass (what launchd runs)
// node tools/workers/push.mjs --stdout print the merged document, publish nothing
// node tools/workers/push.mjs --deploy force the deploy this pass
// node tools/workers/push.mjs --no-deploy never deploy this pass (write the folder copy only)
// Reads ~/.config/igneum/{build-server,env,dl-token,dlsite-dir,fleet-path} and ~/.ssh/igneum_ed25519. Prints nothing
// secret. Log: ~/Library/Logs/igneum-workers-push.log (launchd) or stderr.
import { readFileSync, readdirSync, writeFileSync, renameSync, mkdirSync, existsSync, statSync } from 'node:fs';
import { spawnSync, execFileSync } from 'node:child_process';
import { homedir, hostname, tmpdir } from 'node:os';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseLockDir, summaryOf, deriveKind, BASELINES } from './lib.mjs';
const HERE = dirname(fileURLToPath(import.meta.url));
const argv = process.argv.slice(2);
const has = f => argv.includes(`--${f}`);
const cfg = n => { try { return readFileSync(join(homedir(), '.config', 'igneum', n), 'utf8').trim(); } catch { return ''; } };
const iso = ms => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
const log = (...a) => console.error(`${iso(Date.now())} push: ${a.join(' ')}`);
const KEY = process.env.IGNEUM_BUILD_KEY || join(homedir(), '.ssh', 'igneum_ed25519');
const HOST = (cfg('build-server') || '').split('\n')[0].trim(); // build@<ip>
const DLSITE = cfg('dlsite-dir'), FLEET = cfg('fleet-path');
const STALE_LIVE_S = 360;
const SSH_OPTS = ['-i', KEY, '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=accept-new', '-o', 'ConnectTimeout=10', '-o', 'ControlMaster=auto', '-o', `ControlPath=${join(homedir(), '.ssh', 'cm', 'igneum-workers-%r@%h:%p')}`, '-o', 'ControlPersist=600'];
mkdirSync(join(homedir(), '.ssh', 'cm'), { recursive: true });
// ---- 1. the Mac's slots ----------------------------------------------------------------------------------------------------
const alive = pid => { try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; } };
export function macSlots(now = Date.now(), dir = '/tmp/igneum-locks') {
let files = [];
try { files = readdirSync(dir).map(name => ({ name, text: readFileSync(join(dir, name), 'utf8') })); } catch { files = []; }
const r = parseLockDir(files, { now, aliveFn: alive });
let cap = 3; try { cap = Math.max(1, Math.min(3, parseInt(cfg('build-slots'), 10) || 3)); } catch { cap = 3; }
// waiters: the python with-lock.sh exec'd into, still in its wait loop (`python3 - build /tmp/igneum-locks <cmd>`); the
// holder exec'd on into its command, so a process still carrying that argv is waiting. Our own argv never matches.
const waiters = [];
const ps = spawnSync('ps', ['-eo', 'pid=,etimes=,args='], { encoding: 'utf8' }).stdout || '';
for (const line of ps.split('\n')) {
const m = /^\s*(\d+)\s+(\d+)\s+(\S*python3?) - (build|measure|run) \/tmp\/igneum-locks (.*)$/.exec(line);
if (!m || Number(m[1]) === process.pid) continue;
const held = r.held.find(h => h.pid === Number(m[1]));
if (held) continue; // measure mode keeps python in front? no: it execs too; a listed one is a waiter
waiters.push({ pid: Number(m[1]), mode: m[4], waiting_s: Number(m[2]), command: m[5].slice(0, 160), kind: deriveKind({ command: m[5] }) });
}
for (const h of r.held) h.worktree = h.worktree || worktreeOfPid(h.pid);
return { name: hostname().replace(/\.local$/, ''), collected_at: iso(now), build_slots: cap, slots: { count: cap, held: r.held, free: r.free }, queue: waiters };
}
// the holder's cwd names the worktree it builds in (lsof is slow; /proc does not exist here, so `lsof -p` only when needed)
function worktreeOfPid(pid) {
const out = spawnSync('lsof', ['-a', '-p', String(pid), '-d', 'cwd', '-Fn'], { encoding: 'utf8', timeout: 4000 }).stdout || '';
const m = /\n n(\/Users\/[^\n]*)/.exec('\n' + out.replace(/^n/m, ' n'));
const p = m ? m[1] : (out.split('\n').find(l => l.startsWith('n/')) || '').slice(1);
const w = /\/Projects\/(igneum(?:-wt-[\w.-]+)?)/.exec(p);
return w ? w[1] : null;
}
// ---- 2. the PCs through the intake ---------------------------------------------------------------------------------------
const MACHINES = [
{ id: 'ae432dc7', name: 'PC 1', role: 'builds', note: 'Linux and Windows binaries, the project lead\'s desk' },
{ id: '1ccfe586', name: 'PC 2', role: 'test suites', note: 'cargo test suites and shard runs' },
];
function db() {
const m = /^DATABASE_URL=(.*)$/m.exec(cfg('env'));
if (!m) return null;
const url = m[1].trim().replace(/^['"]|['"]$/g, '');
const host = new URL(url).hostname.replace('-pooler', '');
return async (query, params = []) => {
const r = await fetch(`https://${host}/sql`, { method: 'POST', headers: { 'Neon-Connection-String': url, 'Content-Type': 'application/json' }, body: JSON.stringify({ query, params }), signal: AbortSignal.timeout(15000) });
const j = await r.json();
if (!r.ok) throw new Error(j.message || `http ${r.status}`);
return j.rows;
};
}
async function signedJobs() {
const tok = cfg('dl-token'); if (!tok) return [];
try {
const r = await fetch(`https://dl.igneum.network/dl/${tok}/igneum-jobs.signed.json?t=${Date.now()}`, { headers: { 'Cache-Control': 'no-cache' }, signal: AbortSignal.timeout(15000) });
if (!r.ok) return [];
const env = await r.json(); const f = JSON.parse(env.file);
return Array.isArray(f.jobs) ? f.jobs : [];
} catch (e) { log('jobs file:', e.message); return []; }
}
export function shapePcs(rows, jobs, now = Date.now()) {
// rows: the newest upload per run_id with its machine; a run_id is job-<id>-<machine id>
const byMachine = new Map(MACHINES.map(m => [m.id, { ...m, machine: null, running: null, queue: [], recent: [], last_report_at: null }]));
const seenJob = new Set();
for (const r of rows) {
const mid = (/-([0-9a-f]{8})$/.exec(r.machine || '') || [])[1] || (/-([0-9a-f]{8})$/.exec(r.run_id || '') || [])[1];
const pc = byMachine.get(mid); if (!pc) continue;
pc.machine = pc.machine || r.machine;
const s = summaryOf(r.lines); if (!s) continue;
const jobId = s.job || (r.run_id || '').replace(/^job-/, '').replace(new RegExp(`-${mid}$`), '');
seenJob.add(jobId);
const kind = s.kind || (jobId.split('-')[0]) || 'job';
const at = Date.parse(String(r.received_at).replace(' ', 'T').replace(/([+-]\d\d)$/, '$1:00'));
if (!pc.last_report_at || at > Date.parse(pc.last_report_at)) pc.last_report_at = iso(at);
if (s.status === 'running') {
// a progress report under 15 minutes old is a live job; older is a job whose closing report never came
const fresh = now - at < 15 * 60_000;
const entry = { job: jobId, kind, title: titleOf(jobs, jobId), started_at: s.started_at || null, stage: s.stage, elapsed_s: s.started_at ? Math.max(0, Math.round((now - Date.parse(s.started_at)) / 1000)) : null, last_report_at: iso(at), stale: !fresh };
if (!pc.running || Date.parse(entry.last_report_at) > Date.parse(pc.running.last_report_at)) pc.running = entry;
} else {
pc.recent.push({ job: jobId, kind, title: titleOf(jobs, jobId), status: s.status, ok: s.status === 'done' && Number(s.exit) === 0, exit: s.exit, secs: s.duration_s, started_at: s.started_at || null, finished_at: s.finished_at && s.finished_at !== 'closed' ? s.finished_at : iso(at), summary: String(s.summary || '').slice(0, 240), errors: (s.errors || []).map(e => e.slice(0, 200)), stage: s.stage });
}
}
for (const pc of byMachine.values()) {
pc.recent.sort((a, b) => String(b.finished_at).localeCompare(String(a.finished_at))); pc.recent = pc.recent.slice(0, 20);
if (pc.running && pc.running.stale) { pc.running = null; }
// the queue: signed, unexpired jobs aimed at this machine that no report has mentioned yet
for (const j of jobs) {
const t = j.target || {}; const ids = Array.isArray(t.machine_ids) ? t.machine_ids : t.machine_ids ? [t.machine_ids] : [];
if (!ids.includes(pc.id)) continue;
if (Date.parse(j.expires_at) < now) continue;
if (seenJob.has(j.id)) continue;
pc.queue.push({ job: j.id, kind: j.kind, title: j.title || '', expires_at: j.expires_at });
}
pc.queue.sort((a, b) => String(a.job).localeCompare(String(b.job)));
}
return { collected_at: iso(now), machines: [...byMachine.values()] };
}
const titleOf = (jobs, id) => { const j = jobs.find(j => j.id === id); return j ? (j.title || '') : ''; };
async function pcs() {
const sql = db(); if (!sql) return { error: 'no DATABASE_URL in ~/.config/igneum/env', collected_at: iso(Date.now()), machines: [] };
const rows = await sql(`SELECT DISTINCT ON (run_id) run_id, machine, received_at, lines FROM miner_logs WHERE run_id LIKE 'job-%' AND label LIKE 'job-%' AND received_at > now() - interval '7 days' ORDER BY run_id, received_at DESC`);
const jobs = await signedJobs();
return shapePcs(rows, jobs);
}
// ---- 3 and 4. the box -------------------------------------------------------------------------------------------------------
function ssh(args, input) { return spawnSync('ssh', [...SSH_OPTS, HOST, ...args], { encoding: 'utf8', input, timeout: 30000, maxBuffer: 16 * 1024 * 1024 }); }
function dropToBox(name, doc) {
const r = ssh([`cat > /srv/workers/sources/.${name}.tmp && mv /srv/workers/sources/.${name}.tmp /srv/workers/sources/${name}.json`], JSON.stringify(doc));
if (r.status !== 0) throw new Error(`scp ${name}: ${(r.stderr || '').trim().slice(0, 200) || 'ssh failed'}`);
}
function pullFromBox() {
const r = ssh(['cat /srv/workers/workers.json; echo; echo __SEP__; cat /srv/workers/headline.json 2>/dev/null || echo null']);
if (r.status !== 0) throw new Error((r.stderr || '').trim().slice(0, 200) || 'ssh failed');
const [w, h] = r.stdout.split('__SEP__');
return { workers: JSON.parse(w), headline: JSON.parse((h || 'null').trim() || 'null') };
}
// ---- 5. publish ----------------------------------------------------------------------------------------------------------------
async function liveAge() {
try {
const r = await fetch(`https://dl.igneum.network/${FLEET}/workers.json?t=${Date.now()}`, { cache: 'no-store', signal: AbortSignal.timeout(10000) });
if (!r.ok) return Infinity;
const j = await r.json(); return (Date.now() - Date.parse(j.generated_at)) / 1000;
} catch { return Infinity; }
}
function deploy() {
const r = spawnSync('npx', ['--yes', 'vercel@latest', '--global-config', join(homedir(), '.config', 'igneum', 'vercel'), 'deploy', '--prod', '--yes'], { cwd: DLSITE, encoding: 'utf8', timeout: 240000, env: { ...process.env, PATH: `/opt/homebrew/bin:/usr/local/bin:${process.env.PATH || ''}` } });
return r.status === 0;
}
async function main() {
const now = Date.now();
const doc = { v: 1, generated_at: iso(now), generated_by: `push.mjs on ${hostname().replace(/\.local$/, '')}`, box: null, mac: null, pcs: null, sources: {}, baselines: BASELINES, headline: null };
// the Mac
try { doc.mac = macSlots(now); doc.sources.mac = { ok: true, at: doc.mac.collected_at }; } catch (e) { doc.sources.mac = { ok: false, error: String(e.message || e) }; }
// the PCs
try { doc.pcs = await pcs(); doc.sources.pcs = doc.pcs.error ? { ok: false, error: doc.pcs.error } : { ok: true, at: doc.pcs.collected_at }; } catch (e) { doc.sources.pcs = { ok: false, error: `intake: ${String(e.message || e).slice(0, 160)}` }; }
// the box: drop ours, pull theirs
if (!HOST) doc.sources.box = { ok: false, error: 'no ~/.config/igneum/build-server' };
else {
try {
if (doc.mac) dropToBox('mac', doc.mac);
if (doc.pcs && !doc.pcs.error) dropToBox('pcs', doc.pcs);
const { workers, headline } = pullFromBox();
doc.box = workers.box; doc.headline = headline;
const age = (now - Date.parse(workers.generated_at)) / 1000;
doc.sources.box = { ok: age < 120, at: workers.generated_at, age_s: Math.round(age), ...(age >= 120 ? { error: `the box's collector last wrote ${Math.round(age)} s ago (timer igneum-workers.timer)` } : {}) };
} catch (e) { doc.sources.box = { ok: false, error: `ssh to the box failed: ${String(e.message || e).slice(0, 160)}` }; }
}
if (!doc.headline) doc.headline = null;
const text = JSON.stringify(doc);
if (has('stdout')) { process.stdout.write(text + '\n'); return; }
if (!DLSITE || !FLEET) { log('no dlsite-dir or fleet-path in ~/.config/igneum; nothing published'); return; }
const dir = join(DLSITE, FLEET); mkdirSync(dir, { recursive: true });
const tmp = join(dir, '.workers.json.tmp'); writeFileSync(tmp, text); renameSync(tmp, join(dir, 'workers.json'));
let why = 'folder copy written; the fleet orchestrator deploys the folder within 5 min';
if (!has('no-deploy')) {
const age = has('deploy') ? Infinity : await liveAge();
if (age > STALE_LIVE_S) { const ok = deploy(); why = ok ? `deployed (live copy was ${age === Infinity ? 'missing' : Math.round(age) + ' s old'})` : 'DEPLOY FAILED'; }
else why = `live copy ${Math.round(age)} s old, no deploy`;
}
log(`box ${doc.sources.box.ok ? 'ok' : 'NO: ' + doc.sources.box.error}; mac ${doc.sources.mac.ok ? `${doc.mac.slots.held.length} held, ${doc.mac.queue.length} waiting` : 'NO'}; pcs ${doc.sources.pcs.ok ? doc.pcs.machines.map(m => `${m.name} ${m.running ? 'running ' + m.running.job : 'idle'} q${m.queue.length}`).join(', ') : 'NO: ' + doc.sources.pcs.error}; ${why}`);
}
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
main().catch(e => { log('failed:', e.message || e); process.exit(1); });
}

99
tools/workers/shape.js Normal file
View file

@ -0,0 +1,99 @@
// Data shaping for the worker dashboard page (workers.html loads it as a module; node --test imports the same file).
// Pure functions over workers.json: the analytics strip, the timeline bars, the time formatting. No DOM here.
export const KIND_LABEL = { 'node-linux': 'node Linux', 'node-windows': 'node Windows', app: 'app', 'app-windows': 'app Windows', suite: 'suite', check: 'check', prove: 'prove', pow: 'pow', other: 'other' };
export const kindLabel = k => KIND_LABEL[k] || k || 'other';
export function quantile(sorted, q) {
if (!sorted.length) return null;
const pos = (sorted.length - 1) * q, lo = Math.floor(pos), hi = Math.ceil(pos);
return lo === hi ? sorted[lo] : sorted[lo] + (sorted[hi] - sorted[lo]) * (pos - lo);
}
/// Median and p90 duration per kind over finished builds (failed ones included: a failed build still cost its time).
export function durationByKind(builds) {
const by = new Map();
for (const b of builds || []) { if (!Number.isFinite(b.secs)) continue; const k = b.kind || 'other'; if (!by.has(k)) by.set(k, []); by.get(k).push(b.secs); }
const out = [];
for (const [kind, arr] of by) { arr.sort((a, b) => a - b); out.push({ kind, label: kindLabel(kind), n: arr.length, median_s: quantile(arr, 0.5), p90_s: quantile(arr, 0.9), min_s: arr[0], max_s: arr[arr.length - 1] }); }
return out.sort((a, b) => b.n - a.n || a.kind.localeCompare(b.kind));
}
/// Builds that ENDED in each hour of `day` (UTC date string), 24 buckets of { hour, n, ok, failed }.
export function buildsPerHour(builds, day) {
const buckets = Array.from({ length: 24 }, (_, hour) => ({ hour, n: 0, ok: 0, failed: 0 }));
for (const b of builds || []) {
const t = b.end || b.start; if (!t || !String(t).startsWith(day)) continue;
const h = Number(String(t).slice(11, 13)); if (!(h >= 0 && h < 24)) continue;
buckets[h].n++; if (b.ok) buckets[h].ok++; else buckets[h].failed++;
}
return buckets;
}
/// sccache hit rate per build in time order, for the sparkline: [{ at, pct, kind }].
export function hitRateSeries(builds) {
return (builds || []).filter(b => b.sccache && Number.isFinite(b.sccache.hit_rate_pct)).map(b => ({ at: b.end || b.start, pct: b.sccache.hit_rate_pct, kind: b.kind })).sort((a, b) => String(a.at).localeCompare(String(b.at)));
}
/// Box against Mac for the same kind: the box's median from the log, the Mac's from `baselines` ({ kind: { secs, source } }).
export function compareKinds(builds, baselines) {
const box = new Map(durationByKind(builds).map(d => [d.kind, d]));
const kinds = new Set([...box.keys(), ...Object.keys(baselines || {})]);
return [...kinds].map(kind => {
const b = box.get(kind); const m = baselines && baselines[kind];
const ratio = b && m && m.secs ? Math.round((m.secs / b.median_s) * 10) / 10 : null;
return { kind, label: kindLabel(kind), box_s: b ? b.median_s : null, box_n: b ? b.n : 0, mac_s: m ? m.secs : null, mac_source: m ? m.source || null : null, speedup: ratio };
}).filter(r => r.box_s !== null || r.mac_s !== null).sort((a, b) => (b.box_n - a.box_n) || a.kind.localeCompare(b.kind));
}
/// The timeline: newest first, each with a bar width as a share of the longest, and the lines to show on click.
export function timeline(builds, { limit = 40 } = {}) {
const rows = (builds || []).slice().sort((a, b) => String(b.end || b.start || '').localeCompare(String(a.end || a.start || ''))).slice(0, limit);
const max = Math.max(60, ...rows.map(r => Number(r.secs) || 0));
return rows.map(r => ({
...r,
bar_pct: Math.max(1.5, Math.round(((Number(r.secs) || 0) / max) * 1000) / 10),
closing: closingLines(r),
}));
}
/// What a click on a done build shows: the facts the log carries, one per line.
export function closingLines(b) {
const out = [];
if (b.command) out.push(`$ ${b.command}`);
if (b.worktree) out.push(`${b.worktree}/${b.crate || ''}${b.branch ? ` (${b.branch}${b.sha ? ' ' + String(b.sha).slice(0, 8) : ''})` : ''}`);
out.push(`exit ${b.exit ?? '?'} after ${fmtDur(b.secs)}${b.wait_s ? `, waited ${fmtDur(b.wait_s)} for a slot` : ''}${b.slot !== undefined && b.slot !== null ? ` on build-${b.slot}` : ''}`);
if (b.compiles !== undefined) out.push(`${b.compiles} compiles, sccache ${b.sccache && b.sccache.hit_rate_pct !== null ? b.sccache.hit_rate_pct + '% hits' : 'no stats'} (${b.sccache ? `${b.sccache.hits} hit, ${b.sccache.misses} miss` : ''})`);
if (Array.isArray(b.load_end) && b.load_end.length) out.push(`load at the end ${b.load_end.join(' ')}`);
for (const a of b.artefacts || []) out.push(`${a.path} ${fmtBytes(a.bytes)} sha256 ${String(a.sha256 || '').slice(0, 16)}…`);
if (b.agent) out.push(`agent ${b.agent}`);
return out;
}
export const fmtDur = s => { s = Math.max(0, Math.round(Number(s) || 0)); const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), r = s % 60; return h ? `${h} h ${String(m).padStart(2, '0')} min` : m ? `${m} min ${String(r).padStart(2, '0')} s` : `${r} s`; };
export const fmtDurShort = s => { s = Math.max(0, Math.round(Number(s) || 0)); const h = Math.floor(s / 3600), m = Math.floor((s % 3600) / 60), r = s % 60; return h ? `${h}:${String(m).padStart(2, '0')}:${String(r).padStart(2, '0')}` : `${m}:${String(r).padStart(2, '0')}`; };
export const fmtBytes = n => { n = Number(n) || 0; if (n < 1024) return `${n} B`; if (n < 1048576) return `${(n / 1024).toFixed(1)} KB`; if (n < 1073741824) return `${(n / 1048576).toFixed(1)} MB`; return `${(n / 1073741824).toFixed(2)} GB`; };
export const fmtBps = n => { n = Number(n) || 0; if (n < 1e6) return `${(n / 1e3).toFixed(0)} kb/s`; if (n < 1e9) return `${(n / 1e6).toFixed(1)} Mb/s`; return `${(n / 1e9).toFixed(2)} Gb/s`; };
export const fmtUptime = s => { s = Number(s) || 0; const d = Math.floor(s / 86400), h = Math.floor((s % 86400) / 3600), m = Math.floor((s % 3600) / 60); return d ? `${d} d ${h} h` : h ? `${h} h ${m} min` : `${m} min`; };
/// UK local time for display with the UTC in the title; `tz` defaults to Europe/London.
export function ukTime(isoStr, { tz = 'Europe/London', withDate = false } = {}) {
if (!isoStr) return { text: '', utc: '' };
const d = new Date(isoStr); if (Number.isNaN(d.getTime())) return { text: String(isoStr), utc: '' };
const opts = { timeZone: tz, hour: '2-digit', minute: '2-digit', hour12: false, ...(withDate ? { day: '2-digit', month: 'short' } : {}) };
let text; try { text = new Intl.DateTimeFormat('en-GB', opts).format(d); } catch { text = d.toISOString().slice(11, 16); }
return { text, utc: d.toISOString().replace('T', ' ').slice(0, 19) + ' UTC' };
}
/// "x min ago" for stamps.
export const ago = (isoStr, now = Date.now()) => { if (!isoStr) return ''; const s = Math.max(0, Math.round((now - Date.parse(isoStr)) / 1000)); return s < 60 ? `${s} s ago` : s < 3600 ? `${Math.floor(s / 60)} min ago` : `${Math.floor(s / 3600)} h ${Math.floor((s % 3600) / 60)} min ago`; };
/// The headline timings: the three the build-server agent measures with the Mac baseline beside each.
export function headline(h) {
const rows = [
{ key: 'clean_node', label: 'Clean node build', what: 'cold target dir, Linux node' },
{ key: 'incremental_under_load', label: 'Incremental under load', what: 'one file changed while another build runs' },
{ key: 'windows_cross', label: 'Windows cross', what: 'igneumd.exe and igneum-miner.exe' },
];
return rows.map(r => { const v = (h && h[r.key]) || {}; return { ...r, box_s: Number.isFinite(v.box_s) ? v.box_s : null, mac_s: Number.isFinite(v.mac_s) ? v.mac_s : null, pc_s: Number.isFinite(v.pc_s) ? v.pc_s : null, note: v.note || null, measured_at: v.measured_at || null, speedup: Number.isFinite(v.box_s) && Number.isFinite(v.mac_s) && v.box_s > 0 ? Math.round((v.mac_s / v.box_s) * 10) / 10 : null }; });
}

154
tools/workers/test/parsers.test.mjs Normal file
View file

@ -0,0 +1,154 @@
// node --test tools/workers/test : the collector's parsers against the fixtures, and the page's data shaping.
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { readFileSync, readdirSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { parseProcStat, cpuBusy, parseMeminfo, parseNetDev, parseLoadavg, parseSlotLine, splitLabel, deriveKind, parseLockDir, parseBuildsJsonl, parseSccacheStats, pickTemps, cargoProcess, summaryOf } from '../lib.mjs';
import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, closingLines, ukTime, headline, quantile } from '../shape.js';
const FX = join(dirname(fileURLToPath(import.meta.url)), '..', 'fixtures');
const fx = n => readFileSync(join(FX, n), 'utf8');
test('/proc/stat: cores and busy deltas', () => {
const a = parseProcStat(fx('proc-stat-0.txt')), b = parseProcStat(fx('proc-stat-1.txt'));
assert.equal(a.cores.length, 4);
const busy = cpuBusy(a, b);
assert.equal(busy.per_core.length, 4);
assert.equal(busy.per_core[0], 100); // 100 user jiffies, 0 idle
assert.equal(busy.per_core[1], 0); // idle only
assert.equal(busy.per_core[2], 100); // 50 user + 50 system
assert.equal(busy.per_core[3], 25); // 25 busy of 100
assert.equal(busy.cores_busy, 2);
// whole box: 1000 user + 200 system busy of 2000 -> 60 %
assert.equal(busy.busy_pct, 60);
// a zero delta never divides by zero
assert.equal(cpuBusy(a, a).busy_pct, 0);
});
test('/proc/meminfo, /proc/net/dev, /proc/loadavg', () => {
const m = parseMeminfo('MemTotal: 131556268 kB\nMemFree: 88262160 kB\nMemAvailable: 97647588 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n');
assert.equal(m.total_kb, 131556268); assert.equal(m.used_kb, 131556268 - 97647588); assert.equal(m.used_pct, 25.8);
const n = parseNetDev('Inter-| Receive | Transmit\n face |bytes packets errs drop fifo frame compressed multicast|bytes packets errs drop fifo colls carrier compressed\n lo: 17515759 1 0 0 0 0 0 0 17515759 1 0 0 0 0 0 0\nenp193s0f0np0: 994079034 2 0 0 0 0 0 0 80040958 3 0 0 0 0 0 0\n');
assert.deepEqual(n, [{ iface: 'enp193s0f0np0', rx_bytes: 994079034, tx_bytes: 80040958 }]);
assert.deepEqual(parseLoadavg('13.86 9.61 4.32 1/1235 36952\n'), { load: [13.86, 9.61, 4.32], running: 1, threads: 1235 });
});
test('slot line: pid, since, waited, label, agent, worktree, crate, command, target', () => {
const l = parseSlotLine('pid 36992 since 17:34:00Z waited 0 s: igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server', { day: '2026-10-06', now: '2026-10-06T18:00:00Z' });
assert.equal(l.pid, 36992); assert.equal(l.since, '2026-10-06T17:34:00Z'); assert.equal(l.waited_s, 0);
assert.equal(l.worktree, 'igneum-wt-build-server'); assert.equal(l.crate, 'vendor/igneum-node-bs');
assert.equal(l.target, 'x86_64-pc-windows-gnu'); assert.equal(l.agent, 'build-server');
assert.equal(deriveKind(l), 'node-windows');
// a clock time later than now is yesterday
const y = parseSlotLine('pid 5 since 23:50:00Z waited 2 s: x', { day: '2026-10-06', now: '2026-10-06T00:10:00Z' });
assert.equal(y.since, '2026-10-05T23:50:00Z');
assert.equal(parseSlotLine(''), null); assert.equal(parseSlotLine('garbage'), null);
// a Mac with-lock label is a bare command
const m = splitLabel('cargo build --release -p igneum-miner');
assert.equal(m.worktree, null); assert.equal(m.target, 'x86_64-unknown-linux-gnu');
});
test('kind rule matches the JSONL proposal', () => {
assert.equal(deriveKind({ command: 'cargo test --release -p kaspa-consensus-core', crate: 'vendor/igneum-node-v4' }), 'suite');
assert.equal(deriveKind({ command: 'cargo clippy --all-targets', crate: 'app/igneum-app' }), 'check');
assert.equal(deriveKind({ command: 'cargo build --release -p kaspad', crate: 'vendor/igneum-node-bs' }), 'node-linux');
assert.equal(deriveKind({ command: 'cargo build --release -p kaspad --target x86_64-pc-windows-gnu', crate: 'vendor/igneum-node-bs' }), 'node-windows');
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'app/igneum-app' }), 'app');
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'app/igneum-app', tool: 'cross-remote' }), 'app-windows');
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'proving/igneum-prove' }), 'prove');
assert.equal(deriveKind({ kind: 'suite', command: 'cargo build' }), 'suite'); // an explicit kind wins
assert.equal(deriveKind({ command: 'cargo build --release', crate: 'tools/x' }), 'other');
});
test('lock dir fixture: one held slot, one stale line, one empty, count from slots', () => {
const dir = join(FX, 'locks');
const files = readdirSync(dir).map(name => ({ name, text: readFileSync(join(dir, name), 'utf8') }));
const alive = pid => pid === 36992; // only the real holder is alive; pid 1's line is stale
const r = parseLockDir(files, { day: '2026-10-06', now: '2026-10-06T18:00:00Z', aliveFn: alive });
assert.equal(r.count, 1);
assert.equal(r.held.length, 1); assert.equal(r.held[0].slot, 'build-0'); assert.equal(r.held[0].kind, 'node-windows'); assert.equal(r.held[0].agent, 'build-server');
assert.deepEqual(r.free, ['build-1', 'build-2']);
// an explicit flock verdict overrides the pid check
const r2 = parseLockDir(files.map(f => ({ ...f, held: f.name === 'build-1' })), { day: '2026-10-06', now: '2026-10-06T18:00:00Z' });
assert.deepEqual(r2.held.map(h => h.slot), ['build-1']);
});
test('builds JSONL fixture: five rows, one bad line, newest first, derived fields', () => {
const r = parseBuildsJsonl(fx('builds.jsonl'));
assert.equal(r.total, 5); assert.equal(r.bad, 1);
assert.equal(r.rows[0].id, 'igneum-build-1-1759774200-37300');
const win = r.rows.find(b => b.tool === 'cross-remote');
assert.equal(win.kind, 'node-windows'); assert.equal(win.ok, true); assert.equal(win.sccache.hit_rate_pct, 0); assert.equal(win.artefacts.length, 2);
const suite = r.rows.find(b => b.exit === 101);
assert.equal(suite.kind, 'suite'); assert.equal(suite.ok, false);
const lin = r.rows.find(b => b.id.endsWith('37001'));
assert.equal(lin.kind, 'node-linux'); assert.equal(lin.target, 'x86_64-unknown-linux-gnu'); assert.equal(lin.sccache.hit_rate_pct, 71.7);
// secs derived from start and end when missing
const d = parseBuildsJsonl('{"start":"2026-10-06T10:00:00Z","end":"2026-10-06T10:05:30Z","exit":0,"command":"cargo build"}');
assert.equal(d.rows[0].secs, 330);
});
test('sccache --show-stats', () => {
const s = parseSccacheStats(fx('sccache-stats.txt'));
assert.equal(s.requests, 2294); assert.equal(s.hits, 790); assert.equal(s.misses, 1886); assert.equal(s.hit_rate_pct, 29.5);
assert.equal(s.cache_size, '12 GiB'); assert.equal(s.max_size, '100 GiB');
assert.equal(parseSccacheStats('Cache hits 0\nCache misses 10\n').hit_rate_pct, 0);
});
test('hwmon pick: hottest per class, cpu first, nothing absurd', () => {
const t = pickTemps([{ name: 'bnxt_en', label: null, millic: 55000 }, { name: 'nvme', label: 'Composite', millic: 36850 }, { name: 'nvme', label: 'Composite', millic: 34850 }, { name: 'k10temp', label: 'Tctl', millic: 75125 }, { name: 'k10temp', label: 'Tccd1', millic: 0 }, { name: 'acpitz', label: null, millic: 999000 }]);
assert.deepEqual(t.map(x => [x.name, x.c]), [['cpu', 75.1], ['nvme', 36.9], ['nic', 55]]);
});
test('a cargo process becomes a running build with its worktree, target, kind and start', () => {
const stat = '37210 (cargo) S 37209 37210 36992 0 -1 4194304 1 0 0 0 1 2 0 0 20 0 2 0 62341 1 2 3 4 5 6 7 8 9';
const p = cargoProcess({ pid: 37210, cwd: '/srv/builds/igneum-wt-build-server/vendor/igneum-node-bs', cmdline: ['/home/build/.rustup/toolchains/1.99.0-x86_64-unknown-linux-gnu/bin/cargo','build','--release','-p','kaspad','--target','x86_64-pc-windows-gnu','-j','90',''].join('\0'), stat, bootSec: 1759772000, clk: 100, now: (1759772000 + 1000) * 1000 });
assert.equal(p.worktree, 'igneum-wt-build-server'); assert.equal(p.crate, 'vendor/igneum-node-bs');
assert.equal(p.target, 'x86_64-pc-windows-gnu'); assert.equal(p.kind, 'node-windows');
assert.equal(p.started_at, new Date((1759772000 + 623.41) * 1000).toISOString().replace(/\.\d{3}Z$/, 'Z'));
assert.equal(p.elapsed_s, 377);
assert.equal(cargoProcess({ pid: 1, cmdline: ['rustc','--crate-name','x'].join('\0') }), null);
});
test('relay job report: SUMMARY and closing line, running without finished_at', () => {
const done = summaryOf('IGNEUM-APP 0.3.14\nSUMMARY {"job":"build-20261006-171423","kind":"build","status":"done","exit":0,"duration_s":622,"started_at":"2026-10-06T17:15:09Z","finished_at":"2026-10-06T17:25:31Z","summary":"ok"}\nSTAGE linux start 2026-10-06T17:16:00Z cap 20 min\njob build-20261006-171423: done (exit 0) after 622 s: every stage ok\n');
assert.equal(done.status, 'done'); assert.equal(done.duration_s, 622); assert.equal(done.stage, 'linux');
const run = summaryOf('SUMMARY {"job":"j1","kind":"build","status":"running","started_at":"2026-10-06T17:15:09Z"}\nSTAGE windows start 2026-10-06T17:30:00Z cap 20 min\n');
assert.equal(run.status, 'running'); assert.equal(run.stage, 'windows');
assert.equal(summaryOf('nothing here'), null);
});
// ---- page shaping ----------------------------------------------------------------------------------------------------------
const builds = parseBuildsJsonl(fx('builds.jsonl')).rows;
test('shape: median and p90 by kind, builds per hour, hit-rate series', () => {
const d = durationByKind(builds);
const nl = d.find(x => x.kind === 'node-linux'); assert.equal(nl.n, 1); assert.equal(nl.median_s, 801); assert.equal(nl.p90_s, 801);
assert.equal(quantile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 0.9), 9.1);
const h = buildsPerHour(builds, '2026-10-06');
assert.equal(h.length, 24); assert.equal(h[17].n, 1); assert.equal(h[18].n, 4); assert.equal(h[18].failed, 1); assert.equal(h[3].n, 0);
const s = hitRateSeries(builds);
assert.deepEqual(s.map(x => x.pct), [0, 71.7, 77.6, 85.7, 93.8]);
});
test('shape: box against Mac, timeline bars, closing lines, headline', () => {
const c = compareKinds(builds, { 'node-linux': { secs: 720, source: 'infra/cross/build-linux.sh, 4 Oct' }, 'app-windows': { secs: 900, source: 'runner' } });
const nl = c.find(x => x.kind === 'node-linux'); assert.equal(nl.mac_s, 720); assert.equal(nl.speedup, 0.9);
const aw = c.find(x => x.kind === 'app-windows'); assert.equal(aw.box_s, null); assert.equal(aw.mac_s, 900);
const t = timeline(builds, { limit: 3 });
assert.equal(t.length, 3); assert.equal(t[0].id, 'igneum-build-1-1759774200-37300');
assert.equal(Math.max(...t.map(r => r.bar_pct)), 100);
const lines = closingLines(builds.find(b => b.tool === 'cross-remote'));
assert.ok(lines[0].startsWith('$ cargo build')); assert.ok(lines.some(l => /igneumd\.exe\s+50\.5 MB\s+sha256 0f3a9c2e1d8b7a6f/.test(l)));
const hl = headline({ clean_node: { box_s: 600, mac_s: 2400 } });
assert.equal(hl[0].speedup, 4); assert.equal(hl[1].box_s, null); assert.equal(hl.length, 3);
});
test('shape: UK local time with the UTC in the tooltip', () => {
const t = ukTime('2026-10-06T17:34:00Z');
assert.equal(t.text, '18:34'); // BST on 6 October
assert.equal(t.utc, '2026-10-06 17:34:00 UTC');
assert.equal(ukTime('2026-12-06T17:34:00Z').text, '17:34'); // GMT in December
assert.deepEqual(ukTime(null), { text: '', utc: '' });
});