Worker dashboard: scheduling classes on the cards (kind, nice, cores), gate-queued marker, wait and gate-pending files
The slot line now ends "; kind=<k> nice=<n> cores=<c>; agent=<a>" (build box scheduling,51304fa5): lib.mjs lifts the three into their own fields and keeps the bare label; the old line shape still derives its kind. The collector reads /srv/builds/_locks/wait-<pid> and gate-pending-<pid> directly (dead pids skipped), marks a gate with priority gate and sorts it first; the page shows the class line on Now building and Queue cards, a "gate queued" pill, and the class in the closing lines. JSONL nice, cores, jobs and priority carried through. 14 tests pass. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> (cherry picked from commit9fce5742a6)
This commit is contained in:
igneum-labs
parent
927be8e87c
commit
f9b5b26f94
5 changed files with 62 additions and 15 deletions
|
|
@ -18,7 +18,7 @@ import { spawnSync } from 'node:child_process';
|
|||
import { hostname } from 'node:os';
|
||||
import { join, dirname } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { parseProcStat, cpuBusy, parseMeminfo, parseNetDev, parseLoadavg, parseLockDir, parseBuildsJsonl, parseSccacheStats, pickTemps, cargoProcess, deriveKind, BASELINES } from './lib.mjs';
|
||||
import { parseProcStat, cpuBusy, parseMeminfo, parseNetDev, parseLoadavg, parseLockDir, parseBuildsJsonl, parseSccacheStats, pickTemps, cargoProcess, deriveKind, parseSlotLine, BASELINES } from './lib.mjs';
|
||||
|
||||
const HERE = dirname(fileURLToPath(import.meta.url));
|
||||
const argv = process.argv.slice(2);
|
||||
|
|
@ -68,6 +68,7 @@ function slots(now) {
|
|||
}
|
||||
function procs(bootSec, clk, now) {
|
||||
const running = [], waiters = []; let compilers = 0;
|
||||
const flockParents = new Map(); // ppid -> since, from the flock -w processes the waiters run
|
||||
for (const d of safeList('/proc')) {
|
||||
if (!/^\d+$/.test(d)) continue;
|
||||
const pid = Number(d); const comm = read(`/proc/${d}/comm`).trim();
|
||||
|
|
@ -82,17 +83,26 @@ function procs(bootSec, clk, now) {
|
|||
if (args[1] === '-w' && /^\d+$/.test(args[2] || '')) {
|
||||
const stat = read(`/proc/${d}/stat`); const rest = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
|
||||
const ppid = Number(rest[1]); const startTicks = Number(rest[19]);
|
||||
const since = Number.isFinite(startTicks) ? iso((bootSec + startTicks / clk) * 1000) : null;
|
||||
const wait = read(join(LOCKS, `wait-${ppid}`)).trim(); // a waiter's own line, when the appender writes one
|
||||
waiters.push({ pid: ppid, since, waiting_s: since ? Math.max(0, Math.round((now - Date.parse(since)) / 1000)) : null, label: wait || null, ...(wait ? { kind: deriveKind(parseLabelSafe(wait)) } : {}) });
|
||||
flockParents.set(ppid, Number.isFinite(startTicks) ? iso((bootSec + startTicks / clk) * 1000) : null);
|
||||
}
|
||||
}
|
||||
}
|
||||
// the innermost cargo per worktree: a `cargo build` spawns no nested cargo, but a `cargo test` runs test binaries, not cargo
|
||||
// the waiters' own lines: wait-<pid> (a queued build) and gate-pending-<pid> (a gate that takes the next free slot
|
||||
// ahead of queued suites and benches), written by remote-run.sh in the slot-file format and removed on take or give-up;
|
||||
// a file whose pid is gone is a leftover, not a waiter
|
||||
const seen = new Set();
|
||||
for (const f of safeList(LOCKS)) {
|
||||
const m = /^(wait|gate-pending)-(\d+)$/.exec(f); if (!m) continue;
|
||||
const pid = Number(m[2]); if (!alive(pid) || seen.has(pid)) continue; seen.add(pid);
|
||||
const line = parseSlotLine(read(join(LOCKS, f)).split('\n')[0], { now });
|
||||
const since = (line && line.since) || flockParents.get(pid) || null;
|
||||
waiters.push({ pid, since, waiting_s: since ? Math.max(0, Math.round((now - Date.parse(since)) / 1000)) : null, label: line ? line.label : null, kind: line ? deriveKind(line) : null, agent: line ? line.agent : null, worktree: line ? line.worktree : null, crate: line ? line.crate : null, command: line ? line.command : null, nice: line ? line.nice : null, cores: line ? line.cores : null, priority: m[1] === 'gate-pending' ? 'gate' : 'normal' });
|
||||
}
|
||||
for (const [ppid, since] of flockParents) if (!seen.has(ppid)) waiters.push({ pid: ppid, since, waiting_s: since ? Math.max(0, Math.round((now - Date.parse(since)) / 1000)) : null, label: null, priority: 'normal' });
|
||||
waiters.sort((a, b) => (a.priority === 'gate' ? 0 : 1) - (b.priority === 'gate' ? 0 : 1) || String(a.since || '').localeCompare(String(b.since || '')));
|
||||
running.sort((a, b) => String(a.started_at).localeCompare(String(b.started_at)));
|
||||
return { running, waiters, compilers };
|
||||
}
|
||||
function parseLabelSafe(line) { const r = /: (.*)$/.exec(line); return { command: r ? r[1] : line }; }
|
||||
|
||||
function sccache() {
|
||||
const bin = existsSync('/home/build/.cargo/bin/sccache') ? '/home/build/.cargo/bin/sccache' : 'sccache';
|
||||
|
|
@ -121,6 +131,7 @@ export function collect() {
|
|||
for (const r of p.running) {
|
||||
const h = s.held.find(h => h.worktree === r.worktree && (!h.crate || h.crate === r.crate || !r.crate));
|
||||
r.slot = h ? h.slot : null; r.agent = h ? h.agent : null; r.queued_at = h ? h.since : null; r.waited_s = h ? h.waited_s : null;
|
||||
if (h && h.class) r.kind = h.class; r.nice = h ? h.nice : null; r.cores = h ? h.cores : null;
|
||||
}
|
||||
const logText = read(LOG);
|
||||
const recent = parseBuildsJsonl(logText, { limit: 200 });
|
||||
|
|
|
|||
|
|
@ -65,15 +65,28 @@ export function parseSlotLine(line, { day, now } = {}) {
|
|||
const d = day || new Date(nowMs).toISOString().slice(0, 10);
|
||||
let since = Date.parse(`${d}T${r[2]}:${r[3]}:${r[4]}Z`);
|
||||
if (since > nowMs + 60_000) since -= 86_400_000;
|
||||
const label = r[6].trim();
|
||||
const agent = (/;\s*agent=([^;]+)\s*$/.exec(label) || [])[1] || null;
|
||||
return { pid: Number(r[1]), since: new Date(since).toISOString().replace(/\.\d{3}Z$/, 'Z'), waited_s: Number(r[5]), label: agent ? label.replace(/;\s*agent=[^;]+\s*$/, '').trim() : label, agent, ...splitLabel(label) };
|
||||
const parts = splitLabelParts(r[6]); const sl = splitLabel(parts.label);
|
||||
return { pid: Number(r[1]), since: new Date(since).toISOString().replace(/\.\d{3}Z$/, 'Z'), waited_s: Number(r[5]), label: parts.label, agent: parts.agent, class: parts.class, nice: parts.nice, cores: parts.cores, ...sl, kind: parts.class || deriveKind({ command: sl.command, crate: sl.crate, target: sl.target }) };
|
||||
}
|
||||
|
||||
/// The label's trailing segments (7 October 2026, build box scheduling classes): "<label>; kind=suite nice=10 cores=32; agent=x".
|
||||
/// Returns the bare label and the three class fields; absent segments give null.
|
||||
export function splitLabelParts(full) {
|
||||
let label = String(full || '').trim(), agent = null, cls = null, nice = null, cores = null;
|
||||
const segs = label.split(/;\s*/); label = segs.shift().trim();
|
||||
for (const seg of segs) {
|
||||
const a = /^agent=(.+)$/.exec(seg.trim()); if (a) { agent = a[1].trim(); continue; }
|
||||
const k = /\bkind=([\w-]+)/.exec(seg); const n = /\bnice=(\d+)/.exec(seg); const c = /\bcores=(\d+)/.exec(seg);
|
||||
if (k || n || c) { if (k) cls = k[1]; if (n) nice = Number(n[1]); if (c) cores = Number(c[1]); continue; }
|
||||
label += '; ' + seg.trim(); // a stray semicolon inside the command itself
|
||||
}
|
||||
return { label, agent, class: cls, nice, cores };
|
||||
}
|
||||
|
||||
/// "igneum-wt-x/vendor/igneum-node-bs cross x86_64-pc-windows-gnu" or "igneum-wt-x/app/igneum-app cargo build --release ..."
|
||||
/// -> { worktree, crate, command, target }. A Mac with-lock label is a plain command line: worktree and crate are null.
|
||||
export function splitLabel(label) {
|
||||
const s = String(label || '').replace(/;\s*agent=[^;]+\s*$/, '').trim();
|
||||
const s = splitLabelParts(label).label;
|
||||
const r = /^(igneum(?:-wt-[\w.-]+)?)\/(\S+) (cargo .*|cross .*)$/.exec(s) || /^(igneum(?:-wt-[\w.-]+)?)\/(\S+) (.*)$/.exec(s);
|
||||
if (!r) return { worktree: null, crate: null, command: s || null, target: targetOf(s) };
|
||||
let command = r[3];
|
||||
|
|
@ -95,6 +108,7 @@ export function deriveKind({ kind, command, crate, tool, target } = {}) {
|
|||
const sub = (/\bcargo\s+(\S+)/.exec(cmd) || [])[1] || cmd.split(/\s+/)[0] || '';
|
||||
const tgt = target || targetOf(cmd);
|
||||
const win = tgt === 'x86_64-pc-windows-gnu' || tool === 'cross-remote';
|
||||
if (sub === 'bench') return 'bench';
|
||||
if (sub === 'test' || sub === 'nextest') return 'suite';
|
||||
if (sub === 'check' || sub === 'clippy') return 'check';
|
||||
if (/^vendor\//.test(cr) || /\bkaspad\b/.test(cmd)) return win ? 'node-windows' : 'node-linux';
|
||||
|
|
@ -147,6 +161,7 @@ export function normaliseBuild(o) {
|
|||
ok: Number(o.exit) === 0,
|
||||
sccache: { hits, misses, hit_rate_pct: hits + misses ? Math.round((hits / (hits + misses)) * 1000) / 10 : null, hits_total: sc.hits_total ?? null, misses_total: sc.misses_total ?? null },
|
||||
artefacts: Array.isArray(o.artefacts) ? o.artefacts : [],
|
||||
nice: Number.isFinite(o.nice) ? o.nice : null, cores: Number.isFinite(o.cores) ? o.cores : null, jobs: Number.isFinite(o.jobs) ? o.jobs : null, priority: o.priority || null,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@
|
|||
<div class="sources" id="sources"></div>
|
||||
</div>
|
||||
<script type="module">
|
||||
import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, kindLabel, fmtDur, fmtDurShort, fmtBytes, fmtBps, fmtUptime, ukTime, ago, headline } from './shape.js?v=1';
|
||||
import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, kindLabel, classLine, fmtDur, fmtDurShort, fmtBytes, fmtBps, fmtUptime, ukTime, ago, headline } from './shape.js?v=2';
|
||||
const $ = id => document.getElementById(id);
|
||||
const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c]));
|
||||
const pill = (s, cls) => `<span class="pill ${esc(cls || String(s || '').toLowerCase())}">${esc(s || '?')}</span>`;
|
||||
|
|
@ -251,7 +251,7 @@ function renderCrew() {
|
|||
|
||||
function nowItems() {
|
||||
const out = [];
|
||||
for (const r of (D.box && D.box.running) || []) out.push({ where: D.box.name, slot: r.slot, title: `${kindLabel(r.kind)}`, sub: `${r.worktree || '?'}/${r.crate || ''}${r.agent ? ' · ' + r.agent : ''}`, cmd: r.command, started_at: r.started_at, kind: r.kind, extra: r.target || '' });
|
||||
for (const r of (D.box && D.box.running) || []) out.push({ where: D.box.name, slot: r.slot, title: `${kindLabel(r.kind)}`, sub: `${r.worktree || '?'}/${r.crate || ''}${r.agent ? ' · ' + r.agent : ''}`, cmd: r.command, started_at: r.started_at, kind: r.kind, extra: [r.target || '', classLine(r)].filter(Boolean).join(' · ') });
|
||||
for (const h of (D.mac && D.mac.slots.held) || []) out.push({ where: D.mac.name, slot: h.slot, title: h.slot === 'measure' ? 'measurement' : h.slot.startsWith('run') ? 'functional run' : kindLabel(h.kind), sub: h.worktree || 'this Mac', cmd: h.command || h.label, started_at: h.since, kind: h.kind, extra: h.waited_s ? `waited ${fmtDur(h.waited_s)}` : '' });
|
||||
for (const pc of (D.pcs && D.pcs.machines) || []) if (pc.running) out.push({ where: pc.name, slot: 'relay job', title: `${pc.running.kind} ${pc.running.job}`, sub: pc.running.title || pc.machine || '', cmd: pc.running.stage ? `stage ${pc.running.stage}` : 'reports every 5 min while it runs', started_at: pc.running.started_at, kind: pc.running.kind, extra: pc.running.last_report_at ? `last report ${ago(pc.running.last_report_at)}` : '' });
|
||||
return out;
|
||||
|
|
@ -292,12 +292,12 @@ function renderBackground() {
|
|||
}
|
||||
function renderQueue() {
|
||||
const items = [];
|
||||
for (const w of (D.box && D.box.queue) || []) items.push({ where: D.box.name, title: w.label ? kindLabel(w.kind) : 'a build', sub: w.label || 'label unknown: the waiter writes none until its slot is taken', since: w.since, wait: w.waiting_s });
|
||||
for (const w of (D.box && D.box.queue) || []) items.push({ where: D.box.name, title: w.label ? kindLabel(w.kind) : 'a build', sub: w.label ? `${w.worktree || ''}${w.crate ? '/' + w.crate : ''} ${w.command || ''}`.trim() : 'label unknown: no wait file for this flock', since: w.since, wait: w.waiting_s, gate: w.priority === 'gate', extra: [w.agent || '', classLine(w)].filter(Boolean).join(' · ') });
|
||||
for (const w of (D.mac && D.mac.queue) || []) items.push({ where: D.mac.name, title: `${w.mode} slot`, sub: w.command, since: null, wait: w.waiting_s });
|
||||
for (const pc of (D.pcs && D.pcs.machines) || []) for (const q of pc.queue) items.push({ where: pc.name, title: `${q.kind} ${q.job}`, sub: q.title || '', since: null, wait: null, until: q.expires_at });
|
||||
$('queue-sub').textContent = items.length ? `${items.length} waiting` : '';
|
||||
if (!items.length) { $('queue').innerHTML = `<div class="empty">The queue is empty. On the box a second build waits on the flock of build-0 and shows here; on the Mac a with-lock.sh wait; on a PC a signed job it has not reported on yet.</div>`; return; }
|
||||
$('queue').innerHTML = items.map(i => `<div class="job"><div class="top"><div><div class="title">${esc(i.title)}</div><div class="where">${esc(i.where)}</div></div>${pill('waiting')}</div><div class="cmd" title="${esc(i.sub)}">${esc(i.sub)}</div><div class="foot"><span>${i.since ? 'since ' + t(i.since) : i.until ? 'signed, expires ' + t(i.until, true) : ''}</span><span>${i.wait !== null && i.wait !== undefined ? 'waiting ' + esc(fmtDur(i.wait)) : ''}</span></div></div>`).join('');
|
||||
$('queue').innerHTML = items.map(i => `<div class="job${i.gate ? ' running' : ''}"><div class="top"><div><div class="title">${esc(i.title)}</div><div class="where">${esc(i.where)}${i.extra ? ' · ' + esc(i.extra) : ''}</div></div>${i.gate ? pill('gate queued', 'kind') : pill('waiting')}</div><div class="cmd" title="${esc(i.sub)}">${esc(i.sub)}</div><div class="foot"><span>${i.since ? 'since ' + t(i.since) : i.until ? 'signed, expires ' + t(i.until, true) : ''}</span><span>${i.wait !== null && i.wait !== undefined ? 'waiting ' + esc(fmtDur(i.wait)) : ''}</span></div></div>`).join('');
|
||||
}
|
||||
|
||||
function doneRows() {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
// Data shaping for the worker dashboard page (workers.html loads it as a module; node --test imports the same file).
|
||||
// Pure functions over workers.json: the analytics strip, the timeline bars, the time formatting. No DOM here.
|
||||
|
||||
export const KIND_LABEL = { 'node-linux': 'node Linux', 'node-windows': 'node Windows', app: 'app', 'app-windows': 'app Windows', suite: 'suite', check: 'check', prove: 'prove', pow: 'pow', other: 'other' };
|
||||
export const KIND_LABEL = { 'node-linux': 'node Linux', 'node-windows': 'node Windows', app: 'app', 'app-windows': 'app Windows', suite: 'suite', bench: 'bench', gate: 'gate', check: 'check', prove: 'prove', pow: 'pow', other: 'other' };
|
||||
/// The scheduling class line for a card: "nice 10 · 32 cores · -j 32" or '' when the line carries none (7 October 2026 classes).
|
||||
export const classLine = b => [Number.isFinite(b && b.nice) ? `nice ${b.nice}` : '', Number.isFinite(b && b.cores) ? `${b.cores === 96 ? 'all 96' : 'last ' + b.cores} cores` : '', Number.isFinite(b && b.jobs) ? `-j ${b.jobs}` : '', b && b.priority === 'gate' ? 'gate priority' : ''].filter(Boolean).join(' · ');
|
||||
export const kindLabel = k => KIND_LABEL[k] || k || 'other';
|
||||
|
||||
export function quantile(sorted, q) {
|
||||
|
|
@ -63,6 +65,7 @@ export function closingLines(b) {
|
|||
if (b.command) out.push(`$ ${b.command}`);
|
||||
if (b.worktree) out.push(`${b.worktree}/${b.crate || ''}${b.branch ? ` (${b.branch}${b.sha ? ' ' + String(b.sha).slice(0, 8) : ''})` : ''}`);
|
||||
out.push(`exit ${b.exit ?? '?'} after ${fmtDur(b.secs)}${b.wait_s ? `, waited ${fmtDur(b.wait_s)} for a slot` : ''}${b.slot !== undefined && b.slot !== null ? ` on build-${b.slot}` : ''}`);
|
||||
if (classLine(b)) out.push(`class: ${classLine(b)}`);
|
||||
if (b.compiles !== undefined) out.push(`${b.compiles} compiles, sccache ${b.sccache && b.sccache.hit_rate_pct !== null ? b.sccache.hit_rate_pct + '% hits' : 'no stats'} (${b.sccache ? `${b.sccache.hits} hit, ${b.sccache.misses} miss` : ''})`);
|
||||
if (Array.isArray(b.load_end) && b.load_end.length) out.push(`load at the end ${b.load_end.join(' ')}`);
|
||||
for (const a of b.artefacts || []) out.push(`${a.path} ${fmtBytes(a.bytes)} sha256 ${String(a.sha256 || '').slice(0, 16)}…`);
|
||||
|
|
|
|||
|
|
@ -152,3 +152,21 @@ test('shape: UK local time with the UTC in the tooltip', () => {
|
|||
assert.equal(ukTime('2026-12-06T17:34:00Z').text, '17:34'); // GMT in December
|
||||
assert.deepEqual(ukTime(null), { text: '', utc: '' });
|
||||
});
|
||||
|
||||
test('scheduling classes (7 Oct 2026): kind, nice, cores on the slot line; wait and gate files; JSONL fields', async () => {
|
||||
const { splitLabelParts } = await import('../lib.mjs');
|
||||
const l = parseSlotLine('pid 7 since 12:40:00Z waited 3 s: igneum-wt-build-server/vendor/igneum-node-bs cargo test -p kaspa-build-info --lib; kind=suite nice=10 cores=32; agent=build-server', { day: '2026-10-07', now: '2026-10-07T13:00:00Z' });
|
||||
assert.equal(l.kind, 'suite'); assert.equal(l.nice, 10); assert.equal(l.cores, 32); assert.equal(l.agent, 'build-server');
|
||||
assert.equal(l.label, 'igneum-wt-build-server/vendor/igneum-node-bs cargo test -p kaspa-build-info --lib');
|
||||
assert.equal(l.command, 'cargo test -p kaspa-build-info --lib');
|
||||
const g = parseSlotLine('pid 8 since 12:41:00Z waited 0 s: igneum-wt-build-server/vendor/igneum-node-bs cargo check -p kaspa-build-info; kind=gate nice=0 cores=96; agent=build-server', { day: '2026-10-07' });
|
||||
assert.equal(g.kind, 'gate'); assert.equal(g.nice, 0); assert.equal(g.cores, 96);
|
||||
// the old line shape still parses and derives its kind
|
||||
const o = parseSlotLine('pid 9 since 12:41:00Z waited 0 s: igneum-wt-x/app/igneum-app cargo build --release; agent=x', { day: '2026-10-07' });
|
||||
assert.equal(o.kind, 'app'); assert.equal(o.nice, null); assert.equal(o.cores, null); assert.equal(o.agent, 'x');
|
||||
assert.deepEqual(splitLabelParts('plain cargo build'), { label: 'plain cargo build', agent: null, class: null, nice: null, cores: null });
|
||||
const b = parseBuildsJsonl('{"start":"2026-10-07T12:00:00Z","end":"2026-10-07T12:01:00Z","exit":0,"command":"cargo test -p x","crate":"vendor/igneum-node-bs","nice":10,"cores":32,"jobs":32,"priority":"normal"}').rows[0];
|
||||
assert.equal(b.kind, 'suite'); assert.equal(b.nice, 10); assert.equal(b.cores, 32); assert.equal(b.jobs, 32); assert.equal(b.priority, 'normal');
|
||||
assert.ok(closingLines(b).some(x => x === 'class: nice 10 · last 32 cores · -j 32'));
|
||||
assert.equal(deriveKind({ command: 'cargo bench -p igneum-pow' }), 'bench');
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue