igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

workers page: load by pid per box (9 October 2026, the founder's order): collect.mjs lists every job from /proc beside the cargo builds (node, miner, prover, worker, suite, fuzz, harness, service, chip, archive; counts per kind, browsers counted only), the page renders the list on each box card and reads busy from it; the mini card carries its miner line (app version, node DAA and state, rate, last accepted block, node pid; "no miner report" when absent); the PC cards read their report age honestly (STALE over ten minutes) and show the intake report fields when a PC has posted one.

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-09 08:13:33 +00:00
parent ee4de43219
commit 53a4e84440
2 changed files with 91 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
tools/workers/collect.mjs
View file

@ -66,6 +66,52 @@ function slots(now) {
const files = safeList(LOCKS).map(n => ({ name: n, text: read(join(LOCKS, n)), held: /^(build-\d+|run-\d+|build|measure)$/.test(n) ? flockHeld(join(LOCKS, n)) : undefined }));
return parseLockDir(files, { now, aliveFn: alive });
}
// ---- every job by pid, not only the cargo builds (the founder's order, 9 October 2026 09:15 UK: the page reads node, suite and
// fuzz load by pid per box). A process is listed when its command is one of the known binaries (igneumd, igneum-miner, the prover
// pair, the GPU workers, the chip tools), a test binary under a cargo target dir (a suite; "fuzz" in its arguments makes it a fuzz
// run), a script (bash, sh, node, python) whose path lies under /srv, /tmp or /home/build (a harness, a reader, a collector), or
// an archive stream into /srv/archive. Kernel threads, sshd, systemd and the shells of this collector are not jobs. Fields are read
// from /proc: the start time from stat, the resident set from statm, the CPU seconds from utime + stime. Capped at 48 by RSS.
const PROC_SKIP = new Set(['sshd', 'systemd', 'systemd-journal', 'systemd-logind', 'systemd-resolve', 'systemd-timesyn', 'systemd-network', 'systemd-udevd', 'cron', 'dbus-daemon', 'agetty', 'login', 'kthreadd', 'sleep', 'ps', 'awk', 'grep', 'sed', 'cut', 'sort', 'tail', 'head', 'cat', 'flock', 'sccache', 'caddy', 'gitea', 'docker-proxy', 'dockerd', 'containerd', 'containerd-shim', 'unattended-upgr', 'polkitd', 'rsyslogd', 'chronyd', 'multipathd', 'snapd', 'qemu-ga', 'irqbalance', 'packagekitd', 'fwupd', 'collect.mjs']);
function procKind(comm, args) {
const a0 = args[0] || '', line = args.join(' ');
if (comm === 'igneumd') return 'node';
if (comm === 'igneum-miner') return 'miner';
if (/^igneum-prove/.test(comm)) return 'prover';
if (/^igneum-worker/.test(comm)) return 'worker';
if (/^(openroad|kepler-formal|yosys|klayout|magic|ngspice)/.test(comm)) return 'chip';
if (/\/target[^ ]*\/deps\//.test(a0) || /\/target\/[^ ]*\/(release|debug)\//.test(a0)) return /\bfuzz\b/i.test(line) ? 'fuzz' : 'suite';
if (/^(rsync|tar|zstd|pigz|gzip|xz)$/.test(comm) && /\/srv\/archive/.test(line)) return 'archive';
if (/^(headless_shell|chrome|chromium)/.test(comm)) return 'browser';
if (/^(bash|sh|zsh|node|python3?|perl)$/.test(comm)) {
const script = args.find((x, i) => i > 0 && /^\/(srv|tmp|home\/build)\//.test(x) && !/^-/.test(x)) || (/^\/(srv|tmp|home\/build)\//.test(a0) ? a0 : null);
if (!script) return null;
if (/harness|canary|roll|heal|fuzz|suite|gate/.test(script)) return 'harness';
if (/reader|collect|merge|observer|intake|keep-alive|serve/.test(script)) return 'service';
return 'script';
}
if (/^\/(srv|home\/build)\//.test(a0) && !/\/\.cargo\/|\/\.rustup\//.test(a0)) return 'other';
return null;
}
function jobs(bootSec, clk, now, selfPid) {
const out = [], counts = {};
for (const d of safeList('/proc')) {
if (!/^\d+$/.test(d)) continue;
const pid = Number(d); if (pid === selfPid) continue;
const comm = read(`/proc/${d}/comm`).trim(); if (!comm || PROC_SKIP.has(comm)) continue;
const args = read(`/proc/${d}/cmdline`).split('\0').filter(Boolean); if (!args.length) continue; // a kernel thread
const kind = procKind(comm, args); if (!kind) continue;
const stat = read(`/proc/${d}/stat`); const rest = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
const utime = Number(rest[11]) || 0, stime = Number(rest[12]) || 0, startTicks = Number(rest[19]);
const started = Number.isFinite(startTicks) ? (bootSec + startTicks / clk) * 1000 : null;
const rssPages = Number((read(`/proc/${d}/statm`).split(' ')[1]) || 0);
counts[kind] = (counts[kind] || 0) + 1;
if (kind === 'browser') continue; // counted, not listed: a site gate spawns a dozen renderer processes
out.push({ pid, kind, comm, cmd: args.join(' ').slice(0, 160), started_at: started ? iso(started) : null, elapsed_s: started ? Math.max(0, Math.round((now - started) / 1000)) : null, rss_mb: Math.round(rssPages * 4096 / 1048576), cpu_s: Math.round((utime + stime) / clk) });
}
out.sort((a, b) => b.rss_mb - a.rss_mb);
return { list: out.slice(0, 48), counts };
}
function procs(bootSec, clk, now) {
const running = [], waiters = []; let compilers = 0;
const flockParents = new Map(); // ppid -> since, from the flock -w processes the waiters run
@ -127,6 +173,7 @@ export function collect() {
const bootSec = Math.round((now - m.uptime_s * 1000) / 1000);
const s = slots(now);
const p = procs(bootSec, m.clk, now);
const j = jobs(bootSec, m.clk, now, process.pid);
// attach the slot holder to its build (the label names the worktree and crate the cargo runs in)
for (const r of p.running) {
const h = s.held.find(h => h.worktree === r.worktree && (!h.crate || h.crate === r.crate || !r.crate));
@ -157,6 +204,8 @@ export function collect() {
running: p.running,
queue: p.waiters,
compilers: p.compilers,
jobs: j.list, // every job by pid (node, miner, prover, worker, suite, fuzz, harness, service, chip, archive, other)
jobs_by_kind: j.counts, // counts per kind, browsers counted here only
recent: recent.rows,
log: { path: LOG, present: !!logText, bad_lines: recent.bad, total: recent.total, note: logText ? null : 'builds.jsonl is not written yet: the build-server agent adds the append to bs_remote_run; until then this lane fills from nothing' },
},

45
tools/workers/page/workers.html
View file

@ -150,6 +150,11 @@
.sources b { color: var(--good); font-weight: 500; } .sources b.no { color: var(--bad); } .sources b.warn { color: var(--warn); }
a:focus-visible, button:focus-visible { outline: 2px solid var(--ember-2); outline-offset: 2px; }
@media (prefers-reduced-motion: reduce) { .live b, .server.hot::after, .job .pulse, .cores i.hot::after { animation: none !important; } .cores i::after, .g .arc { transition: none; } }
.jobs { margin: 8px 0 2px; font-size: 12px; line-height: 1.5; }
.jobshead { color: var(--ember, #f2541b); margin-bottom: 2px; }
.jobrow { display: flex; gap: 10px; flex-wrap: wrap; border-top: 1px solid rgba(255,255,255,.06); padding: 2px 0; }
.jobrow b { min-width: 56px; }
.muted { opacity: .7; }
</style></head><body>
<div class="wrap">
<header>
@ -218,6 +223,26 @@ function boxState(b) { // live | down | provisioning, with the line the cards
}
const buildHeld = b => (b.slots && b.slots.held || []).filter(h => /^build-\d+$/.test(h.slot)).length;
const boxesOf = d => (Array.isArray(d.boxes) && d.boxes.length ? d.boxes : d.box ? [{ ...d.box, source: d.sources && d.sources.box, headline: d.headline }] : []);
// load by pid (9 October 2026): every job the box's collector lists from /proc (node, miner, prover, worker, suite, fuzz, harness,
// service, chip, archive), not only the cargo builds; counts per kind, then the list with pid, age and resident set
const JOB_ORDER = ['node', 'fuzz', 'suite', 'harness', 'prover', 'miner', 'worker', 'chip', 'archive', 'service', 'script', 'other', 'browser'];
function jobsSummary(b) {
const c = (b && b.jobs_by_kind) || {}; const parts = JOB_ORDER.filter(k => c[k]).map(k => `${c[k]} ${k}${c[k] === 1 ? '' : (k === 'browser' ? 's' : k.endsWith('s') ? '' : 's')}`);
return parts.join(', ');
}
function jobsBlock(b) {
const jobs = (b && b.jobs) || []; const sum = jobsSummary(b);
if (!jobs.length && !sum) return `<div class="jobs"><span class="muted">load by pid: nothing but the collector (no node, suite, fuzz or harness process on this box)</span></div>`;
const rows = jobs.slice(0, 14).map(j => `<div class="jobrow"><b>${esc(j.kind)}</b> <span>pid ${j.pid}</span> <span>${esc(j.comm)}</span> <span>${j.elapsed_s === null ? '' : esc(fmtDurShort(j.elapsed_s)) + ' up'}</span> <span>${j.rss_mb} MB</span> <span class="muted" title="${esc(j.cmd)}">${esc(j.cmd.slice(0, 96))}</span></div>`).join('');
return `<div class="jobs"><div class="jobshead">load by pid: ${esc(sum || 'none')}${jobs.length > 14 ? ` (${jobs.length - 14} more in workers.json)` : ''}</div>${rows}</div>`;
}
function miniMinerLine(m) {
const x = m && m.miner; if (!x) return 'no miner report';
const daa = x.node_daa === null || x.node_daa === undefined ? '?' : Number(x.node_daa).toLocaleString('en-GB');
const rate = x.rate_mhs === null || x.rate_mhs === undefined ? '0 MH/s' : `${Number(x.rate_mhs).toFixed(1)} MH/s`;
const last = x.last_accepted_at ? String(x.last_accepted_at).replace(/^\d{4}-\d{2}-\d{2}T/, '') : 'none';
return `igneum-app ${esc(x.app_version || '?')} · node DAA ${daa} (${esc(x.node_state || (x.synced ? 'synced' : 'unknown'))}) · ${rate} ${esc(x.mining || '')} · last block ${esc(last)} · node pid ${x.node_pid ?? '?'}`;
}
function serverSection(b, i) {
const id = `srv${i}`;
const st = boxState(b);
@ -232,6 +257,7 @@ function serverSection(b, i) {
<div class="facts">${[...temps, b.net ? `net ↓<b>${esc(fmtBps(b.net.rx_bps))}</b> ↑<b>${esc(fmtBps(b.net.tx_bps))}</b>` : '', `<span title="${esc(b.collected_at)}" ${stale ? 'style="color:var(--warn)"' : ''}>${stale ? 'STALE, ' : ''}read ${esc(ago(b.collected_at))}</span>`].filter(Boolean).map(x => `<span>${x}</span>`).join('')}</div></div>
<div class="cores" style="grid-template-columns:repeat(${n > 64 ? 48 : n > 32 ? 32 : n}, 1fr)" aria-label="${n} cores, busy share per core">${Array.from({ length: n }, (_, i) => { const p = per[i] ?? 0; return `<i class="${p >= 50 ? 'hot' : ''}" style="--h:${Math.max(2, p)}%" title="core ${i}: ${p}%"></i>`; }).join('')}</div>
<div class="corelabel"><span>${n} cores, ${b.cpu ? b.cpu.cores_busy : 0} busy</span><span>${b.cpu ? `${b.cpu.busy_pct}% of the box over the last second, ${b.compilers || 0} compilers running` : ''}</span></div>
${jobsBlock(b)}
<div class="gauges">
<div class="g">${arc(b.cpu ? b.cpu.busy_pct : 0)}<div><div class="l">CPU</div><div class="v">${b.cpu ? b.cpu.busy_pct : 0}%</div><div class="s">load 1 min ${b.load ? Number(b.load[0]).toFixed(1) : '?'} of ${b.cores}</div></div></div>
<div class="g">${arc(m.used_pct, tone(m.used_pct))}<div><div class="l">Memory</div><div class="v">${m.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes((m.used_kb || 0) * 1024))} of ${esc(fmtBytes((m.total_kb || 0) * 1024))}</div></div></div>
@ -263,7 +289,7 @@ function renderCrew() {
const st = boxState(bx);
if (st.kind === 'down') { cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx.name)}</div><div class="meta">build server</div></div>${pill('down', 'error')}</div><div class="doing">${esc(st.note)}</div></div>`); continue; }
if (st.kind === 'provisioning' && !st.live) { cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx && bx.name || 'build server')}</div><div class="meta">build server</div></div>${pill('provisioning', 'queued')}</div><div class="doing">${esc(st.note)}</div></div>`); continue; }
if (bx && bx.cores) cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx.name)}</div><div class="meta">Hetzner, ${bx.cores} threads, ${esc(fmtBytes((bx.mem && bx.mem.total_kb || 0) * 1024))}, RAID 1 NVMe</div></div>${pill(bx.running && bx.running.length ? 'building' : 'idle', bx.running && bx.running.length ? 'running' : '')}</div><div class="doing">${bx.running && bx.running.length ? bx.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('<br>') : (st.kind === 'provisioning' ? esc(st.note) : 'Slot free. The next build-remote.sh or cross-remote.sh routed here takes it.')}</div>${slotBar(bx.slots, bx.slots ? bx.slots.count : 1)}<div class="foot"><span>${bx.recent ? bx.recent.length : 0} builds logged</span><span>${esc(ago(bx.collected_at))}</span></div></div>`);
if (bx && bx.cores) cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx.name)}</div><div class="meta">Hetzner, ${bx.cores} threads, ${esc(fmtBytes((bx.mem && bx.mem.total_kb || 0) * 1024))}, RAID 1 NVMe</div></div>${pill(bx.running && bx.running.length ? 'building' : jobsSummary(bx) ? 'busy' : 'idle', bx.running && bx.running.length ? 'running' : '')}</div><div class="doing">${bx.running && bx.running.length ? bx.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('<br>') : (st.kind === 'provisioning' ? esc(st.note) : 'Slot free. The next build-remote.sh or cross-remote.sh routed here takes it.')}</div>${slotBar(bx.slots, bx.slots ? bx.slots.count : 1)}<div class="foot"><span>${bx.recent ? bx.recent.length : 0} builds logged</span><span>${esc(ago(bx.collected_at))}</span></div></div>`);
else cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx && bx.name || 'build server')}</div><div class="meta">build server</div></div>${pill('unreachable', 'error')}</div><div class="doing">${esc(bx && bx.source && bx.source.error || 'no facts from this box')}</div></div>`);
}
if (mac) {
@ -272,10 +298,23 @@ function renderCrew() {
cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(mac.name)}</div><div class="meta">this MacBook, ${mac.build_slots} build slot${mac.build_slots === 1 ? '' : 's'}, 3 run slots</div></div>${pill(meas.length ? 'measuring' : builds.length ? 'building' : runs.length ? 'running' : 'idle', held.length ? 'running' : '')}</div><div class="doing">${doing}</div><div class="slots">${['build-0', 'build-1', 'build-2', 'run-0', 'run-1', 'run-2', 'measure'].map(nm => { const h = held.find(x => x.slot === nm); return `<i class="${h ? (nm === 'measure' ? 'measure' : nm.startsWith('run') ? 'run' : 'held') : ''}" title="${esc(nm)}${h ? ': ' + esc(h.label) : ': free'}"></i>`; }).join('')}</div><div class="foot"><span>${mac.queue.length ? mac.queue.length + ' waiting for a slot' : 'nobody waiting'}</span><span>${esc(ago(mac.collected_at))}</span></div></div>`);
} else cards.push(`<div class="w"><div class="top"><div><div class="card">MacBook-Pro</div><div class="meta">this MacBook</div></div>${pill('no push', 'error')}</div><div class="doing">${MAC_COPY}</div></div>`);
{ const m = miniLive(); const held = m && m.slots && m.slots.held || [];
cards.push(`<div class="w"><div class="top"><div><div class="card">${MINI.name}</div><div class="meta">${MINI.label}</div></div>${pill(!m ? 'no report' : held.length ? 'building' : 'idle', !m ? 'queued' : held.length ? 'running' : '')}</div><div class="doing">${!m ? 'no report yet' : held.length ? held.map(h => esc(`${h.slot}: ${h.worktree || h.label || ''}`)).join('<br>') : 'macOS binaries build here from tonight; slot free.'}</div>${m && m.slots ? slotBar(m.slots, m.slots.count || 1) : ''}<div class="foot"><span>${m ? `${(m.recent || []).length} builds logged` : 'its collector pushes to build-1'}</span><span>${m ? esc(ago(m.collected_at)) : ''}</span></div></div>`); }
cards.push(`<div class="w"><div class="top"><div><div class="card">${MINI.name}</div><div class="meta">${MINI.label}</div></div>${pill(!m ? 'no report' : held.length ? 'building' : 'idle', !m ? 'queued' : held.length ? 'running' : '')}</div><div class="doing">${!m ? 'no report yet' : (esc(miniMinerLine(m)) + '<br>') + (held.length ? held.map(h => esc(`${h.slot}: ${h.worktree || h.label || ''}`)).join('<br>') : 'macOS binaries build here from tonight; slot free.')}</div>${m && m.slots ? slotBar(m.slots, m.slots.count || 1) : ''}<div class="foot"><span>${m ? `${(m.recent || []).length} builds logged` : 'its collector pushes to build-1'}</span><span>${m ? esc(ago(m.collected_at)) : ''}</span></div></div>`); }
const pcsAt = D.pcs && D.pcs.collected_at ? Date.parse(D.pcs.collected_at) : null; const pcsAge = pcsAt ? Math.round((Date.now() - pcsAt) / 1000) : null;
const pcsAgeText = pcsAge === null ? 'report age unknown' : `report ${fmtDurShort(pcsAge)} old${pcsAge > 600 ? ', STALE' : ''}`;
// a PC's five-minute report through the intake (9 October 2026): the known fields on one line, the rest of the report left in workers.json
const pcReportLine = pc => { const r = pc.report; if (!r || !Object.keys(r).length) return ''; const f = [];
if (r.app_version) f.push(`igneum-app ${esc(String(r.app_version))}`);
if (r.node_daa !== undefined && r.node_daa !== null) f.push(`node DAA ${Number(r.node_daa).toLocaleString('en-GB')}${r.node_state ? ' (' + esc(String(r.node_state)) + ')' : r.synced === false ? ' (behind)' : ''}`);
if (r.rate_mhs !== undefined && r.rate_mhs !== null) f.push(`${Number(r.rate_mhs).toFixed(1)} MH/s`);
if (Array.isArray(r.cards)) f.push(r.cards.map(c => `${esc(String(c.name || c.id || 'card'))} ${c.rate_mhs === undefined || c.rate_mhs === null ? '?' : Number(c.rate_mhs).toFixed(1)} MH/s`).join(', '));
if (r.last_accepted_at) f.push(`last block ${esc(String(r.last_accepted_at).replace(/^\d{4}-\d{2}-\d{2}T/, ''))}`);
if (r.agent_pid || r.node_pid) f.push(`pids agent ${r.agent_pid ?? '?'} node ${r.node_pid ?? '?'}`);
return f.join(' · '); };
for (const pc of pcs) {
const r = pc.running;
cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(pc.name)}</div><div class="meta">${esc(pc.machine || pc.id)} · ${esc(pc.role)}</div></div>${pill(r ? 'running' : pc.queue.length ? 'queued' : 'idle', r ? 'running' : pc.queue.length ? 'queued' : '')}</div><div class="doing">${r ? `${esc(r.kind)} ${esc(r.job)}${r.title ? '<br>' + esc(r.title) : ''}${r.stage ? `<br><span class="pill">stage ${esc(r.stage)}</span>` : ''}` : pc.queue.length ? `Next: ${esc(pc.queue[0].kind)} ${esc(pc.queue[0].job)}${pc.queue[0].title ? ', ' + esc(pc.queue[0].title) : ''}` : esc(pc.note || 'idle on jobs; the relay polls every 10 min')}</div><div class="foot"><span>${pc.recent.length} reports kept</span><span>${pc.last_report_at ? 'last report ' + esc(ago(pc.last_report_at)) : 'no report in 7 days'}</span></div></div>`);
const pcAge = pc.report_age_s === undefined || pc.report_age_s === null ? pcsAge : pc.report_age_s;
const pcAgeText = pcAge === null ? 'report age unknown' : `report ${fmtDurShort(pcAge)} old${pcAge > 600 ? ', STALE' : ''}${pc.source === 'intake' ? ', from the PC' : ''}`;
cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(pc.name)}</div><div class="meta">${esc(pc.machine || pc.id)} · ${esc(pc.role)} · <span ${pcAge !== null && pcAge > 600 ? 'style="color:var(--warn)"' : ''}>${esc(pcAgeText)}</span></div></div>${pill(r ? 'running' : pc.queue.length ? 'queued' : 'idle', r ? 'running' : pc.queue.length ? 'queued' : '')}</div><div class="doing">${pcReportLine(pc) ? pcReportLine(pc) + '<br>' : ''}${r ? `${esc(r.kind)} ${esc(r.job)}${r.title ? '<br>' + esc(r.title) : ''}${r.stage ? `<br><span class="pill">stage ${esc(r.stage)}</span>` : ''}` : pc.queue.length ? `Next: ${esc(pc.queue[0].kind)} ${esc(pc.queue[0].job)}${pc.queue[0].title ? ', ' + esc(pc.queue[0].title) : ''}` : esc(pc.note || 'idle on jobs; the relay polls every 10 min')}</div><div class="foot"><span>${pc.recent.length} reports kept</span><span>${pc.last_report_at ? 'last report ' + esc(ago(pc.last_report_at)) : 'no report in 7 days'}</span></div></div>`);
}
if (!pcs.length) cards.push(`<div class="w"><div class="top"><div><div class="card">PC 1 and PC 2</div><div class="meta">relay jobs</div></div>${pill('no data', 'error')}</div><div class="doing">${esc(D.sources && D.sources.pcs && D.sources.pcs.error || 'the Mac pusher reads the relay intake; nothing has arrived')}</div></div>`);
$('crew').innerHTML = cards.join('');