diff --git a/tools/workers/collect.mjs b/tools/workers/collect.mjs index fad99beb4..78c37cc1d 100644 --- a/tools/workers/collect.mjs +++ b/tools/workers/collect.mjs @@ -66,6 +66,52 @@ function slots(now) { const files = safeList(LOCKS).map(n => ({ name: n, text: read(join(LOCKS, n)), held: /^(build-\d+|run-\d+|build|measure)$/.test(n) ? flockHeld(join(LOCKS, n)) : undefined })); return parseLockDir(files, { now, aliveFn: alive }); } +// ---- every job by pid, not only the cargo builds (the founder's order, 9 October 2026 09:15 UK: the page reads node, suite and +// fuzz load by pid per box). A process is listed when its command is one of the known binaries (igneumd, igneum-miner, the prover +// pair, the GPU workers, the chip tools), a test binary under a cargo target dir (a suite; "fuzz" in its arguments makes it a fuzz +// run), a script (bash, sh, node, python) whose path lies under /srv, /tmp or /home/build (a harness, a reader, a collector), or +// an archive stream into /srv/archive. Kernel threads, sshd, systemd and the shells of this collector are not jobs. Fields are read +// from /proc: the start time from stat, the resident set from statm, the CPU seconds from utime + stime. Capped at 48 by RSS. +const PROC_SKIP = new Set(['sshd', 'systemd', 'systemd-journal', 'systemd-logind', 'systemd-resolve', 'systemd-timesyn', 'systemd-network', 'systemd-udevd', 'cron', 'dbus-daemon', 'agetty', 'login', 'kthreadd', 'sleep', 'ps', 'awk', 'grep', 'sed', 'cut', 'sort', 'tail', 'head', 'cat', 'flock', 'sccache', 'caddy', 'gitea', 'docker-proxy', 'dockerd', 'containerd', 'containerd-shim', 'unattended-upgr', 'polkitd', 'rsyslogd', 'chronyd', 'multipathd', 'snapd', 'qemu-ga', 'irqbalance', 'packagekitd', 'fwupd', 'collect.mjs']); +function procKind(comm, args) { + const a0 = args[0] || '', line = args.join(' '); + if (comm === 'igneumd') return 'node'; + if (comm === 'igneum-miner') return 'miner'; + if (/^igneum-prove/.test(comm)) return 'prover'; + if (/^igneum-worker/.test(comm)) return 'worker'; + if (/^(openroad|kepler-formal|yosys|klayout|magic|ngspice)/.test(comm)) return 'chip'; + if (/\/target[^ ]*\/deps\//.test(a0) || /\/target\/[^ ]*\/(release|debug)\//.test(a0)) return /\bfuzz\b/i.test(line) ? 'fuzz' : 'suite'; + if (/^(rsync|tar|zstd|pigz|gzip|xz)$/.test(comm) && /\/srv\/archive/.test(line)) return 'archive'; + if (/^(headless_shell|chrome|chromium)/.test(comm)) return 'browser'; + if (/^(bash|sh|zsh|node|python3?|perl)$/.test(comm)) { + const script = args.find((x, i) => i > 0 && /^\/(srv|tmp|home\/build)\//.test(x) && !/^-/.test(x)) || (/^\/(srv|tmp|home\/build)\//.test(a0) ? a0 : null); + if (!script) return null; + if (/harness|canary|roll|heal|fuzz|suite|gate/.test(script)) return 'harness'; + if (/reader|collect|merge|observer|intake|keep-alive|serve/.test(script)) return 'service'; + return 'script'; + } + if (/^\/(srv|home\/build)\//.test(a0) && !/\/\.cargo\/|\/\.rustup\//.test(a0)) return 'other'; + return null; +} +function jobs(bootSec, clk, now, selfPid) { + const out = [], counts = {}; + for (const d of safeList('/proc')) { + if (!/^\d+$/.test(d)) continue; + const pid = Number(d); if (pid === selfPid) continue; + const comm = read(`/proc/${d}/comm`).trim(); if (!comm || PROC_SKIP.has(comm)) continue; + const args = read(`/proc/${d}/cmdline`).split('\0').filter(Boolean); if (!args.length) continue; // a kernel thread + const kind = procKind(comm, args); if (!kind) continue; + const stat = read(`/proc/${d}/stat`); const rest = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + const utime = Number(rest[11]) || 0, stime = Number(rest[12]) || 0, startTicks = Number(rest[19]); + const started = Number.isFinite(startTicks) ? (bootSec + startTicks / clk) * 1000 : null; + const rssPages = Number((read(`/proc/${d}/statm`).split(' ')[1]) || 0); + counts[kind] = (counts[kind] || 0) + 1; + if (kind === 'browser') continue; // counted, not listed: a site gate spawns a dozen renderer processes + out.push({ pid, kind, comm, cmd: args.join(' ').slice(0, 160), started_at: started ? iso(started) : null, elapsed_s: started ? Math.max(0, Math.round((now - started) / 1000)) : null, rss_mb: Math.round(rssPages * 4096 / 1048576), cpu_s: Math.round((utime + stime) / clk) }); + } + out.sort((a, b) => b.rss_mb - a.rss_mb); + return { list: out.slice(0, 48), counts }; +} function procs(bootSec, clk, now) { const running = [], waiters = []; let compilers = 0; const flockParents = new Map(); // ppid -> since, from the flock -w processes the waiters run @@ -127,6 +173,7 @@ export function collect() { const bootSec = Math.round((now - m.uptime_s * 1000) / 1000); const s = slots(now); const p = procs(bootSec, m.clk, now); + const j = jobs(bootSec, m.clk, now, process.pid); // attach the slot holder to its build (the label names the worktree and crate the cargo runs in) for (const r of p.running) { const h = s.held.find(h => h.worktree === r.worktree && (!h.crate || h.crate === r.crate || !r.crate)); @@ -157,6 +204,8 @@ export function collect() { running: p.running, queue: p.waiters, compilers: p.compilers, + jobs: j.list, // every job by pid (node, miner, prover, worker, suite, fuzz, harness, service, chip, archive, other) + jobs_by_kind: j.counts, // counts per kind, browsers counted here only recent: recent.rows, log: { path: LOG, present: !!logText, bad_lines: recent.bad, total: recent.total, note: logText ? null : 'builds.jsonl is not written yet: the build-server agent adds the append to bs_remote_run; until then this lane fills from nothing' }, }, diff --git a/tools/workers/page/workers.html b/tools/workers/page/workers.html index abb2c5465..995f841fe 100644 --- a/tools/workers/page/workers.html +++ b/tools/workers/page/workers.html @@ -150,6 +150,11 @@ .sources b { color: var(--good); font-weight: 500; } .sources b.no { color: var(--bad); } .sources b.warn { color: var(--warn); } a:focus-visible, button:focus-visible { outline: 2px solid var(--ember-2); outline-offset: 2px; } @media (prefers-reduced-motion: reduce) { .live b, .server.hot::after, .job .pulse, .cores i.hot::after { animation: none !important; } .cores i::after, .g .arc { transition: none; } } + .jobs { margin: 8px 0 2px; font-size: 12px; line-height: 1.5; } + .jobshead { color: var(--ember, #f2541b); margin-bottom: 2px; } + .jobrow { display: flex; gap: 10px; flex-wrap: wrap; border-top: 1px solid rgba(255,255,255,.06); padding: 2px 0; } + .jobrow b { min-width: 56px; } + .muted { opacity: .7; }
@@ -218,6 +223,26 @@ function boxState(b) { // live | down | provisioning, with the line the cards } const buildHeld = b => (b.slots && b.slots.held || []).filter(h => /^build-\d+$/.test(h.slot)).length; const boxesOf = d => (Array.isArray(d.boxes) && d.boxes.length ? d.boxes : d.box ? [{ ...d.box, source: d.sources && d.sources.box, headline: d.headline }] : []); +// load by pid (9 October 2026): every job the box's collector lists from /proc (node, miner, prover, worker, suite, fuzz, harness, +// service, chip, archive), not only the cargo builds; counts per kind, then the list with pid, age and resident set +const JOB_ORDER = ['node', 'fuzz', 'suite', 'harness', 'prover', 'miner', 'worker', 'chip', 'archive', 'service', 'script', 'other', 'browser']; +function jobsSummary(b) { + const c = (b && b.jobs_by_kind) || {}; const parts = JOB_ORDER.filter(k => c[k]).map(k => `${c[k]} ${k}${c[k] === 1 ? '' : (k === 'browser' ? 's' : k.endsWith('s') ? '' : 's')}`); + return parts.join(', '); +} +function jobsBlock(b) { + const jobs = (b && b.jobs) || []; const sum = jobsSummary(b); + if (!jobs.length && !sum) return `
load by pid: nothing but the collector (no node, suite, fuzz or harness process on this box)
`; + const rows = jobs.slice(0, 14).map(j => `
${esc(j.kind)} pid ${j.pid} ${esc(j.comm)} ${j.elapsed_s === null ? '' : esc(fmtDurShort(j.elapsed_s)) + ' up'} ${j.rss_mb} MB ${esc(j.cmd.slice(0, 96))}
`).join(''); + return `
load by pid: ${esc(sum || 'none')}${jobs.length > 14 ? ` (${jobs.length - 14} more in workers.json)` : ''}
${rows}
`; +} +function miniMinerLine(m) { + const x = m && m.miner; if (!x) return 'no miner report'; + const daa = x.node_daa === null || x.node_daa === undefined ? '?' : Number(x.node_daa).toLocaleString('en-GB'); + const rate = x.rate_mhs === null || x.rate_mhs === undefined ? '0 MH/s' : `${Number(x.rate_mhs).toFixed(1)} MH/s`; + const last = x.last_accepted_at ? String(x.last_accepted_at).replace(/^\d{4}-\d{2}-\d{2}T/, '') : 'none'; + return `igneum-app ${esc(x.app_version || '?')} · node DAA ${daa} (${esc(x.node_state || (x.synced ? 'synced' : 'unknown'))}) · ${rate} ${esc(x.mining || '')} · last block ${esc(last)} · node pid ${x.node_pid ?? '?'}`; +} function serverSection(b, i) { const id = `srv${i}`; const st = boxState(b); @@ -232,6 +257,7 @@ function serverSection(b, i) {
${[...temps, b.net ? `net ↓${esc(fmtBps(b.net.rx_bps))} ↑${esc(fmtBps(b.net.tx_bps))}` : '', `${stale ? 'STALE, ' : ''}read ${esc(ago(b.collected_at))}`].filter(Boolean).map(x => `${x}`).join('')}
${Array.from({ length: n }, (_, i) => { const p = per[i] ?? 0; return ``; }).join('')}
${n} cores, ${b.cpu ? b.cpu.cores_busy : 0} busy${b.cpu ? `${b.cpu.busy_pct}% of the box over the last second, ${b.compilers || 0} compilers running` : ''}
+ ${jobsBlock(b)}
${arc(b.cpu ? b.cpu.busy_pct : 0)}
CPU
${b.cpu ? b.cpu.busy_pct : 0}%
load 1 min ${b.load ? Number(b.load[0]).toFixed(1) : '?'} of ${b.cores}
${arc(m.used_pct, tone(m.used_pct))}
Memory
${m.used_pct ?? '?'}%
${esc(fmtBytes((m.used_kb || 0) * 1024))} of ${esc(fmtBytes((m.total_kb || 0) * 1024))}
@@ -263,7 +289,7 @@ function renderCrew() { const st = boxState(bx); if (st.kind === 'down') { cards.push(`
${esc(bx.name)}
build server
${pill('down', 'error')}
${esc(st.note)}
`); continue; } if (st.kind === 'provisioning' && !st.live) { cards.push(`
${esc(bx && bx.name || 'build server')}
build server
${pill('provisioning', 'queued')}
${esc(st.note)}
`); continue; } - if (bx && bx.cores) cards.push(`
${esc(bx.name)}
Hetzner, ${bx.cores} threads, ${esc(fmtBytes((bx.mem && bx.mem.total_kb || 0) * 1024))}, RAID 1 NVMe
${pill(bx.running && bx.running.length ? 'building' : 'idle', bx.running && bx.running.length ? 'running' : '')}
${bx.running && bx.running.length ? bx.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('
') : (st.kind === 'provisioning' ? esc(st.note) : 'Slot free. The next build-remote.sh or cross-remote.sh routed here takes it.')}
${slotBar(bx.slots, bx.slots ? bx.slots.count : 1)}
${bx.recent ? bx.recent.length : 0} builds logged${esc(ago(bx.collected_at))}
`); + if (bx && bx.cores) cards.push(`
${esc(bx.name)}
Hetzner, ${bx.cores} threads, ${esc(fmtBytes((bx.mem && bx.mem.total_kb || 0) * 1024))}, RAID 1 NVMe
${pill(bx.running && bx.running.length ? 'building' : jobsSummary(bx) ? 'busy' : 'idle', bx.running && bx.running.length ? 'running' : '')}
${bx.running && bx.running.length ? bx.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('
') : (st.kind === 'provisioning' ? esc(st.note) : 'Slot free. The next build-remote.sh or cross-remote.sh routed here takes it.')}
${slotBar(bx.slots, bx.slots ? bx.slots.count : 1)}
${bx.recent ? bx.recent.length : 0} builds logged${esc(ago(bx.collected_at))}
`); else cards.push(`
${esc(bx && bx.name || 'build server')}
build server
${pill('unreachable', 'error')}
${esc(bx && bx.source && bx.source.error || 'no facts from this box')}
`); } if (mac) { @@ -272,10 +298,23 @@ function renderCrew() { cards.push(`
${esc(mac.name)}
this MacBook, ${mac.build_slots} build slot${mac.build_slots === 1 ? '' : 's'}, 3 run slots
${pill(meas.length ? 'measuring' : builds.length ? 'building' : runs.length ? 'running' : 'idle', held.length ? 'running' : '')}
${doing}
${['build-0', 'build-1', 'build-2', 'run-0', 'run-1', 'run-2', 'measure'].map(nm => { const h = held.find(x => x.slot === nm); return ``; }).join('')}
${mac.queue.length ? mac.queue.length + ' waiting for a slot' : 'nobody waiting'}${esc(ago(mac.collected_at))}
`); } else cards.push(`
MacBook-Pro
this MacBook
${pill('no push', 'error')}
${MAC_COPY}
`); { const m = miniLive(); const held = m && m.slots && m.slots.held || []; - cards.push(`
${MINI.name}
${MINI.label}
${pill(!m ? 'no report' : held.length ? 'building' : 'idle', !m ? 'queued' : held.length ? 'running' : '')}
${!m ? 'no report yet' : held.length ? held.map(h => esc(`${h.slot}: ${h.worktree || h.label || ''}`)).join('
') : 'macOS binaries build here from tonight; slot free.'}
${m && m.slots ? slotBar(m.slots, m.slots.count || 1) : ''}
${m ? `${(m.recent || []).length} builds logged` : 'its collector pushes to build-1'}${m ? esc(ago(m.collected_at)) : ''}
`); } + cards.push(`
${MINI.name}
${MINI.label}
${pill(!m ? 'no report' : held.length ? 'building' : 'idle', !m ? 'queued' : held.length ? 'running' : '')}
${!m ? 'no report yet' : (esc(miniMinerLine(m)) + '
') + (held.length ? held.map(h => esc(`${h.slot}: ${h.worktree || h.label || ''}`)).join('
') : 'macOS binaries build here from tonight; slot free.')}
${m && m.slots ? slotBar(m.slots, m.slots.count || 1) : ''}
${m ? `${(m.recent || []).length} builds logged` : 'its collector pushes to build-1'}${m ? esc(ago(m.collected_at)) : ''}
`); } + const pcsAt = D.pcs && D.pcs.collected_at ? Date.parse(D.pcs.collected_at) : null; const pcsAge = pcsAt ? Math.round((Date.now() - pcsAt) / 1000) : null; + const pcsAgeText = pcsAge === null ? 'report age unknown' : `report ${fmtDurShort(pcsAge)} old${pcsAge > 600 ? ', STALE' : ''}`; + // a PC's five-minute report through the intake (9 October 2026): the known fields on one line, the rest of the report left in workers.json + const pcReportLine = pc => { const r = pc.report; if (!r || !Object.keys(r).length) return ''; const f = []; + if (r.app_version) f.push(`igneum-app ${esc(String(r.app_version))}`); + if (r.node_daa !== undefined && r.node_daa !== null) f.push(`node DAA ${Number(r.node_daa).toLocaleString('en-GB')}${r.node_state ? ' (' + esc(String(r.node_state)) + ')' : r.synced === false ? ' (behind)' : ''}`); + if (r.rate_mhs !== undefined && r.rate_mhs !== null) f.push(`${Number(r.rate_mhs).toFixed(1)} MH/s`); + if (Array.isArray(r.cards)) f.push(r.cards.map(c => `${esc(String(c.name || c.id || 'card'))} ${c.rate_mhs === undefined || c.rate_mhs === null ? '?' : Number(c.rate_mhs).toFixed(1)} MH/s`).join(', ')); + if (r.last_accepted_at) f.push(`last block ${esc(String(r.last_accepted_at).replace(/^\d{4}-\d{2}-\d{2}T/, ''))}`); + if (r.agent_pid || r.node_pid) f.push(`pids agent ${r.agent_pid ?? '?'} node ${r.node_pid ?? '?'}`); + return f.join(' · '); }; for (const pc of pcs) { const r = pc.running; - cards.push(`
${esc(pc.name)}
${esc(pc.machine || pc.id)} · ${esc(pc.role)}
${pill(r ? 'running' : pc.queue.length ? 'queued' : 'idle', r ? 'running' : pc.queue.length ? 'queued' : '')}
${r ? `${esc(r.kind)} ${esc(r.job)}${r.title ? '
' + esc(r.title) : ''}${r.stage ? `
stage ${esc(r.stage)}` : ''}` : pc.queue.length ? `Next: ${esc(pc.queue[0].kind)} ${esc(pc.queue[0].job)}${pc.queue[0].title ? ', ' + esc(pc.queue[0].title) : ''}` : esc(pc.note || 'idle on jobs; the relay polls every 10 min')}
${pc.recent.length} reports kept${pc.last_report_at ? 'last report ' + esc(ago(pc.last_report_at)) : 'no report in 7 days'}
`); + const pcAge = pc.report_age_s === undefined || pc.report_age_s === null ? pcsAge : pc.report_age_s; + const pcAgeText = pcAge === null ? 'report age unknown' : `report ${fmtDurShort(pcAge)} old${pcAge > 600 ? ', STALE' : ''}${pc.source === 'intake' ? ', from the PC' : ''}`; + cards.push(`
${esc(pc.name)}
${esc(pc.machine || pc.id)} · ${esc(pc.role)} · 600 ? 'style="color:var(--warn)"' : ''}>${esc(pcAgeText)}
${pill(r ? 'running' : pc.queue.length ? 'queued' : 'idle', r ? 'running' : pc.queue.length ? 'queued' : '')}
${pcReportLine(pc) ? pcReportLine(pc) + '
' : ''}${r ? `${esc(r.kind)} ${esc(r.job)}${r.title ? '
' + esc(r.title) : ''}${r.stage ? `
stage ${esc(r.stage)}` : ''}` : pc.queue.length ? `Next: ${esc(pc.queue[0].kind)} ${esc(pc.queue[0].job)}${pc.queue[0].title ? ', ' + esc(pc.queue[0].title) : ''}` : esc(pc.note || 'idle on jobs; the relay polls every 10 min')}
${pc.recent.length} reports kept${pc.last_report_at ? 'last report ' + esc(ago(pc.last_report_at)) : 'no report in 7 days'}
`); } if (!pcs.length) cards.push(`
PC 1 and PC 2
relay jobs
${pill('no data', 'error')}
${esc(D.sources && D.sources.pcs && D.sources.pcs.error || 'the Mac pusher reads the relay intake; nothing has arrived')}
`); $('crew').innerHTML = cards.join('');