Merge remote-tracking branch 'origin/master' into site-redesign
This commit is contained in:
igneum-labs
commit
63b127150a
5 changed files with 96 additions and 72 deletions
|
|
@ -1,11 +1,18 @@
|
|||
#!/usr/bin/env bash
|
||||
# Install or refresh the worker dashboard collector on igneum-build-1 from this Mac.
|
||||
# infra/build-server/workers/install.sh copies tools/workers/{lib,collect}.mjs and the two units, enables the timer
|
||||
# Needs root over ssh (root@<ip> with ~/.ssh/igneum_ed25519); the host ip comes from ~/.config/igneum/build-server (build@<ip>).
|
||||
# infra/build-server/workers/install.sh build-1: copies tools/workers/{lib,collect}.mjs and the two units, enables the timer
|
||||
# infra/build-server/workers/install.sh 2 igneum-build-2 (host line in ~/.config/igneum/build-server-2), 3 for build-3
|
||||
# infra/build-server/workers/install.sh build@<ip> any box by its host line
|
||||
# Needs root over ssh (root@<ip> with ~/.ssh/igneum_ed25519); the host ip comes from ~/.config/igneum/build-server[-N] (build@<ip>).
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../../.." && pwd)"
|
||||
KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
|
||||
HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')"
|
||||
case "${1:-}" in
|
||||
"") HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')" ;;
|
||||
[2-9]) HOST_LINE="$(head -1 "$HOME/.config/igneum/build-server-$1" | tr -d '[:space:]')" ;;
|
||||
*@*) HOST_LINE="$1" ;;
|
||||
*) echo "usage: install.sh [N | build@<ip>]" >&2; exit 2 ;;
|
||||
esac
|
||||
IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server" >&2; exit 1; }
|
||||
SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10)
|
||||
"${SSH[@]}" "root@$IP" 'install -d -o build -g build /srv/workers /srv/workers/bin /srv/workers/sources /srv/builds/_log; chown build:build /srv/builds/_log'
|
||||
|
|
|
|||
|
|
@ -130,6 +130,7 @@ export function parseLockDir(files, { day, now, aliveFn } = {}) {
|
|||
const alive = line && aliveFn ? aliveFn(line.pid) : null;
|
||||
const isHeld = f.held ?? (line ? alive !== false : false);
|
||||
if (line && isHeld) held.push({ slot: f.name, ...line, kind: deriveKind(line) });
|
||||
else if (f.held === true) held.push({ slot: f.name, pid: null, since: null, waited_s: null, label: null, agent: null, worktree: null, crate: null, command: null, target: null, kind: null, nice: null, cores: null, no_line: true }); // the flock is taken but the holder wrote no line (7 Oct 2026: the scheduling-class path on build-1)
|
||||
else free.push(f.name);
|
||||
}
|
||||
held.sort((a, b) => a.slot.localeCompare(b.slot));
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@
|
|||
time[title] { cursor: help; border-bottom: 1px dotted var(--line); }
|
||||
|
||||
/* the server */
|
||||
.server + .server { margin-top: 12px; }
|
||||
.server { margin-top: 18px; background: var(--panel); border: 1px solid var(--line); position: relative; overflow: hidden; }
|
||||
.server.hot::after { content: ""; position: absolute; inset: auto -30% -70% -30%; height: 90%; background: radial-gradient(ellipse at center, rgba(242,84,27,.18), transparent 60%); pointer-events: none; animation: breathe 3.2s ease-in-out infinite; }
|
||||
@keyframes breathe { 0%, 100% { opacity: .5; } 50% { opacity: 1; } }
|
||||
|
|
@ -149,15 +150,7 @@
|
|||
<div class="live" id="live"><b></b><span id="stamp">waiting for the first write</span></div>
|
||||
</header>
|
||||
|
||||
<section class="server" id="server" aria-label="The build server">
|
||||
<div class="head">
|
||||
<div><div class="name" id="s-name">igneum-build-1<small>build server</small></div><div class="facts" id="s-facts"></div></div>
|
||||
<div class="facts" id="s-right"></div>
|
||||
</div>
|
||||
<div class="cores" id="cores" aria-label="96 cores, busy share per core"></div>
|
||||
<div class="corelabel"><span id="cores-l">cores</span><span id="cores-r"></span></div>
|
||||
<div class="gauges" id="gauges"></div>
|
||||
</section>
|
||||
<div id="servers"></div>
|
||||
|
||||
<h2>Workers <span>every machine that builds, tests or measures</span></h2>
|
||||
<div class="crew" id="crew"></div>
|
||||
|
|
@ -183,7 +176,7 @@
|
|||
<div class="sources" id="sources"></div>
|
||||
</div>
|
||||
<script type="module">
|
||||
import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, kindLabel, classLine, fmtDur, fmtDurShort, fmtBytes, fmtBps, fmtUptime, ukTime, ago, headline } from './shape.js?v=2';
|
||||
import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, kindLabel, classLine, fmtDur, fmtDurShort, fmtBytes, fmtBps, fmtUptime, ukTime, ago, headline } from './shape.js?v=3';
|
||||
const $ = id => document.getElementById(id);
|
||||
const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c]));
|
||||
const pill = (s, cls) => `<span class="pill ${esc(cls || String(s || '').toLowerCase())}">${esc(s || '?')}</span>`;
|
||||
|
|
@ -198,33 +191,31 @@ function arc(pct, cls) {
|
|||
}
|
||||
const tone = (pct, warn = 70, bad = 90) => pct >= bad ? 'bad' : pct >= warn ? 'warn' : '';
|
||||
|
||||
function renderServer(b, src) {
|
||||
const el = $('server');
|
||||
if (!b) {
|
||||
$('s-facts').innerHTML = `<span>${esc(src && src.error || 'no box facts yet')}</span>`; $('s-right').innerHTML = '';
|
||||
$('cores').innerHTML = Array.from({ length: 96 }, () => '<i style="--h:2%"></i>').join(''); $('cores-r').textContent = 'no sample'; $('gauges').innerHTML = ''; el.classList.remove('hot'); return;
|
||||
}
|
||||
$('s-name').innerHTML = `${esc(b.name)}<small>${esc(b.os || 'build server')}</small>`;
|
||||
$('s-facts').innerHTML = [`<b>${b.cores}</b> threads`, `<b>${esc(fmtUptime(b.uptime_s))}</b> up`, `load <b>${(b.load || []).map(x => Number(x).toFixed(1)).join(' / ')}</b>`, b.kernel ? `kernel <b>${esc(b.kernel)}</b>` : ''].filter(Boolean).map(x => `<span>${x}</span>`).join('');
|
||||
const FEEDS = { 'igneum-build-1': 'https://build.igneum.network/workers.json', 'igneum-build-2': 'https://build-2.igneum.network/workers.json', 'igneum-build-3': 'https://build-3.igneum.network/workers.json' };
|
||||
const buildHeld = b => (b.slots && b.slots.held || []).filter(h => /^build-\d+$/.test(h.slot)).length;
|
||||
const boxesOf = d => (Array.isArray(d.boxes) && d.boxes.length ? d.boxes : d.box ? [{ ...d.box, source: d.sources && d.sources.box, headline: d.headline }] : []);
|
||||
function serverSection(b, i) {
|
||||
const id = `srv${i}`;
|
||||
if (!b || !b.cores) return `<section class="server" aria-label="${esc(b && b.name || 'build server')}"><div class="head"><div><div class="name">${esc(b && b.name || 'build server')}<small>build server</small></div><div class="facts"><span>${esc(b && b.source && b.source.error || 'no facts from this box yet')}</span></div></div></div><div class="cores">${Array.from({ length: 96 }, () => '<i style="--h:2%"></i>').join('')}</div><div class="corelabel"><span>no sample</span><span></span></div></section>`;
|
||||
const per = (b.cpu && b.cpu.per_core) || []; const n = Math.max(per.length, b.cores || 96);
|
||||
const temps = (b.temps || []).map(x => `${esc(x.name)} <b>${x.c.toFixed(0)}°</b>`);
|
||||
$('s-right').innerHTML = [...temps, b.net ? `net ↓<b>${esc(fmtBps(b.net.rx_bps))}</b> ↑<b>${esc(fmtBps(b.net.tx_bps))}</b>` : '', `<span title="${esc(b.collected_at)}">read ${esc(ago(b.collected_at))}</span>`].filter(Boolean).map(x => `<span>${x}</span>`).join('');
|
||||
const per = (b.cpu && b.cpu.per_core) || [];
|
||||
const n = Math.max(per.length, b.cores || 96);
|
||||
$('cores').style.gridTemplateColumns = `repeat(${n > 64 ? 48 : n > 32 ? 32 : n}, 1fr)`;
|
||||
$('cores').innerHTML = Array.from({ length: n }, (_, i) => { const p = per[i] ?? 0; return `<i class="${p >= 50 ? 'hot' : ''}" style="--h:${Math.max(2, p)}%" title="core ${i}: ${p}%"></i>`; }).join('');
|
||||
$('cores-l').textContent = `${n} cores, ${b.cpu ? b.cpu.cores_busy : 0} busy`;
|
||||
$('cores-r').textContent = b.cpu ? `${b.cpu.busy_pct}% of the box over the last second, ${b.compilers || 0} compilers running` : '';
|
||||
el.classList.toggle('hot', !!(b.running && b.running.length));
|
||||
const m = b.mem || {}, d = b.disk || {}, sc = b.sccache;
|
||||
const hit = sc && sc.hit_rate_pct !== null && sc.hit_rate_pct !== undefined ? sc.hit_rate_pct : null;
|
||||
$('gauges').innerHTML = [
|
||||
`<div class="g">${arc(b.cpu ? b.cpu.busy_pct : 0, tone(b.cpu ? b.cpu.busy_pct : 0, 101, 101))}<div><div class="l">CPU</div><div class="v">${b.cpu ? b.cpu.busy_pct : 0}%</div><div class="s">load 1 min ${b.load ? Number(b.load[0]).toFixed(1) : '?'} of ${b.cores}</div></div></div>`,
|
||||
`<div class="g">${arc(m.used_pct, tone(m.used_pct))}<div><div class="l">Memory</div><div class="v">${m.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes((m.used_kb || 0) * 1024))} of ${esc(fmtBytes((m.total_kb || 0) * 1024))}</div></div></div>`,
|
||||
`<div class="g">${arc(d.used_pct, tone(d.used_pct, 80, 92))}<div><div class="l">Disk ${esc(d.mount || '/srv')}</div><div class="v">${d.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes(d.used_bytes))} used, ${esc(fmtBytes(d.avail_bytes))} free</div></div></div>`,
|
||||
`<div class="g">${arc(hit ?? 0, 'cool')}<div><div class="l">sccache hits</div><div class="v">${hit === null ? '–' : hit + '%'}</div><div class="s">${sc ? `${sc.hits ?? 0} hit, ${sc.misses ?? 0} miss, ${esc(sc.cache_size || '?')} of ${esc(sc.max_size || '?')}` : 'sccache not answering'}</div></div></div>`,
|
||||
`<div class="g">${arc(b.slots ? (b.slots.held.length / Math.max(1, b.slots.count)) * 100 : 0, 'good')}<div><div class="l">Build slots</div><div class="v">${b.slots ? `${b.slots.held.length}/${b.slots.count}` : '–'}</div><div class="s">${b.queue && b.queue.length ? `${b.queue.length} waiting` : 'nobody waiting'}</div></div></div>`,
|
||||
].join('');
|
||||
const stale = b.source && !b.source.ok;
|
||||
return `<section class="server ${b.running && b.running.length ? 'hot' : ''}" aria-label="${esc(b.name)}">
|
||||
<div class="head"><div><div class="name">${esc(b.name)}<small>${esc(b.os || 'build server')}</small></div><div class="facts">${[`<b>${b.cores}</b> threads`, `<b>${esc(fmtUptime(b.uptime_s))}</b> up`, `load <b>${(b.load || []).map(x => Number(x).toFixed(1)).join(' / ')}</b>`, b.kernel ? `kernel <b>${esc(b.kernel)}</b>` : ''].filter(Boolean).map(x => `<span>${x}</span>`).join('')}</div></div>
|
||||
<div class="facts">${[...temps, b.net ? `net ↓<b>${esc(fmtBps(b.net.rx_bps))}</b> ↑<b>${esc(fmtBps(b.net.tx_bps))}</b>` : '', `<span title="${esc(b.collected_at)}" ${stale ? 'style="color:var(--warn)"' : ''}>${stale ? 'STALE, ' : ''}read ${esc(ago(b.collected_at))}</span>`].filter(Boolean).map(x => `<span>${x}</span>`).join('')}</div></div>
|
||||
<div class="cores" style="grid-template-columns:repeat(${n > 64 ? 48 : n > 32 ? 32 : n}, 1fr)" aria-label="${n} cores, busy share per core">${Array.from({ length: n }, (_, i) => { const p = per[i] ?? 0; return `<i class="${p >= 50 ? 'hot' : ''}" style="--h:${Math.max(2, p)}%" title="core ${i}: ${p}%"></i>`; }).join('')}</div>
|
||||
<div class="corelabel"><span>${n} cores, ${b.cpu ? b.cpu.cores_busy : 0} busy</span><span>${b.cpu ? `${b.cpu.busy_pct}% of the box over the last second, ${b.compilers || 0} compilers running` : ''}</span></div>
|
||||
<div class="gauges">
|
||||
<div class="g">${arc(b.cpu ? b.cpu.busy_pct : 0)}<div><div class="l">CPU</div><div class="v">${b.cpu ? b.cpu.busy_pct : 0}%</div><div class="s">load 1 min ${b.load ? Number(b.load[0]).toFixed(1) : '?'} of ${b.cores}</div></div></div>
|
||||
<div class="g">${arc(m.used_pct, tone(m.used_pct))}<div><div class="l">Memory</div><div class="v">${m.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes((m.used_kb || 0) * 1024))} of ${esc(fmtBytes((m.total_kb || 0) * 1024))}</div></div></div>
|
||||
<div class="g">${arc(d.used_pct, tone(d.used_pct, 80, 92))}<div><div class="l">Disk ${esc(d.mount || '/srv')}</div><div class="v">${d.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes(d.used_bytes))} used, ${esc(fmtBytes(d.avail_bytes))} free</div></div></div>
|
||||
<div class="g">${arc(hit ?? 0, 'cool')}<div><div class="l">sccache hits</div><div class="v">${hit === null ? '–' : hit + '%'}</div><div class="s">${sc ? `${sc.hits ?? 0} hit, ${sc.misses ?? 0} miss, ${esc(sc.cache_size || '?')} of ${esc(sc.max_size || '?')}` : 'sccache not answering'}</div></div></div>
|
||||
<div class="g">${arc(b.slots ? (buildHeld(b) / Math.max(1, b.slots.count)) * 100 : 0, 'good')}<div><div class="l">Build slots</div><div class="v">${b.slots ? `${buildHeld(b)}/${b.slots.count}` : '–'}</div><div class="s">${b.queue && b.queue.length ? `${b.queue.length} waiting${b.queue.some(q => q.priority === 'gate') ? ', a gate first' : ''}` : 'nobody waiting'}</div></div></div>
|
||||
</div></section>`;
|
||||
}
|
||||
function renderServers() { const bx = boxesOf(D); $('servers').innerHTML = bx.length ? bx.map(serverSection).join('') : serverSection(null, 0); }
|
||||
|
||||
function slotBar(slots, count) {
|
||||
const held = new Map((slots && slots.held || []).map(h => [h.slot, h]));
|
||||
|
|
@ -232,10 +223,12 @@ function slotBar(slots, count) {
|
|||
return `<div class="slots">${names.map(nm => `<i class="${held.has(nm) ? 'held' : ''}" title="${esc(nm)}${held.has(nm) ? ': ' + esc(held.get(nm).label) : ': free'}"></i>`).join('')}</div>`;
|
||||
}
|
||||
function renderCrew() {
|
||||
const b = D.box, mac = D.mac, pcs = (D.pcs && D.pcs.machines) || [];
|
||||
const mac = D.mac, pcs = (D.pcs && D.pcs.machines) || [];
|
||||
const cards = [];
|
||||
if (b) cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(b.name)}</div><div class="meta">Hetzner, ${b.cores} threads, ${esc(fmtBytes((b.mem && b.mem.total_kb || 0) * 1024))}, RAID 1 NVMe</div></div>${pill(b.running && b.running.length ? 'building' : 'idle', b.running && b.running.length ? 'running' : '')}</div><div class="doing">${b.running && b.running.length ? b.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('<br>') : 'Slot free. The next build-remote.sh or cross-remote.sh from any worktree takes it.'}</div>${slotBar(b.slots, b.slots ? b.slots.count : 1)}<div class="foot"><span>${b.recent ? b.recent.length : 0} builds logged</span><span>${esc(ago(b.collected_at))}</span></div></div>`);
|
||||
else cards.push(`<div class="w"><div class="top"><div><div class="card">igneum-build-1</div><div class="meta">build server</div></div>${pill('unreachable', 'error')}</div><div class="doing">${esc(D.sources && D.sources.box && D.sources.box.error || 'no facts from the box')}</div></div>`);
|
||||
for (const bx of boxesOf(D)) {
|
||||
if (bx && bx.cores) cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx.name)}</div><div class="meta">Hetzner, ${bx.cores} threads, ${esc(fmtBytes((bx.mem && bx.mem.total_kb || 0) * 1024))}, RAID 1 NVMe</div></div>${pill(bx.running && bx.running.length ? 'building' : 'idle', bx.running && bx.running.length ? 'running' : '')}</div><div class="doing">${bx.running && bx.running.length ? bx.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('<br>') : 'Slot free. The next build-remote.sh or cross-remote.sh routed here takes it.'}</div>${slotBar(bx.slots, bx.slots ? bx.slots.count : 1)}<div class="foot"><span>${bx.recent ? bx.recent.length : 0} builds logged</span><span>${esc(ago(bx.collected_at))}</span></div></div>`);
|
||||
else cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx && bx.name || 'build server')}</div><div class="meta">build server</div></div>${pill('unreachable', 'error')}</div><div class="doing">${esc(bx && bx.source && bx.source.error || 'no facts from this box')}</div></div>`);
|
||||
}
|
||||
if (mac) {
|
||||
const held = mac.slots.held || [], runs = held.filter(h => /^run-/.test(h.slot)), builds = held.filter(h => /^build-/.test(h.slot)), meas = held.filter(h => h.slot === 'measure');
|
||||
const doing = held.length ? held.map(h => `${esc(h.slot)}: ${esc((h.worktree ? h.worktree + ' ' : '') + (h.command || h.label || ''))}`.slice(0, 140)).join('<br>') : 'Nothing under with-lock.sh. macOS binaries and the DMG build here; everything else goes to the box or the PCs.';
|
||||
|
|
@ -251,7 +244,8 @@ function renderCrew() {
|
|||
|
||||
function nowItems() {
|
||||
const out = [];
|
||||
for (const r of (D.box && D.box.running) || []) out.push({ where: D.box.name, slot: r.slot, title: `${kindLabel(r.kind)}`, sub: `${r.worktree || '?'}/${r.crate || ''}${r.agent ? ' · ' + r.agent : ''}`, cmd: r.command, started_at: r.started_at, kind: r.kind, extra: [r.target || '', classLine(r)].filter(Boolean).join(' · ') });
|
||||
for (const bx of boxesOf(D)) for (const r of bx.running || []) out.push({ where: bx.name, slot: r.slot, title: `${kindLabel(r.kind)}`, sub: `${r.worktree || '?'}/${r.crate || ''}${r.agent ? ' · ' + r.agent : ''}`, cmd: r.command, started_at: r.started_at, kind: r.kind, extra: [r.target || '', classLine(r)].filter(Boolean).join(' · ') });
|
||||
for (const bx of boxesOf(D)) for (const h of (bx.slots && bx.slots.held || []).filter(h => h.no_line && !(bx.running || []).some(r => r.slot === h.slot))) out.push({ where: bx.name, slot: h.slot, title: 'held, no holder line', sub: 'the slot is flock-held but its file is empty (the holder wrote no line)', cmd: '', started_at: null, kind: null, extra: '' });
|
||||
for (const h of (D.mac && D.mac.slots.held) || []) out.push({ where: D.mac.name, slot: h.slot, title: h.slot === 'measure' ? 'measurement' : h.slot.startsWith('run') ? 'functional run' : kindLabel(h.kind), sub: h.worktree || 'this Mac', cmd: h.command || h.label, started_at: h.since, kind: h.kind, extra: h.waited_s ? `waited ${fmtDur(h.waited_s)}` : '' });
|
||||
for (const pc of (D.pcs && D.pcs.machines) || []) if (pc.running) out.push({ where: pc.name, slot: 'relay job', title: `${pc.running.kind} ${pc.running.job}`, sub: pc.running.title || pc.machine || '', cmd: pc.running.stage ? `stage ${pc.running.stage}` : 'reports every 5 min while it runs', started_at: pc.running.started_at, kind: pc.running.kind, extra: pc.running.last_report_at ? `last report ${ago(pc.running.last_report_at)}` : '' });
|
||||
return out;
|
||||
|
|
@ -292,7 +286,7 @@ function renderBackground() {
|
|||
}
|
||||
function renderQueue() {
|
||||
const items = [];
|
||||
for (const w of (D.box && D.box.queue) || []) items.push({ where: D.box.name, title: w.label ? kindLabel(w.kind) : 'a build', sub: w.label ? `${w.worktree || ''}${w.crate ? '/' + w.crate : ''} ${w.command || ''}`.trim() : 'label unknown: no wait file for this flock', since: w.since, wait: w.waiting_s, gate: w.priority === 'gate', extra: [w.agent || '', classLine(w)].filter(Boolean).join(' · ') });
|
||||
for (const bx of boxesOf(D)) for (const w of bx.queue || []) items.push({ where: bx.name, title: w.label ? kindLabel(w.kind) : 'a build', sub: w.label ? `${w.worktree || ''}${w.crate ? '/' + w.crate : ''} ${w.command || ''}`.trim() : 'label unknown: no wait file for this flock', since: w.since, wait: w.waiting_s, gate: w.priority === 'gate', extra: [w.agent || '', classLine(w)].filter(Boolean).join(' · ') });
|
||||
for (const w of (D.mac && D.mac.queue) || []) items.push({ where: D.mac.name, title: `${w.mode} slot`, sub: w.command, since: null, wait: w.waiting_s });
|
||||
for (const pc of (D.pcs && D.pcs.machines) || []) for (const q of pc.queue) items.push({ where: pc.name, title: `${q.kind} ${q.job}`, sub: q.title || '', since: null, wait: null, until: q.expires_at });
|
||||
$('queue-sub').textContent = items.length ? `${items.length} waiting` : '';
|
||||
|
|
@ -302,14 +296,14 @@ function renderQueue() {
|
|||
|
||||
function doneRows() {
|
||||
const rows = [];
|
||||
for (const b of (D.box && D.box.recent) || []) rows.push({ key: b.id, where: D.box.name, at: b.end || b.start, title: `${kindLabel(b.kind)} · ${b.worktree || '?'}${b.agent ? ' · ' + b.agent : ''}`, meta: b.command || '', ok: b.ok, secs: b.secs, status: b.ok ? 'ok' : `exit ${b.exit}`, lines: b.closing || null, raw: b });
|
||||
for (const bx of boxesOf(D)) for (const b of bx.recent || []) rows.push({ key: b.id, where: bx.name, at: b.end || b.start, title: `${kindLabel(b.kind)} · ${b.worktree || '?'}${b.agent ? ' · ' + b.agent : ''}`, meta: b.command || '', ok: b.ok, secs: b.secs, status: b.ok ? 'ok' : `exit ${b.exit}`, lines: b.closing || null, raw: b });
|
||||
for (const pc of (D.pcs && D.pcs.machines) || []) for (const r of pc.recent) rows.push({ key: `${pc.id}-${r.job}`, where: pc.name, at: r.finished_at, title: `${r.kind} ${r.job}`, meta: r.title || r.summary || '', ok: r.ok, secs: r.secs, status: r.status, lines: [r.summary, ...(r.errors || [])].filter(Boolean), raw: r });
|
||||
rows.sort((a, b) => String(b.at || '').localeCompare(String(a.at || '')));
|
||||
return rows;
|
||||
}
|
||||
function renderDone() {
|
||||
const rows = doneRows().slice(0, 40);
|
||||
const boxRows = timeline((D.box && D.box.recent) || [], { limit: 200 }); const closing = new Map(boxRows.map(r => [r.id, r.closing]));
|
||||
const boxRows = timeline(boxesOf(D).flatMap(bx => bx.recent || []), { limit: 600 }); const closing = new Map(boxRows.map(r => [r.id, r.closing]));
|
||||
if (!rows.length) { $('done').innerHTML = `<div class="empty">No finished work yet. The box appends one line per build to /srv/builds/_log/builds.jsonl when bs_remote_run ends${D.box && D.box.log && !D.box.log.present ? ' (the file does not exist yet)' : ''}; the PCs' closing reports come from the relay intake.</div>`; return; }
|
||||
const max = Math.max(60, ...rows.map(r => Number(r.secs) || 0));
|
||||
$('done').innerHTML = rows.map(r => {
|
||||
|
|
@ -322,7 +316,7 @@ function renderDone() {
|
|||
}
|
||||
|
||||
function renderHeadline() {
|
||||
const rows = headline(D.headline);
|
||||
const bx = boxesOf(D).filter(b => b.headline); const rows = bx.length ? bx.flatMap(b => headline(b.headline).map(r => ({ ...r, label: bx.length > 1 ? `${r.label} · ${b.name}` : r.label }))) : headline(D.headline);
|
||||
$('headline').innerHTML = rows.map(r => {
|
||||
const max = Math.max(r.box_s || 0, r.mac_s || 0, r.pc_s || 0, 1);
|
||||
return `<div class="h3"><div class="l">${esc(r.label)}</div><div class="v">${r.box_s === null ? '<span style="color:var(--muted);font-size:14px">not measured yet</span>' : esc(fmtDur(r.box_s)) + '<small>box</small>'}</div><div class="cmp">${r.mac_s !== null ? `Mac <b>${esc(fmtDur(r.mac_s))}</b>${r.speedup ? ` (${r.speedup}× slower)` : ''}` : 'Mac <b>not measured on this tree</b>'}${r.pc_s !== null ? ` · PC 1 <b>${esc(fmtDur(r.pc_s))}</b>` : ''}</div>${r.box_s !== null ? `<div class="meter"><i style="width:${(r.box_s / max * 100).toFixed(1)}%"></i>${r.mac_s !== null ? `<em style="left:${(r.mac_s / max * 100).toFixed(1)}%" title="Mac"></em>` : ''}${r.pc_s !== null ? `<em style="left:${(r.pc_s / max * 100).toFixed(1)}%;background:var(--cool)" title="PC 1"></em>` : ''}</div>` : ''}<div class="s">${esc(r.note || r.what)}${r.measured_at ? ` · measured ${t(r.measured_at, true)}` : ''}</div></div>`;
|
||||
|
|
@ -344,7 +338,7 @@ function bars(buckets) {
|
|||
return `<svg viewBox="0 0 ${W} ${H}" aria-label="builds per hour today"><line x1="4" y1="${H - B}" x2="${W - 4}" y2="${H - B}" stroke="#26262d"/>${buckets.map((b, i) => { const h = (b.n / max) * (H - T - B); const ok = b.n ? (b.ok / b.n) * h : 0; const x = 4 + i * bw; return `<g><title>${String(i).padStart(2, '0')}:00 UTC: ${b.n} build${b.n === 1 ? '' : 's'}, ${b.failed} failed</title><rect x="${(x + 1).toFixed(1)}" y="${(H - B - h).toFixed(1)}" width="${(bw - 2).toFixed(1)}" height="${h.toFixed(1)}" fill="#ff6b5e"/><rect x="${(x + 1).toFixed(1)}" y="${(H - B - ok).toFixed(1)}" width="${(bw - 2).toFixed(1)}" height="${ok.toFixed(1)}" fill="#5fd08a"/></g>`; }).join('')}${[0, 6, 12, 18, 23].map(h => `<text x="${(4 + h * bw + bw / 2).toFixed(1)}" y="${H - 5}" text-anchor="middle" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${String(h).padStart(2, '0')}Z</text>`).join('')}<text x="4" y="${T + 4}" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${max}</text></svg>`;
|
||||
}
|
||||
function renderAnalytics() {
|
||||
const builds = (D.box && D.box.recent) || [];
|
||||
const builds = boxesOf(D).flatMap(bx => (bx.recent || []).map(b => ({ ...b, host: b.host || bx.name })));
|
||||
const day = new Date().toISOString().slice(0, 10);
|
||||
const todays = builds.filter(b => String(b.end || b.start || '').startsWith(day));
|
||||
$('an-sub').textContent = builds.length ? `${builds.length} logged builds, ${todays.length} today, ${todays.filter(b => b.ok).length} passed` : '';
|
||||
|
|
@ -365,7 +359,7 @@ function renderAnalytics() {
|
|||
function renderSources() {
|
||||
const s = D.sources || {};
|
||||
const one = (name, v, what) => `<span>${esc(name)} <b class="${v && v.ok ? '' : v && v.at ? 'warn' : 'no'}">${v && v.ok ? 'live' : v && v.at ? 'stale' : 'missing'}</b>${v && v.at ? ' ' + esc(ago(v.at)) : ''}${v && v.error ? ': ' + esc(v.error) : ''}${!v || v.ok ? '' : ' · ' + esc(what)}</span>`;
|
||||
$('sources').innerHTML = [one('box', s.box, 'collect.mjs on the box, systemd timer every 30 s'), one('mac', s.mac, 'push.mjs on the Mac, launchd every 60 s'), one('pcs', s.pcs, 'relay intake read by push.mjs'), `<span>${esc(D._feed || '')}: fetched ${esc(ago(D._fetched))}, written ${t(D.generated_at)} by ${esc(D.generated_by || '?')}</span>`, D.box && D.box.log ? `<span>log ${D.box.log.present ? `${D.box.log.total} lines${D.box.log.bad_lines ? `, ${D.box.log.bad_lines} unreadable` : ''}` : 'not written yet'}</span>` : ''].join('');
|
||||
$('sources').innerHTML = [...boxesOf(D).map(b => one(b.name || 'box', b.source || s.box, 'collect.mjs on the box, systemd timer every 30 s')), one('mac', s.mac, 'push.mjs on the Mac, launchd every 60 s'), one('pcs', s.pcs, 'relay intake read by push.mjs'), `<span>${esc(D._feed || '')}: fetched ${esc(ago(D._fetched))}, written ${t(D.generated_at)} by ${esc(D.generated_by || '?')}</span>`, ...boxesOf(D).filter(b => b.log).map(b => `<span>${esc(b.name)} log ${b.log.present ? `${b.log.total} lines${b.log.bad_lines ? `, ${b.log.bad_lines} unreadable` : ''}` : 'not written yet'}</span>`)].join('');
|
||||
}
|
||||
|
||||
function apply(d) {
|
||||
|
|
@ -374,20 +368,30 @@ function apply(d) {
|
|||
const live = $('live'); live.className = 'live' + (age > 900 ? ' down' : age > 420 ? ' stale' : '');
|
||||
const u = ukTime(d.generated_at);
|
||||
$('stamp').innerHTML = `${age > 420 ? 'stale: ' : ''}written ${t(d.generated_at)} (${esc(ago(d.generated_at))})${d._feed ? ' · ' + esc(d._feed) : ''}`;
|
||||
renderServer(d.box, d.sources && d.sources.box); renderCrew(); renderNow(); renderQueue(); renderBackground(); renderDone(); renderHeadline(); renderAnalytics(); renderSources();
|
||||
const el = $('servers'); if (el && !el.innerHTML) el.innerHTML = '';
|
||||
renderServers(); renderCrew(); renderNow(); renderQueue(); renderBackground(); renderDone(); renderHeadline(); renderAnalytics(); renderSources();
|
||||
}
|
||||
const LIVE_URL = 'https://build.igneum.network/workers.json'; // the box itself, 30 s fresh (Caddy, read-only); the folder copy is up to 5 min behind
|
||||
let skipLive = 0, feed = '';
|
||||
async function fetchJson(url, ms) { const r = await fetch(`${url}?t=${Date.now()}`, { cache: 'no-store', signal: AbortSignal.timeout(ms) }); if (!r.ok) throw new Error(`http ${r.status}`); return r.json(); }
|
||||
async function load() {
|
||||
let d = null, err = null;
|
||||
if (!demo && skipLive <= 0) {
|
||||
try { const r = await fetch(`${LIVE_URL}?t=${Date.now()}`, { cache: 'no-store', signal: AbortSignal.timeout(4000) }); if (!r.ok) throw new Error(`http ${r.status}`); d = await r.json(); feed = 'live from the box'; }
|
||||
catch (e) { skipLive = 4; err = e; } // the box not answering: use the edge copy for the next minute before trying again
|
||||
let edge = null, d = null;
|
||||
try { const file = demo ? 'workers.demo.json' : 'workers.json'; edge = await fetchJson(`${base}/${file}`, 8000); }
|
||||
catch (e) { if (!D && (demo || skipLive > 0)) { $('live').className = 'live down'; $('stamp').textContent = `workers.json not reachable (${e && e.message || e})`; } }
|
||||
if (demo) { d = edge; feed = 'demo document'; }
|
||||
else if (skipLive <= 0) {
|
||||
// the boxes the edge copy knows (or build-1 alone before the first push), each from its own Caddy feed, 4 s each, in parallel
|
||||
const names = edge ? boxesOf(edge).map(b => b.name).filter(Boolean) : ['igneum-build-1'];
|
||||
const results = await Promise.all(names.map(n => FEEDS[n] ? fetchJson(FEEDS[n], 4000).then(j => ({ n, j })).catch(() => ({ n, j: null })) : Promise.resolve({ n, j: null })));
|
||||
const liveOnes = results.filter(r => r.j);
|
||||
if (liveOnes.length) {
|
||||
const first = liveOnes[0].j;
|
||||
const boxes = results.map(r => r.j ? { ...r.j.box, headline: r.j.headline, source: { ok: true, at: r.j.generated_at } } : (edge && boxesOf(edge).find(b => b.name === r.n)) || { name: r.n, source: { ok: false, error: 'no live feed and no edge copy for this box' } });
|
||||
d = { ...first, boxes, mac: first.mac || (edge && edge.mac), pcs: first.pcs || (edge && edge.pcs), baselines: first.baselines || (edge && edge.baselines), sources: { ...(first.sources || {}), box: boxes[0].source } };
|
||||
feed = liveOnes.length === results.length ? `live from ${liveOnes.length === 1 ? 'the box' : liveOnes.length + ' boxes'}` : `live from ${liveOnes.map(r => r.n).join(', ')}, edge copy for the rest`;
|
||||
} else skipLive = 4; // nothing answered: use the edge copy for the next minute before trying again
|
||||
} else skipLive--;
|
||||
if (!d) {
|
||||
try { const file = demo ? 'workers.demo.json' : 'workers.json'; const r = await fetch(`${base}/${file}?t=${Date.now()}`, { cache: 'no-store' }); if (!r.ok) throw new Error(`http ${r.status}`); d = await r.json(); feed = demo ? 'demo document' : 'edge copy, up to 5 min behind the box'; }
|
||||
catch (e) { $('live').className = 'live down'; $('stamp').textContent = `workers.json not reachable (${e && e.message || e})`; if (!D) { $('now').innerHTML = $('queue').innerHTML = `<div class="empty">No workers.json next to this page yet and the box does not answer. The Mac pusher writes the copy every 60 s and the fleet deploy carries it within 5 min.</div>`; } return; }
|
||||
}
|
||||
if (!d) { d = edge; feed = 'edge copy, up to 5 min behind the box'; }
|
||||
if (!d) { if (!D) { $('now').innerHTML = $('queue').innerHTML = `<div class="empty">No workers.json next to this page yet and no box answers. The Mac pusher writes the copy every 60 s and the fleet deploy carries it within 5 min.</div>`; } return; }
|
||||
d._feed = feed; apply(d);
|
||||
}
|
||||
await load(); setInterval(load, 15000); setInterval(tick, 1000); setInterval(() => { if (D) { renderSources(); } }, 10000);
|
||||
|
|
|
|||
|
|
@ -31,7 +31,10 @@ const cfg = n => { try { return readFileSync(join(homedir(), '.config', 'igneum'
|
|||
const iso = ms => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
|
||||
const log = (...a) => console.error(`${iso(Date.now())} push: ${a.join(' ')}`);
|
||||
const KEY = process.env.IGNEUM_BUILD_KEY || join(homedir(), '.ssh', 'igneum_ed25519');
|
||||
const HOST = (cfg('build-server') || '').split('\n')[0].trim(); // build@<ip>
|
||||
// every build box: ~/.config/igneum/build-server (build-1), build-server-2, build-server-3 ... one line build@<ip> each
|
||||
const HOSTS = [['build-server', 'igneum-build-1'], ['build-server-2', 'igneum-build-2'], ['build-server-3', 'igneum-build-3']]
|
||||
.map(([f, n]) => ({ file: f, name: n, host: (cfg(f) || '').split('\n')[0].trim() })).filter(h => h.host);
|
||||
const HOST = HOSTS.length ? HOSTS[0].host : '';
|
||||
const DLSITE = cfg('dlsite-dir'), FLEET = cfg('fleet-path');
|
||||
const STALE_LIVE_S = 360;
|
||||
const SSH_OPTS = ['-i', KEY, '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=accept-new', '-o', 'ConnectTimeout=10', '-o', 'ControlMaster=auto', '-o', `ControlPath=${join(homedir(), '.ssh', 'cm', 'igneum-workers-%r@%h:%p')}`, '-o', 'ControlPersist=600'];
|
||||
|
|
@ -140,13 +143,13 @@ async function pcs() {
|
|||
}
|
||||
|
||||
// ---- 3 and 4. the box -------------------------------------------------------------------------------------------------------
|
||||
function ssh(args, input) { return spawnSync('ssh', [...SSH_OPTS, HOST, ...args], { encoding: 'utf8', input, timeout: 30000, maxBuffer: 16 * 1024 * 1024 }); }
|
||||
function dropToBox(name, doc) {
|
||||
const r = ssh([`cat > /srv/workers/sources/.${name}.tmp && mv /srv/workers/sources/.${name}.tmp /srv/workers/sources/${name}.json`], JSON.stringify(doc));
|
||||
function ssh(args, input, host = HOST) { return spawnSync('ssh', [...SSH_OPTS, host, ...args], { encoding: 'utf8', input, timeout: 30000, maxBuffer: 16 * 1024 * 1024 }); }
|
||||
function dropToBox(name, doc, host = HOST) {
|
||||
const r = ssh([`cat > /srv/workers/sources/.${name}.tmp && mv /srv/workers/sources/.${name}.tmp /srv/workers/sources/${name}.json`], JSON.stringify(doc), host);
|
||||
if (r.status !== 0) throw new Error(`scp ${name}: ${(r.stderr || '').trim().slice(0, 200) || 'ssh failed'}`);
|
||||
}
|
||||
function pullFromBox() {
|
||||
const r = ssh(['cat /srv/workers/workers.json; echo; echo __SEP__; cat /srv/workers/headline.json 2>/dev/null || echo null']);
|
||||
function pullFromBox(host = HOST) {
|
||||
const r = ssh(['cat /srv/workers/workers.json; echo; echo __SEP__; cat /srv/workers/headline.json 2>/dev/null || echo null'], undefined, host);
|
||||
if (r.status !== 0) throw new Error((r.stderr || '').trim().slice(0, 200) || 'ssh failed');
|
||||
const [w, h] = r.stdout.split('__SEP__');
|
||||
return { workers: JSON.parse(w), headline: JSON.parse((h || 'null').trim() || 'null') };
|
||||
|
|
@ -173,17 +176,23 @@ async function main() {
|
|||
try { doc.mac = macSlots(now); doc.sources.mac = { ok: true, at: doc.mac.collected_at }; } catch (e) { doc.sources.mac = { ok: false, error: String(e.message || e) }; }
|
||||
// the PCs
|
||||
try { doc.pcs = await pcs(); doc.sources.pcs = doc.pcs.error ? { ok: false, error: doc.pcs.error } : { ok: true, at: doc.pcs.collected_at }; } catch (e) { doc.sources.pcs = { ok: false, error: `intake: ${String(e.message || e).slice(0, 160)}` }; }
|
||||
// the box: drop ours, pull theirs
|
||||
if (!HOST) doc.sources.box = { ok: false, error: 'no ~/.config/igneum/build-server' };
|
||||
else {
|
||||
// the boxes: drop ours on each, pull each one's file; boxes[] carries every box, box stays the first for older readers
|
||||
doc.boxes = [];
|
||||
if (!HOSTS.length) doc.sources.box = { ok: false, error: 'no ~/.config/igneum/build-server' };
|
||||
for (const h of HOSTS) {
|
||||
try {
|
||||
if (doc.mac) dropToBox('mac', doc.mac);
|
||||
if (doc.pcs && !doc.pcs.error) dropToBox('pcs', doc.pcs);
|
||||
const { workers, headline } = pullFromBox();
|
||||
doc.box = workers.box; doc.headline = headline;
|
||||
if (doc.mac) dropToBox('mac', doc.mac, h.host);
|
||||
if (doc.pcs && !doc.pcs.error) dropToBox('pcs', doc.pcs, h.host);
|
||||
const { workers, headline } = pullFromBox(h.host);
|
||||
const age = (now - Date.parse(workers.generated_at)) / 1000;
|
||||
doc.sources.box = { ok: age < 120, at: workers.generated_at, age_s: Math.round(age), ...(age >= 120 ? { error: `the box's collector last wrote ${Math.round(age)} s ago (timer igneum-workers.timer)` } : {}) };
|
||||
} catch (e) { doc.sources.box = { ok: false, error: `ssh to the box failed: ${String(e.message || e).slice(0, 160)}` }; }
|
||||
const b = { ...workers.box, headline, feed: `https://${h.name === 'igneum-build-1' ? 'build' : h.name.replace('igneum-', '')}.igneum.network/workers.json`, source: { ok: age < 120, at: workers.generated_at, age_s: Math.round(age), ...(age >= 120 ? { error: `the box's collector last wrote ${Math.round(age)} s ago (timer igneum-workers.timer)` } : {}) } };
|
||||
doc.boxes.push(b);
|
||||
if (!doc.box) { doc.box = workers.box; doc.headline = headline; doc.sources.box = b.source; }
|
||||
} catch (e) {
|
||||
const src = { ok: false, error: `ssh to ${h.name} failed: ${String(e.message || e).slice(0, 160)}` };
|
||||
doc.boxes.push({ name: h.name, source: src });
|
||||
if (!doc.sources.box) doc.sources.box = src;
|
||||
}
|
||||
}
|
||||
if (!doc.headline) doc.headline = null;
|
||||
const text = JSON.stringify(doc);
|
||||
|
|
@ -197,7 +206,7 @@ async function main() {
|
|||
if (age > STALE_LIVE_S) { const ok = deploy(); why = ok ? `deployed (live copy was ${age === Infinity ? 'missing' : Math.round(age) + ' s old'})` : 'DEPLOY FAILED'; }
|
||||
else why = `live copy ${Math.round(age)} s old, no deploy`;
|
||||
}
|
||||
log(`box ${doc.sources.box.ok ? 'ok' : 'NO: ' + doc.sources.box.error}; mac ${doc.sources.mac.ok ? `${doc.mac.slots.held.length} held, ${doc.mac.queue.length} waiting` : 'NO'}; pcs ${doc.sources.pcs.ok ? doc.pcs.machines.map(m => `${m.name} ${m.running ? 'running ' + m.running.job : 'idle'} q${m.queue.length}`).join(', ') : 'NO: ' + doc.sources.pcs.error}; ${why}`);
|
||||
log(`boxes ${doc.boxes.map(b => `${b.name} ${b.source && b.source.ok ? 'ok' : 'NO'}`).join(', ')}; mac ${doc.sources.mac.ok ? `${doc.mac.slots.held.length} held, ${doc.mac.queue.length} waiting` : 'NO'}; pcs ${doc.sources.pcs.ok ? doc.pcs.machines.map(m => `${m.name} ${m.running ? 'running ' + m.running.job : 'idle'} q${m.queue.length}`).join(', ') : 'NO: ' + doc.sources.pcs.error}; ${why}`);
|
||||
}
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
||||
|
|
|
|||
|
|
@ -72,6 +72,9 @@ test('lock dir fixture: one held slot, one stale line, one empty, count from slo
|
|||
// an explicit flock verdict overrides the pid check
|
||||
const r2 = parseLockDir(files.map(f => ({ ...f, held: f.name === 'build-1' })), { day: '2026-10-06', now: '2026-10-06T18:00:00Z' });
|
||||
assert.deepEqual(r2.held.map(h => h.slot), ['build-1']);
|
||||
// a flock-held slot whose file is empty is still held (7 Oct 2026 on build-1: two held, both empty, the box full)
|
||||
const r3 = parseLockDir([{ name: 'build-2', text: '', held: true }, { name: 'build-0', text: '', held: false }, { name: 'slots', text: '2' }], { now: '2026-10-07T10:39:00Z' });
|
||||
assert.equal(r3.held.length, 1); assert.equal(r3.held[0].slot, 'build-2'); assert.equal(r3.held[0].no_line, true); assert.deepEqual(r3.free, ['build-0']);
|
||||
});
|
||||
|
||||
test('builds JSONL fixture: five rows, one bad line, newest first, derived fields', () => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue