igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

Merge workers-page-nine a433836e into master (gate: green on 0f5d6eec, recorded by tools/ci/pre-push.sh; landed on the box mirror)

Browse source
This commit is contained in:
igneum-labs 2026-10-08 21:01:35 +00:00
commit e0f7e6762c
2 changed files with 50 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
tools/ci/route-spill-check.sh
View file

@ -36,8 +36,9 @@ BS_ROUTE_STATE_1="free=1 slots=2 load1=80" BS_ROUTE_STATE_2="free=3 slots=3 load
BS_ROUTE_STATE_1="free=2 slots=2 load1=70" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "load 70 stays on build-1 (the line is 80)" build 1 0
# the ssh path itself under the hook's shell (/bin/bash is 3.2 on the Mac; the pool lane found `BS_SSH_OPTS[@]: unbound variable`
# at the first unpinned route, 7 Oct 2026): a host file pointing at a port nothing answers on must read "down" in a few seconds,
# not die on an unset array
printf 'build@127.0.0.1\n' > "$t/hosts"
# not die on an unset array. Port 1 in the ssh URI form (8 Oct 2026, the workers-page lane): on a build box sshd answers on
# 127.0.0.1:22 and a forwarded agent logs in, so the bare address read "free=..." there and the gate was red only off the Mac
printf 'ssh://build@127.0.0.1:1\n' > "$t/hosts"
out=$(IGNEUM_BUILD_KEY="$t/no-such-key" /bin/bash -c '. infra/build-server/lib.sh; bs_box_state 1' 2>&1 || true)
if [ "$out" = down ]; then echo "route-spill: the ssh probe under /bin/bash $(/bin/bash -c 'echo $BASH_VERSION') reads an unreachable box as down"
else echo "route-spill: the ssh probe under /bin/bash failed: '$out' (expected 'down')" >&2; fail=1; fi

70
tools/workers/page/workers.html
View file

@ -1,6 +1,13 @@
<!doctype html>
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1, viewport-fit=cover"><meta name="robots" content="noindex">
<title>Igneum Workers</title>
<!-- the main site's icon set (site/index.html's head, the ember mark); the files sit next to this page in /srv/workers -->
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 1024 1024'%3E%3Crect width='1024' height='1024' fill='%230C0C0E'/%3E%3Cg transform='translate(166.95 166.95) scale(6.901)'%3E%3Cpolygon points='50,4 74,34 67,58 80,54 61,96 39,96 20,54 33,58 26,34' fill='%23F2541B'/%3E%3Cpolygon points='50,42 59,58 50,82 41,58' fill='%230C0C0E'/%3E%3C/g%3E%3C/svg%3E" type="image/svg+xml">
<link rel="icon" href="/favicon.ico" sizes="48x48">
<link rel="icon" href="/favicon-32.png" type="image/png" sizes="32x32">
<link rel="icon" href="/icon-192.png" type="image/png" sizes="192x192">
<link rel="icon" href="/icon-512.png" type="image/png" sizes="512x512">
<link rel="apple-touch-icon" href="/apple-touch-icon.png" sizes="180x180">
<link rel="stylesheet" href="https://fonts.googleapis.com/css2?family=Unbounded:wght@500;700&family=IBM+Plex+Mono:wght@400;500;600&family=IBM+Plex+Sans:wght@400;500;600&display=swap">
<style>
:root {
@ -195,17 +202,31 @@ function arc(pct, cls) {
}
const tone = (pct, warn = 70, bad = 90) => pct >= bad ? 'bad' : pct >= warn ? 'warn' : '';
const FEEDS = { 'igneum-build-1': 'https://build.igneum.network/workers.json', 'igneum-build-2': 'https://build-2.igneum.network/workers.json', 'igneum-build-3': 'https://build-3.igneum.network/workers.json' };
// one card per entry of boxes[] in the merged workers.json (merge-workers.mjs on build-1, every 20 s); no per-box feed map (8 October 2026:
// the old three-entry map drew build-1 to build-3 only while the merge carried nine boxes)
const STALE_S = 120;
const BOX_NOTES = { 'igneum-build-3': 'down since 16:40 UK, Hetzner switch fault' };
const PROVISIONING = 'provisioning, first build logged when the sources land';
function boxState(b) { // live | down | provisioning, with the line the cards print
const age = b && b.collected_at ? (Date.now() - Date.parse(b.collected_at)) / 1000 : null;
const stale = !b || b.down || !b.cores || (b.source && b.source.ok === false) || age === null || age > STALE_S;
if (b && stale && BOX_NOTES[b.name]) return { kind: 'down', note: BOX_NOTES[b.name] };
if (b && b.down) return { kind: 'down', note: String(b.reason || 'no answer from this box') };
if (!b || !b.cores) return { kind: 'provisioning', note: PROVISIONING };
if (!b.log || !b.log.present) return { kind: 'provisioning', note: PROVISIONING, live: true };
return { kind: 'live', stale: age !== null && age > STALE_S };
}
const buildHeld = b => (b.slots && b.slots.held || []).filter(h => /^build-\d+$/.test(h.slot)).length;
const boxesOf = d => (Array.isArray(d.boxes) && d.boxes.length ? d.boxes : d.box ? [{ ...d.box, source: d.sources && d.sources.box, headline: d.headline }] : []);
function serverSection(b, i) {
const id = `srv${i}`;
if (!b || !b.cores) return `<section class="server" aria-label="${esc(b && b.name || 'build server')}"><div class="head"><div><div class="name">${esc(b && b.name || 'build server')}<small>build server</small></div><div class="facts"><span>${esc(b && b.source && b.source.error || 'provisioning, first build logged when the sources land')}</span></div></div></div><div class="cores">${Array.from({ length: 96 }, () => '<i style="--h:2%"></i>').join('')}</div><div class="corelabel"><span>no sample</span><span></span></div></section>`;
const st = boxState(b);
if (st.kind === 'down' || !b || !b.cores) return `<section class="server" aria-label="${esc(b && b.name || 'build server')}"><div class="head"><div><div class="name">${esc(b && b.name || 'build server')}<small>build server</small></div><div class="facts"><span>${esc(st.note || PROVISIONING)}</span></div></div></div><div class="cores">${Array.from({ length: 96 }, () => '<i style="--h:2%"></i>').join('')}</div><div class="corelabel"><span>no sample</span><span></span></div></section>`;
const per = (b.cpu && b.cpu.per_core) || []; const n = Math.max(per.length, b.cores || 96);
const temps = (b.temps || []).map(x => `${esc(x.name)} <b>${x.c.toFixed(0)}°</b>`);
const m = b.mem || {}, d = b.disk || {}, sc = b.sccache;
const hit = sc && sc.hit_rate_pct !== null && sc.hit_rate_pct !== undefined ? sc.hit_rate_pct : null;
const stale = b.source && !b.source.ok;
const stale = (b.source && !b.source.ok) || st.stale;
return `<section class="server ${b.running && b.running.length ? 'hot' : ''}" aria-label="${esc(b.name)}">
<div class="head"><div><div class="name">${esc(b.name)}<small>${esc(b.os || 'build server')}</small></div><div class="facts">${[`<b>${b.cores}</b> threads`, `<b>${esc(fmtUptime(b.uptime_s))}</b> up`, `load <b>${(b.load || []).map(x => Number(x).toFixed(1)).join(' / ')}</b>`, b.kernel ? `kernel <b>${esc(b.kernel)}</b>` : ''].filter(Boolean).map(x => `<span>${x}</span>`).join('')}</div></div>
<div class="facts">${[...temps, b.net ? `net ↓<b>${esc(fmtBps(b.net.rx_bps))}</b> ↑<b>${esc(fmtBps(b.net.tx_bps))}</b>` : '', `<span title="${esc(b.collected_at)}" ${stale ? 'style="color:var(--warn)"' : ''}>${stale ? 'STALE, ' : ''}read ${esc(ago(b.collected_at))}</span>`].filter(Boolean).map(x => `<span>${x}</span>`).join('')}</div></div>
@ -219,25 +240,39 @@ function serverSection(b, i) {
<div class="g">${arc(b.slots ? (buildHeld(b) / Math.max(1, b.slots.count)) * 100 : 0, 'good')}<div><div class="l">Build slots</div><div class="v">${b.slots ? `${buildHeld(b)}/${b.slots.count}` : '–'}</div><div class="s">${b.queue && b.queue.length ? `${b.queue.length} waiting${b.queue.some(q => q.priority === 'gate') ? ', a gate first' : ''}` : 'nobody waiting'}</div></div></div>
</div></section>`;
}
function renderServers() { const bx = boxesOf(D); $('servers').innerHTML = bx.length ? bx.map(serverSection).join('') : serverSection(null, 0); }
// the Mac mini (igneum-mini, the Mac build box, M6): its own collector pushes mini.json to build-1; until then its card reads no report yet
const MINI = { name: 'igneum-mini', label: 'the Mac build box, M6' };
const miniLive = () => { const m = D && D.mini; return m && m.source && m.source.ok && m.cores ? m : null; };
function miniSection() {
const m = miniLive();
if (!m) return `<section class="server" aria-label="${MINI.name}"><div class="head"><div><div class="name">${MINI.name}<small>${MINI.label}</small></div><div class="facts"><span>no report yet</span></div></div></div></section>`;
return serverSection({ ...m, name: MINI.name, os: MINI.label, log: m.log || { present: true } }, 99);
}
function renderServers() { const bx = boxesOf(D); $('servers').innerHTML = (bx.length ? bx.map(serverSection).join('') : serverSection(null, 0)) + miniSection(); }
function slotBar(slots, count) {
const held = new Map((slots && slots.held || []).map(h => [h.slot, h]));
const names = Array.from({ length: count }, (_, i) => `build-${i}`);
return `<div class="slots">${names.map(nm => `<i class="${held.has(nm) ? 'held' : ''}" title="${esc(nm)}${held.has(nm) ? ': ' + esc(held.get(nm).label) : ': free'}"></i>`).join('')}</div>`;
}
const MAC_COPY = 'nothing builds here except the one DMG; the mini carries macOS binaries from tonight';
function renderCrew() {
const mac = D.mac, pcs = (D.pcs && D.pcs.machines) || [];
const cards = [];
for (const bx of boxesOf(D)) {
if (bx && bx.cores) cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx.name)}</div><div class="meta">Hetzner, ${bx.cores} threads, ${esc(fmtBytes((bx.mem && bx.mem.total_kb || 0) * 1024))}, RAID 1 NVMe</div></div>${pill(bx.running && bx.running.length ? 'building' : 'idle', bx.running && bx.running.length ? 'running' : '')}</div><div class="doing">${bx.running && bx.running.length ? bx.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('<br>') : 'Slot free. The next build-remote.sh or cross-remote.sh routed here takes it.'}</div>${slotBar(bx.slots, bx.slots ? bx.slots.count : 1)}<div class="foot"><span>${bx.recent ? bx.recent.length : 0} builds logged</span><span>${esc(ago(bx.collected_at))}</span></div></div>`);
const st = boxState(bx);
if (st.kind === 'down') { cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx.name)}</div><div class="meta">build server</div></div>${pill('down', 'error')}</div><div class="doing">${esc(st.note)}</div></div>`); continue; }
if (st.kind === 'provisioning' && !st.live) { cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx && bx.name || 'build server')}</div><div class="meta">build server</div></div>${pill('provisioning', 'queued')}</div><div class="doing">${esc(st.note)}</div></div>`); continue; }
if (bx && bx.cores) cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx.name)}</div><div class="meta">Hetzner, ${bx.cores} threads, ${esc(fmtBytes((bx.mem && bx.mem.total_kb || 0) * 1024))}, RAID 1 NVMe</div></div>${pill(bx.running && bx.running.length ? 'building' : 'idle', bx.running && bx.running.length ? 'running' : '')}</div><div class="doing">${bx.running && bx.running.length ? bx.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('<br>') : (st.kind === 'provisioning' ? esc(st.note) : 'Slot free. The next build-remote.sh or cross-remote.sh routed here takes it.')}</div>${slotBar(bx.slots, bx.slots ? bx.slots.count : 1)}<div class="foot"><span>${bx.recent ? bx.recent.length : 0} builds logged</span><span>${esc(ago(bx.collected_at))}</span></div></div>`);
else cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx && bx.name || 'build server')}</div><div class="meta">build server</div></div>${pill('unreachable', 'error')}</div><div class="doing">${esc(bx && bx.source && bx.source.error || 'no facts from this box')}</div></div>`);
}
if (mac) {
const held = mac.slots.held || [], runs = held.filter(h => /^run-/.test(h.slot)), builds = held.filter(h => /^build-/.test(h.slot)), meas = held.filter(h => h.slot === 'measure');
const doing = held.length ? held.map(h => `${esc(h.slot)}: ${esc((h.worktree ? h.worktree + ' ' : '') + (h.command || h.label || ''))}`.slice(0, 140)).join('<br>') : 'Nothing under with-lock.sh. macOS binaries and the DMG build here; everything else goes to the box or the PCs.';
cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(mac.name)}</div><div class="meta">this MacBook: the one DMG only, nothing else builds here (the rule since 7 October 2026); ${mac.build_slots} build slot${mac.build_slots === 1 ? '' : 's'}, 3 run slots</div></div>${pill(meas.length ? 'measuring' : builds.length ? 'building' : runs.length ? 'running' : 'idle', held.length ? 'running' : '')}</div><div class="doing">${doing}</div><div class="slots">${['build-0', 'build-1', 'build-2', 'run-0', 'run-1', 'run-2', 'measure'].map(nm => { const h = held.find(x => x.slot === nm); return `<i class="${h ? (nm === 'measure' ? 'measure' : nm.startsWith('run') ? 'run' : 'held') : ''}" title="${esc(nm)}${h ? ': ' + esc(h.label) : ': free'}"></i>`; }).join('')}</div><div class="foot"><span>${mac.queue.length ? mac.queue.length + ' waiting for a slot' : 'nobody waiting'}</span><span>${esc(ago(mac.collected_at))}</span></div></div>`);
} else cards.push(`<div class="w"><div class="top"><div><div class="card">MacBook-Pro</div><div class="meta">this Mac</div></div>${pill('no push', 'error')}</div><div class="doing">${esc(D.sources && D.sources.mac && D.sources.mac.error || 'the Mac pusher has not written yet (launchd com.igneum.workers-push, every 60 s)')}</div></div>`);
const doing = held.length ? held.map(h => `${esc(h.slot)}: ${esc((h.worktree ? h.worktree + ' ' : '') + (h.command || h.label || ''))}`.slice(0, 140)).join('<br>') : MAC_COPY;
cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(mac.name)}</div><div class="meta">this MacBook, ${mac.build_slots} build slot${mac.build_slots === 1 ? '' : 's'}, 3 run slots</div></div>${pill(meas.length ? 'measuring' : builds.length ? 'building' : runs.length ? 'running' : 'idle', held.length ? 'running' : '')}</div><div class="doing">${doing}</div><div class="slots">${['build-0', 'build-1', 'build-2', 'run-0', 'run-1', 'run-2', 'measure'].map(nm => { const h = held.find(x => x.slot === nm); return `<i class="${h ? (nm === 'measure' ? 'measure' : nm.startsWith('run') ? 'run' : 'held') : ''}" title="${esc(nm)}${h ? ': ' + esc(h.label) : ': free'}"></i>`; }).join('')}</div><div class="foot"><span>${mac.queue.length ? mac.queue.length + ' waiting for a slot' : 'nobody waiting'}</span><span>${esc(ago(mac.collected_at))}</span></div></div>`);
} else cards.push(`<div class="w"><div class="top"><div><div class="card">MacBook-Pro</div><div class="meta">this MacBook</div></div>${pill('no push', 'error')}</div><div class="doing">${MAC_COPY}</div></div>`);
{ const m = miniLive(); const held = m && m.slots && m.slots.held || [];
cards.push(`<div class="w"><div class="top"><div><div class="card">${MINI.name}</div><div class="meta">${MINI.label}</div></div>${pill(!m ? 'no report' : held.length ? 'building' : 'idle', !m ? 'queued' : held.length ? 'running' : '')}</div><div class="doing">${!m ? 'no report yet' : held.length ? held.map(h => esc(`${h.slot}: ${h.worktree || h.label || ''}`)).join('<br>') : 'macOS binaries build here from tonight; slot free.'}</div>${m && m.slots ? slotBar(m.slots, m.slots.count || 1) : ''}<div class="foot"><span>${m ? `${(m.recent || []).length} builds logged` : 'its collector pushes to build-1'}</span><span>${m ? esc(ago(m.collected_at)) : ''}</span></div></div>`); }
for (const pc of pcs) {
const r = pc.running;
cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(pc.name)}</div><div class="meta">${esc(pc.machine || pc.id)} · ${esc(pc.role)}</div></div>${pill(r ? 'running' : pc.queue.length ? 'queued' : 'idle', r ? 'running' : pc.queue.length ? 'queued' : '')}</div><div class="doing">${r ? `${esc(r.kind)} ${esc(r.job)}${r.title ? '<br>' + esc(r.title) : ''}${r.stage ? `<br><span class="pill">stage ${esc(r.stage)}</span>` : ''}` : pc.queue.length ? `Next: ${esc(pc.queue[0].kind)} ${esc(pc.queue[0].job)}${pc.queue[0].title ? ', ' + esc(pc.queue[0].title) : ''}` : esc(pc.note || 'idle on jobs; the relay polls every 10 min')}</div><div class="foot"><span>${pc.recent.length} reports kept</span><span>${pc.last_report_at ? 'last report ' + esc(ago(pc.last_report_at)) : 'no report in 7 days'}</span></div></div>`);
@ -363,7 +398,7 @@ function renderAnalytics() {
function renderSources() {
const s = D.sources || {};
const one = (name, v, what) => `<span>${esc(name)} <b class="${v && v.ok ? '' : v && v.at ? 'warn' : 'no'}">${v && v.ok ? 'live' : v && v.at ? 'stale' : 'missing'}</b>${v && v.at ? ' ' + esc(ago(v.at)) : ''}${v && v.error ? ': ' + esc(v.error) : ''}${!v || v.ok ? '' : ' · ' + esc(what)}</span>`;
$('sources').innerHTML = [...boxesOf(D).map(b => one(b.name || 'box', b.source || s.box, 'collect.mjs on the box, systemd timer every 30 s')), one('mac', s.mac, 'push.mjs on the Mac, launchd every 60 s'), one('pcs', s.pcs, 'relay intake read by push.mjs'), `<span>${esc(D._feed || '')}: fetched ${esc(ago(D._fetched))}, written ${t(D.generated_at)} by ${esc(D.generated_by || '?')}</span>`, ...boxesOf(D).filter(b => b.log).map(b => `<span>${esc(b.name)} log ${b.log.present ? `${b.log.total} lines${b.log.bad_lines ? `, ${b.log.bad_lines} unreadable` : ''}` : 'not written yet'}</span>`)].join('');
$('sources').innerHTML = [...boxesOf(D).map(b => one(b.name || 'box', b.source || s.box, 'collect.mjs on the box, systemd timer every 30 s')), one('mac', s.mac, 'push.mjs on the Mac, launchd every 60 s'), one('mini', D.mini && D.mini.source, 'its collector on the mini, pushed to build-1'), one('pcs', s.pcs, 'relay intake read by push.mjs'), `<span>${esc(D._feed || '')}: fetched ${esc(ago(D._fetched))}, written ${t(D.generated_at)} by ${esc(D.generated_by || '?')}</span>`, ...boxesOf(D).filter(b => b.log).map(b => `<span>${esc(b.name)} log ${b.log.present ? `${b.log.total} lines${b.log.bad_lines ? `, ${b.log.bad_lines} unreadable` : ''}` : 'not written yet'}</span>`)].join('');
}
function apply(d) {
@ -375,25 +410,14 @@ function apply(d) {
const el = $('servers'); if (el && !el.innerHTML) el.innerHTML = '';
renderServers(); renderCrew(); renderNow(); renderQueue(); renderBackground(); renderDone(); renderHeadline(); renderAnalytics(); renderSources();
}
let skipLive = 0, feed = '';
let feed = '';
async function fetchJson(url, ms) { const r = await fetch(`${url}?t=${Date.now()}`, { cache: 'no-store', signal: AbortSignal.timeout(ms) }); if (!r.ok) throw new Error(`http ${r.status}`); return r.json(); }
async function load() {
let edge = null, d = null;
try { const file = demo ? 'workers.demo.json' : 'workers.json'; edge = await fetchJson(`${base}/${file}`, 8000); }
catch (e) { if (!D && (demo || skipLive > 0)) { $('live').className = 'live down'; $('stamp').textContent = `workers.json not reachable (${e && e.message || e})`; } }
catch (e) { if (!D) { $('live').className = 'live down'; $('stamp').textContent = `workers.json not reachable (${e && e.message || e})`; } }
if (demo) { d = edge; feed = 'demo document'; }
else if (skipLive <= 0) {
// the boxes the edge copy knows (or build-1 alone before the first push), each from its own Caddy feed, 4 s each, in parallel
const names = edge ? boxesOf(edge).map(b => b.name).filter(Boolean) : ['igneum-build-1'];
const results = await Promise.all(names.map(n => FEEDS[n] ? fetchJson(FEEDS[n], 4000).then(j => ({ n, j })).catch(() => ({ n, j: null })) : Promise.resolve({ n, j: null })));
const liveOnes = results.filter(r => r.j);
if (liveOnes.length) {
const first = liveOnes[0].j;
const boxes = results.map(r => r.j ? { ...r.j.box, headline: r.j.headline, source: { ok: true, at: r.j.generated_at } } : (edge && boxesOf(edge).find(b => b.name === r.n)) || { name: r.n, source: { ok: false, error: 'no live feed and no edge copy for this box' } });
d = { ...first, boxes, mac: first.mac || (edge && edge.mac), pcs: first.pcs || (edge && edge.pcs), baselines: first.baselines || (edge && edge.baselines), sources: { ...(first.sources || {}), box: boxes[0].source } };
feed = liveOnes.length === results.length ? `live from ${liveOnes.length === 1 ? 'the box' : liveOnes.length + ' boxes'}` : `live from ${liveOnes.map(r => r.n).join(', ')}, edge copy for the rest`;
} else skipLive = 4; // nothing answered: use the edge copy for the next minute before trying again
} else skipLive--;
else { d = edge; feed = edge ? 'the merged copy from build-1' : feed; }
if (!d) { d = edge; feed = 'edge copy, up to 5 min behind the box'; }
if (!d) { if (!D) { $('now').innerHTML = $('queue').innerHTML = `<div class="empty">No workers.json next to this page yet and no box answers. The Mac pusher writes the copy every 60 s and the fleet deploy carries it within 5 min.</div>`; } return; }
d._feed = feed; apply(d);