From 1bddf2cbf6046e80d79b693043e8ae63a0de944d Mon Sep 17 00:00:00 2001
From: igneum-labs <337424239+igneum-labs@users.noreply.github.com>
Date: Wed, 7 Oct 2026 10:37:33 +0000
Subject: [PATCH] Worker dashboard: one server section per box, per-box live
 feeds, installer and pusher take build-2 and build-3

The build-server agent has igneum-build-2 and -3 on order (suites and benches; proving and the fast-time nodes). The
installer takes a host argument (N reads ~/.config/igneum/build-server-N, or build@<ip>); the pusher drops the Mac and
PC facts on every box it has a host file for, pulls each box's file and publishes boxes[] (box stays the first for the
old shape); the page draws one server section and one crew card per box, reads every box's Caddy feed in parallel
(build, build-2, build-3.igneum.network) with the edge copy filling any box that does not answer, and merges every
box's builds into the lanes, the timeline and the analytics. Nothing changes for build-1 until the host files exist.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 infra/build-server/workers/install.sh |  13 +++-
 tools/workers/page/workers.html       | 108 +++++++++++++-------------
 tools/workers/push.mjs                |  41 ++++++----
 3 files changed, 90 insertions(+), 72 deletions(-)

diff --git a/infra/build-server/workers/install.sh b/infra/build-server/workers/install.sh
index 6842091e3..91bce76d4 100755
--- a/infra/build-server/workers/install.sh
+++ b/infra/build-server/workers/install.sh
@@ -1,11 +1,18 @@
 #!/usr/bin/env bash
 # Install or refresh the worker dashboard collector on igneum-build-1 from this Mac.
-#   infra/build-server/workers/install.sh            copies tools/workers/{lib,collect}.mjs and the two units, enables the timer
-# Needs root over ssh (root@<ip> with ~/.ssh/igneum_ed25519); the host ip comes from ~/.config/igneum/build-server (build@<ip>).
+#   infra/build-server/workers/install.sh            build-1: copies tools/workers/{lib,collect}.mjs and the two units, enables the timer
+#   infra/build-server/workers/install.sh 2           igneum-build-2 (host line in ~/.config/igneum/build-server-2), 3 for build-3
+#   infra/build-server/workers/install.sh build@<ip>  any box by its host line
+# Needs root over ssh (root@<ip> with ~/.ssh/igneum_ed25519); the host ip comes from ~/.config/igneum/build-server[-N] (build@<ip>).
 set -euo pipefail
 HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../../.." && pwd)"
 KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
-HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')"
+case "${1:-}" in
+  "") HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')" ;;
+  [2-9]) HOST_LINE="$(head -1 "$HOME/.config/igneum/build-server-$1" | tr -d '[:space:]')" ;;
+  *@*) HOST_LINE="$1" ;;
+  *) echo "usage: install.sh [N | build@<ip>]" >&2; exit 2 ;;
+esac
 IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server" >&2; exit 1; }
 SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10)
 "${SSH[@]}" "root@$IP" 'install -d -o build -g build /srv/workers /srv/workers/bin /srv/workers/sources /srv/builds/_log; chown build:build /srv/builds/_log'
diff --git a/tools/workers/page/workers.html b/tools/workers/page/workers.html
index c83c9f0a7..d668e6f50 100644
--- a/tools/workers/page/workers.html
+++ b/tools/workers/page/workers.html
@@ -40,6 +40,7 @@
   time[title] { cursor: help; border-bottom: 1px dotted var(--line); }
 
   /* the server */
+  .server + .server { margin-top: 12px; }
   .server { margin-top: 18px; background: var(--panel); border: 1px solid var(--line); position: relative; overflow: hidden; }
   .server.hot::after { content: ""; position: absolute; inset: auto -30% -70% -30%; height: 90%; background: radial-gradient(ellipse at center, rgba(242,84,27,.18), transparent 60%); pointer-events: none; animation: breathe 3.2s ease-in-out infinite; }
   @keyframes breathe { 0%, 100% { opacity: .5; } 50% { opacity: 1; } }
@@ -149,15 +150,7 @@
     <div class="live" id="live"><b></b><span id="stamp">waiting for the first write</span></div>
   </header>
 
-  <section class="server" id="server" aria-label="The build server">
-    <div class="head">
-      <div><div class="name" id="s-name">igneum-build-1<small>build server</small></div><div class="facts" id="s-facts"></div></div>
-      <div class="facts" id="s-right"></div>
-    </div>
-    <div class="cores" id="cores" aria-label="96 cores, busy share per core"></div>
-    <div class="corelabel"><span id="cores-l">cores</span><span id="cores-r"></span></div>
-    <div class="gauges" id="gauges"></div>
-  </section>
+  <div id="servers"></div>
 
   <h2>Workers <span>every machine that builds, tests or measures</span></h2>
   <div class="crew" id="crew"></div>
@@ -183,7 +176,7 @@
   <div class="sources" id="sources"></div>
 </div>
 <script type="module">
-import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, kindLabel, classLine, fmtDur, fmtDurShort, fmtBytes, fmtBps, fmtUptime, ukTime, ago, headline } from './shape.js?v=2';
+import { durationByKind, buildsPerHour, hitRateSeries, compareKinds, timeline, kindLabel, classLine, fmtDur, fmtDurShort, fmtBytes, fmtBps, fmtUptime, ukTime, ago, headline } from './shape.js?v=3';
 const $ = id => document.getElementById(id);
 const esc = s => String(s ?? '').replace(/[&<>"]/g, c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));
 const pill = (s, cls) => `<span class="pill ${esc(cls || String(s || '').toLowerCase())}">${esc(s || '?')}</span>`;
@@ -198,33 +191,30 @@ function arc(pct, cls) {
 }
 const tone = (pct, warn = 70, bad = 90) => pct >= bad ? 'bad' : pct >= warn ? 'warn' : '';
 
-function renderServer(b, src) {
-  const el = $('server');
-  if (!b) {
-    $('s-facts').innerHTML = `<span>${esc(src && src.error || 'no box facts yet')}</span>`; $('s-right').innerHTML = '';
-    $('cores').innerHTML = Array.from({ length: 96 }, () => '<i style="--h:2%"></i>').join(''); $('cores-r').textContent = 'no sample'; $('gauges').innerHTML = ''; el.classList.remove('hot'); return;
-  }
-  $('s-name').innerHTML = `${esc(b.name)}<small>${esc(b.os || 'build server')}</small>`;
-  $('s-facts').innerHTML = [`<b>${b.cores}</b> threads`, `<b>${esc(fmtUptime(b.uptime_s))}</b> up`, `load <b>${(b.load || []).map(x => Number(x).toFixed(1)).join(' / ')}</b>`, b.kernel ? `kernel <b>${esc(b.kernel)}</b>` : ''].filter(Boolean).map(x => `<span>${x}</span>`).join('');
+const FEEDS = { 'igneum-build-1': 'https://build.igneum.network/workers.json', 'igneum-build-2': 'https://build-2.igneum.network/workers.json', 'igneum-build-3': 'https://build-3.igneum.network/workers.json' };
+const boxesOf = d => (Array.isArray(d.boxes) && d.boxes.length ? d.boxes : d.box ? [{ ...d.box, source: d.sources && d.sources.box, headline: d.headline }] : []);
+function serverSection(b, i) {
+  const id = `srv${i}`;
+  if (!b || !b.cores) return `<section class="server" aria-label="${esc(b && b.name || 'build server')}"><div class="head"><div><div class="name">${esc(b && b.name || 'build server')}<small>build server</small></div><div class="facts"><span>${esc(b && b.source && b.source.error || 'no facts from this box yet')}</span></div></div></div><div class="cores">${Array.from({ length: 96 }, () => '<i style="--h:2%"></i>').join('')}</div><div class="corelabel"><span>no sample</span><span></span></div></section>`;
+  const per = (b.cpu && b.cpu.per_core) || []; const n = Math.max(per.length, b.cores || 96);
   const temps = (b.temps || []).map(x => `${esc(x.name)} <b>${x.c.toFixed(0)}°</b>`);
-  $('s-right').innerHTML = [...temps, b.net ? `net ↓<b>${esc(fmtBps(b.net.rx_bps))}</b> ↑<b>${esc(fmtBps(b.net.tx_bps))}</b>` : '', `<span title="${esc(b.collected_at)}">read ${esc(ago(b.collected_at))}</span>`].filter(Boolean).map(x => `<span>${x}</span>`).join('');
-  const per = (b.cpu && b.cpu.per_core) || [];
-  const n = Math.max(per.length, b.cores || 96);
-  $('cores').style.gridTemplateColumns = `repeat(${n > 64 ? 48 : n > 32 ? 32 : n}, 1fr)`;
-  $('cores').innerHTML = Array.from({ length: n }, (_, i) => { const p = per[i] ?? 0; return `<i class="${p >= 50 ? 'hot' : ''}" style="--h:${Math.max(2, p)}%" title="core ${i}: ${p}%"></i>`; }).join('');
-  $('cores-l').textContent = `${n} cores, ${b.cpu ? b.cpu.cores_busy : 0} busy`;
-  $('cores-r').textContent = b.cpu ? `${b.cpu.busy_pct}% of the box over the last second, ${b.compilers || 0} compilers running` : '';
-  el.classList.toggle('hot', !!(b.running && b.running.length));
   const m = b.mem || {}, d = b.disk || {}, sc = b.sccache;
   const hit = sc && sc.hit_rate_pct !== null && sc.hit_rate_pct !== undefined ? sc.hit_rate_pct : null;
-  $('gauges').innerHTML = [
-    `<div class="g">${arc(b.cpu ? b.cpu.busy_pct : 0, tone(b.cpu ? b.cpu.busy_pct : 0, 101, 101))}<div><div class="l">CPU</div><div class="v">${b.cpu ? b.cpu.busy_pct : 0}%</div><div class="s">load 1 min ${b.load ? Number(b.load[0]).toFixed(1) : '?'} of ${b.cores}</div></div></div>`,
-    `<div class="g">${arc(m.used_pct, tone(m.used_pct))}<div><div class="l">Memory</div><div class="v">${m.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes((m.used_kb || 0) * 1024))} of ${esc(fmtBytes((m.total_kb || 0) * 1024))}</div></div></div>`,
-    `<div class="g">${arc(d.used_pct, tone(d.used_pct, 80, 92))}<div><div class="l">Disk ${esc(d.mount || '/srv')}</div><div class="v">${d.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes(d.used_bytes))} used, ${esc(fmtBytes(d.avail_bytes))} free</div></div></div>`,
-    `<div class="g">${arc(hit ?? 0, 'cool')}<div><div class="l">sccache hits</div><div class="v">${hit === null ? '–' : hit + '%'}</div><div class="s">${sc ? `${sc.hits ?? 0} hit, ${sc.misses ?? 0} miss, ${esc(sc.cache_size || '?')} of ${esc(sc.max_size || '?')}` : 'sccache not answering'}</div></div></div>`,
-    `<div class="g">${arc(b.slots ? (b.slots.held.length / Math.max(1, b.slots.count)) * 100 : 0, 'good')}<div><div class="l">Build slots</div><div class="v">${b.slots ? `${b.slots.held.length}/${b.slots.count}` : '–'}</div><div class="s">${b.queue && b.queue.length ? `${b.queue.length} waiting` : 'nobody waiting'}</div></div></div>`,
-  ].join('');
+  const stale = b.source && !b.source.ok;
+  return `<section class="server ${b.running && b.running.length ? 'hot' : ''}" aria-label="${esc(b.name)}">
+    <div class="head"><div><div class="name">${esc(b.name)}<small>${esc(b.os || 'build server')}</small></div><div class="facts">${[`<b>${b.cores}</b> threads`, `<b>${esc(fmtUptime(b.uptime_s))}</b> up`, `load <b>${(b.load || []).map(x => Number(x).toFixed(1)).join(' / ')}</b>`, b.kernel ? `kernel <b>${esc(b.kernel)}</b>` : ''].filter(Boolean).map(x => `<span>${x}</span>`).join('')}</div></div>
+    <div class="facts">${[...temps, b.net ? `net ↓<b>${esc(fmtBps(b.net.rx_bps))}</b> ↑<b>${esc(fmtBps(b.net.tx_bps))}</b>` : '', `<span title="${esc(b.collected_at)}" ${stale ? 'style="color:var(--warn)"' : ''}>${stale ? 'STALE, ' : ''}read ${esc(ago(b.collected_at))}</span>`].filter(Boolean).map(x => `<span>${x}</span>`).join('')}</div></div>
+    <div class="cores" style="grid-template-columns:repeat(${n > 64 ? 48 : n > 32 ? 32 : n}, 1fr)" aria-label="${n} cores, busy share per core">${Array.from({ length: n }, (_, i) => { const p = per[i] ?? 0; return `<i class="${p >= 50 ? 'hot' : ''}" style="--h:${Math.max(2, p)}%" title="core ${i}: ${p}%"></i>`; }).join('')}</div>
+    <div class="corelabel"><span>${n} cores, ${b.cpu ? b.cpu.cores_busy : 0} busy</span><span>${b.cpu ? `${b.cpu.busy_pct}% of the box over the last second, ${b.compilers || 0} compilers running` : ''}</span></div>
+    <div class="gauges">
+      <div class="g">${arc(b.cpu ? b.cpu.busy_pct : 0)}<div><div class="l">CPU</div><div class="v">${b.cpu ? b.cpu.busy_pct : 0}%</div><div class="s">load 1 min ${b.load ? Number(b.load[0]).toFixed(1) : '?'} of ${b.cores}</div></div></div>
+      <div class="g">${arc(m.used_pct, tone(m.used_pct))}<div><div class="l">Memory</div><div class="v">${m.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes((m.used_kb || 0) * 1024))} of ${esc(fmtBytes((m.total_kb || 0) * 1024))}</div></div></div>
+      <div class="g">${arc(d.used_pct, tone(d.used_pct, 80, 92))}<div><div class="l">Disk ${esc(d.mount || '/srv')}</div><div class="v">${d.used_pct ?? '?'}%</div><div class="s">${esc(fmtBytes(d.used_bytes))} used, ${esc(fmtBytes(d.avail_bytes))} free</div></div></div>
+      <div class="g">${arc(hit ?? 0, 'cool')}<div><div class="l">sccache hits</div><div class="v">${hit === null ? '–' : hit + '%'}</div><div class="s">${sc ? `${sc.hits ?? 0} hit, ${sc.misses ?? 0} miss, ${esc(sc.cache_size || '?')} of ${esc(sc.max_size || '?')}` : 'sccache not answering'}</div></div></div>
+      <div class="g">${arc(b.slots ? (b.slots.held.length / Math.max(1, b.slots.count)) * 100 : 0, 'good')}<div><div class="l">Build slots</div><div class="v">${b.slots ? `${b.slots.held.length}/${b.slots.count}` : '–'}</div><div class="s">${b.queue && b.queue.length ? `${b.queue.length} waiting${b.queue.some(q => q.priority === 'gate') ? ', a gate first' : ''}` : 'nobody waiting'}</div></div></div>
+    </div></section>`;
 }
+function renderServers() { const bx = boxesOf(D); $('servers').innerHTML = bx.length ? bx.map(serverSection).join('') : serverSection(null, 0); }
 
 function slotBar(slots, count) {
   const held = new Map((slots && slots.held || []).map(h => [h.slot, h]));
@@ -232,10 +222,12 @@ function slotBar(slots, count) {
   return `<div class="slots">${names.map(nm => `<i class="${held.has(nm) ? 'held' : ''}" title="${esc(nm)}${held.has(nm) ? ': ' + esc(held.get(nm).label) : ': free'}"></i>`).join('')}</div>`;
 }
 function renderCrew() {
-  const b = D.box, mac = D.mac, pcs = (D.pcs && D.pcs.machines) || [];
+  const mac = D.mac, pcs = (D.pcs && D.pcs.machines) || [];
   const cards = [];
-  if (b) cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(b.name)}</div><div class="meta">Hetzner, ${b.cores} threads, ${esc(fmtBytes((b.mem && b.mem.total_kb || 0) * 1024))}, RAID 1 NVMe</div></div>${pill(b.running && b.running.length ? 'building' : 'idle', b.running && b.running.length ? 'running' : '')}</div><div class="doing">${b.running && b.running.length ? b.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('<br>') : 'Slot free. The next build-remote.sh or cross-remote.sh from any worktree takes it.'}</div>${slotBar(b.slots, b.slots ? b.slots.count : 1)}<div class="foot"><span>${b.recent ? b.recent.length : 0} builds logged</span><span>${esc(ago(b.collected_at))}</span></div></div>`);
-  else cards.push(`<div class="w"><div class="top"><div><div class="card">igneum-build-1</div><div class="meta">build server</div></div>${pill('unreachable', 'error')}</div><div class="doing">${esc(D.sources && D.sources.box && D.sources.box.error || 'no facts from the box')}</div></div>`);
+  for (const bx of boxesOf(D)) {
+    if (bx && bx.cores) cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx.name)}</div><div class="meta">Hetzner, ${bx.cores} threads, ${esc(fmtBytes((bx.mem && bx.mem.total_kb || 0) * 1024))}, RAID 1 NVMe</div></div>${pill(bx.running && bx.running.length ? 'building' : 'idle', bx.running && bx.running.length ? 'running' : '')}</div><div class="doing">${bx.running && bx.running.length ? bx.running.map(r => esc(`${r.worktree || '?'}: ${kindLabel(r.kind)}`)).join('<br>') : 'Slot free. The next build-remote.sh or cross-remote.sh routed here takes it.'}</div>${slotBar(bx.slots, bx.slots ? bx.slots.count : 1)}<div class="foot"><span>${bx.recent ? bx.recent.length : 0} builds logged</span><span>${esc(ago(bx.collected_at))}</span></div></div>`);
+    else cards.push(`<div class="w"><div class="top"><div><div class="card">${esc(bx && bx.name || 'build server')}</div><div class="meta">build server</div></div>${pill('unreachable', 'error')}</div><div class="doing">${esc(bx && bx.source && bx.source.error || 'no facts from this box')}</div></div>`);
+  }
   if (mac) {
     const held = mac.slots.held || [], runs = held.filter(h => /^run-/.test(h.slot)), builds = held.filter(h => /^build-/.test(h.slot)), meas = held.filter(h => h.slot === 'measure');
     const doing = held.length ? held.map(h => `${esc(h.slot)}: ${esc((h.worktree ? h.worktree + ' ' : '') + (h.command || h.label || ''))}`.slice(0, 140)).join('<br>') : 'Nothing under with-lock.sh. macOS binaries and the DMG build here; everything else goes to the box or the PCs.';
@@ -251,7 +243,7 @@ function renderCrew() {
 
 function nowItems() {
   const out = [];
-  for (const r of (D.box && D.box.running) || []) out.push({ where: D.box.name, slot: r.slot, title: `${kindLabel(r.kind)}`, sub: `${r.worktree || '?'}/${r.crate || ''}${r.agent ? ' · ' + r.agent : ''}`, cmd: r.command, started_at: r.started_at, kind: r.kind, extra: [r.target || '', classLine(r)].filter(Boolean).join(' · ') });
+  for (const bx of boxesOf(D)) for (const r of bx.running || []) out.push({ where: bx.name, slot: r.slot, title: `${kindLabel(r.kind)}`, sub: `${r.worktree || '?'}/${r.crate || ''}${r.agent ? ' · ' + r.agent : ''}`, cmd: r.command, started_at: r.started_at, kind: r.kind, extra: [r.target || '', classLine(r)].filter(Boolean).join(' · ') });
   for (const h of (D.mac && D.mac.slots.held) || []) out.push({ where: D.mac.name, slot: h.slot, title: h.slot === 'measure' ? 'measurement' : h.slot.startsWith('run') ? 'functional run' : kindLabel(h.kind), sub: h.worktree || 'this Mac', cmd: h.command || h.label, started_at: h.since, kind: h.kind, extra: h.waited_s ? `waited ${fmtDur(h.waited_s)}` : '' });
   for (const pc of (D.pcs && D.pcs.machines) || []) if (pc.running) out.push({ where: pc.name, slot: 'relay job', title: `${pc.running.kind} ${pc.running.job}`, sub: pc.running.title || pc.machine || '', cmd: pc.running.stage ? `stage ${pc.running.stage}` : 'reports every 5 min while it runs', started_at: pc.running.started_at, kind: pc.running.kind, extra: pc.running.last_report_at ? `last report ${ago(pc.running.last_report_at)}` : '' });
   return out;
@@ -292,7 +284,7 @@ function renderBackground() {
 }
 function renderQueue() {
   const items = [];
-  for (const w of (D.box && D.box.queue) || []) items.push({ where: D.box.name, title: w.label ? kindLabel(w.kind) : 'a build', sub: w.label ? `${w.worktree || ''}${w.crate ? '/' + w.crate : ''} ${w.command || ''}`.trim() : 'label unknown: no wait file for this flock', since: w.since, wait: w.waiting_s, gate: w.priority === 'gate', extra: [w.agent || '', classLine(w)].filter(Boolean).join(' · ') });
+  for (const bx of boxesOf(D)) for (const w of bx.queue || []) items.push({ where: bx.name, title: w.label ? kindLabel(w.kind) : 'a build', sub: w.label ? `${w.worktree || ''}${w.crate ? '/' + w.crate : ''} ${w.command || ''}`.trim() : 'label unknown: no wait file for this flock', since: w.since, wait: w.waiting_s, gate: w.priority === 'gate', extra: [w.agent || '', classLine(w)].filter(Boolean).join(' · ') });
   for (const w of (D.mac && D.mac.queue) || []) items.push({ where: D.mac.name, title: `${w.mode} slot`, sub: w.command, since: null, wait: w.waiting_s });
   for (const pc of (D.pcs && D.pcs.machines) || []) for (const q of pc.queue) items.push({ where: pc.name, title: `${q.kind} ${q.job}`, sub: q.title || '', since: null, wait: null, until: q.expires_at });
   $('queue-sub').textContent = items.length ? `${items.length} waiting` : '';
@@ -302,14 +294,14 @@ function renderQueue() {
 
 function doneRows() {
   const rows = [];
-  for (const b of (D.box && D.box.recent) || []) rows.push({ key: b.id, where: D.box.name, at: b.end || b.start, title: `${kindLabel(b.kind)} · ${b.worktree || '?'}${b.agent ? ' · ' + b.agent : ''}`, meta: b.command || '', ok: b.ok, secs: b.secs, status: b.ok ? 'ok' : `exit ${b.exit}`, lines: b.closing || null, raw: b });
+  for (const bx of boxesOf(D)) for (const b of bx.recent || []) rows.push({ key: b.id, where: bx.name, at: b.end || b.start, title: `${kindLabel(b.kind)} · ${b.worktree || '?'}${b.agent ? ' · ' + b.agent : ''}`, meta: b.command || '', ok: b.ok, secs: b.secs, status: b.ok ? 'ok' : `exit ${b.exit}`, lines: b.closing || null, raw: b });
   for (const pc of (D.pcs && D.pcs.machines) || []) for (const r of pc.recent) rows.push({ key: `${pc.id}-${r.job}`, where: pc.name, at: r.finished_at, title: `${r.kind} ${r.job}`, meta: r.title || r.summary || '', ok: r.ok, secs: r.secs, status: r.status, lines: [r.summary, ...(r.errors || [])].filter(Boolean), raw: r });
   rows.sort((a, b) => String(b.at || '').localeCompare(String(a.at || '')));
   return rows;
 }
 function renderDone() {
   const rows = doneRows().slice(0, 40);
-  const boxRows = timeline((D.box && D.box.recent) || [], { limit: 200 }); const closing = new Map(boxRows.map(r => [r.id, r.closing]));
+  const boxRows = timeline(boxesOf(D).flatMap(bx => bx.recent || []), { limit: 600 }); const closing = new Map(boxRows.map(r => [r.id, r.closing]));
   if (!rows.length) { $('done').innerHTML = `<div class="empty">No finished work yet. The box appends one line per build to /srv/builds/_log/builds.jsonl when bs_remote_run ends${D.box && D.box.log && !D.box.log.present ? ' (the file does not exist yet)' : ''}; the PCs' closing reports come from the relay intake.</div>`; return; }
   const max = Math.max(60, ...rows.map(r => Number(r.secs) || 0));
   $('done').innerHTML = rows.map(r => {
@@ -322,7 +314,7 @@ function renderDone() {
 }
 
 function renderHeadline() {
-  const rows = headline(D.headline);
+  const bx = boxesOf(D).filter(b => b.headline); const rows = bx.length ? bx.flatMap(b => headline(b.headline).map(r => ({ ...r, label: bx.length > 1 ? `${r.label} · ${b.name}` : r.label }))) : headline(D.headline);
   $('headline').innerHTML = rows.map(r => {
     const max = Math.max(r.box_s || 0, r.mac_s || 0, r.pc_s || 0, 1);
     return `<div class="h3"><div class="l">${esc(r.label)}</div><div class="v">${r.box_s === null ? '<span style="color:var(--muted);font-size:14px">not measured yet</span>' : esc(fmtDur(r.box_s)) + '<small>box</small>'}</div><div class="cmp">${r.mac_s !== null ? `Mac <b>${esc(fmtDur(r.mac_s))}</b>${r.speedup ? ` (${r.speedup}× slower)` : ''}` : 'Mac <b>not measured on this tree</b>'}${r.pc_s !== null ? ` · PC 1 <b>${esc(fmtDur(r.pc_s))}</b>` : ''}</div>${r.box_s !== null ? `<div class="meter"><i style="width:${(r.box_s / max * 100).toFixed(1)}%"></i>${r.mac_s !== null ? `<em style="left:${(r.mac_s / max * 100).toFixed(1)}%" title="Mac"></em>` : ''}${r.pc_s !== null ? `<em style="left:${(r.pc_s / max * 100).toFixed(1)}%;background:var(--cool)" title="PC 1"></em>` : ''}</div>` : ''}<div class="s">${esc(r.note || r.what)}${r.measured_at ? ` · measured ${t(r.measured_at, true)}` : ''}</div></div>`;
@@ -344,7 +336,7 @@ function bars(buckets) {
   return `<svg viewBox="0 0 ${W} ${H}" aria-label="builds per hour today"><line x1="4" y1="${H - B}" x2="${W - 4}" y2="${H - B}" stroke="#26262d"/>${buckets.map((b, i) => { const h = (b.n / max) * (H - T - B); const ok = b.n ? (b.ok / b.n) * h : 0; const x = 4 + i * bw; return `<g><title>${String(i).padStart(2, '0')}:00 UTC: ${b.n} build${b.n === 1 ? '' : 's'}, ${b.failed} failed</title><rect x="${(x + 1).toFixed(1)}" y="${(H - B - h).toFixed(1)}" width="${(bw - 2).toFixed(1)}" height="${h.toFixed(1)}" fill="#ff6b5e"/><rect x="${(x + 1).toFixed(1)}" y="${(H - B - ok).toFixed(1)}" width="${(bw - 2).toFixed(1)}" height="${ok.toFixed(1)}" fill="#5fd08a"/></g>`; }).join('')}${[0, 6, 12, 18, 23].map(h => `<text x="${(4 + h * bw + bw / 2).toFixed(1)}" y="${H - 5}" text-anchor="middle" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${String(h).padStart(2, '0')}Z</text>`).join('')}<text x="4" y="${T + 4}" font-family="IBM Plex Mono, monospace" font-size="9" fill="#8e8b86">${max}</text></svg>`;
 }
 function renderAnalytics() {
-  const builds = (D.box && D.box.recent) || [];
+  const builds = boxesOf(D).flatMap(bx => (bx.recent || []).map(b => ({ ...b, host: b.host || bx.name })));
   const day = new Date().toISOString().slice(0, 10);
   const todays = builds.filter(b => String(b.end || b.start || '').startsWith(day));
   $('an-sub').textContent = builds.length ? `${builds.length} logged builds, ${todays.length} today, ${todays.filter(b => b.ok).length} passed` : '';
@@ -365,7 +357,7 @@ function renderAnalytics() {
 function renderSources() {
   const s = D.sources || {};
   const one = (name, v, what) => `<span>${esc(name)} <b class="${v && v.ok ? '' : v && v.at ? 'warn' : 'no'}">${v && v.ok ? 'live' : v && v.at ? 'stale' : 'missing'}</b>${v && v.at ? ' ' + esc(ago(v.at)) : ''}${v && v.error ? ': ' + esc(v.error) : ''}${!v || v.ok ? '' : ' · ' + esc(what)}</span>`;
-  $('sources').innerHTML = [one('box', s.box, 'collect.mjs on the box, systemd timer every 30 s'), one('mac', s.mac, 'push.mjs on the Mac, launchd every 60 s'), one('pcs', s.pcs, 'relay intake read by push.mjs'), `<span>${esc(D._feed || '')}: fetched ${esc(ago(D._fetched))}, written ${t(D.generated_at)} by ${esc(D.generated_by || '?')}</span>`, D.box && D.box.log ? `<span>log ${D.box.log.present ? `${D.box.log.total} lines${D.box.log.bad_lines ? `, ${D.box.log.bad_lines} unreadable` : ''}` : 'not written yet'}</span>` : ''].join('');
+  $('sources').innerHTML = [...boxesOf(D).map(b => one(b.name || 'box', b.source || s.box, 'collect.mjs on the box, systemd timer every 30 s')), one('mac', s.mac, 'push.mjs on the Mac, launchd every 60 s'), one('pcs', s.pcs, 'relay intake read by push.mjs'), `<span>${esc(D._feed || '')}: fetched ${esc(ago(D._fetched))}, written ${t(D.generated_at)} by ${esc(D.generated_by || '?')}</span>`, ...boxesOf(D).filter(b => b.log).map(b => `<span>${esc(b.name)} log ${b.log.present ? `${b.log.total} lines${b.log.bad_lines ? `, ${b.log.bad_lines} unreadable` : ''}` : 'not written yet'}</span>`)].join('');
 }
 
 function apply(d) {
@@ -374,20 +366,30 @@ function apply(d) {
   const live = $('live'); live.className = 'live' + (age > 900 ? ' down' : age > 420 ? ' stale' : '');
   const u = ukTime(d.generated_at);
   $('stamp').innerHTML = `${age > 420 ? 'stale: ' : ''}written ${t(d.generated_at)} (${esc(ago(d.generated_at))})${d._feed ? ' · ' + esc(d._feed) : ''}`;
-  renderServer(d.box, d.sources && d.sources.box); renderCrew(); renderNow(); renderQueue(); renderBackground(); renderDone(); renderHeadline(); renderAnalytics(); renderSources();
+  const el = $('servers'); if (el && !el.innerHTML) el.innerHTML = '';
+  renderServers(); renderCrew(); renderNow(); renderQueue(); renderBackground(); renderDone(); renderHeadline(); renderAnalytics(); renderSources();
 }
-const LIVE_URL = 'https://build.igneum.network/workers.json';   // the box itself, 30 s fresh (Caddy, read-only); the folder copy is up to 5 min behind
 let skipLive = 0, feed = '';
+async function fetchJson(url, ms) { const r = await fetch(`${url}?t=${Date.now()}`, { cache: 'no-store', signal: AbortSignal.timeout(ms) }); if (!r.ok) throw new Error(`http ${r.status}`); return r.json(); }
 async function load() {
-  let d = null, err = null;
-  if (!demo && skipLive <= 0) {
-    try { const r = await fetch(`${LIVE_URL}?t=${Date.now()}`, { cache: 'no-store', signal: AbortSignal.timeout(4000) }); if (!r.ok) throw new Error(`http ${r.status}`); d = await r.json(); feed = 'live from the box'; }
-    catch (e) { skipLive = 4; err = e; }   // the box not answering: use the edge copy for the next minute before trying again
+  let edge = null, d = null;
+  try { const file = demo ? 'workers.demo.json' : 'workers.json'; edge = await fetchJson(`${base}/${file}`, 8000); }
+  catch (e) { if (!D && (demo || skipLive > 0)) { $('live').className = 'live down'; $('stamp').textContent = `workers.json not reachable (${e && e.message || e})`; } }
+  if (demo) { d = edge; feed = 'demo document'; }
+  else if (skipLive <= 0) {
+    // the boxes the edge copy knows (or build-1 alone before the first push), each from its own Caddy feed, 4 s each, in parallel
+    const names = edge ? boxesOf(edge).map(b => b.name).filter(Boolean) : ['igneum-build-1'];
+    const results = await Promise.all(names.map(n => FEEDS[n] ? fetchJson(FEEDS[n], 4000).then(j => ({ n, j })).catch(() => ({ n, j: null })) : Promise.resolve({ n, j: null })));
+    const liveOnes = results.filter(r => r.j);
+    if (liveOnes.length) {
+      const first = liveOnes[0].j;
+      const boxes = results.map(r => r.j ? { ...r.j.box, headline: r.j.headline, source: { ok: true, at: r.j.generated_at } } : (edge && boxesOf(edge).find(b => b.name === r.n)) || { name: r.n, source: { ok: false, error: 'no live feed and no edge copy for this box' } });
+      d = { ...first, boxes, mac: first.mac || (edge && edge.mac), pcs: first.pcs || (edge && edge.pcs), baselines: first.baselines || (edge && edge.baselines), sources: { ...(first.sources || {}), box: boxes[0].source } };
+      feed = liveOnes.length === results.length ? `live from ${liveOnes.length === 1 ? 'the box' : liveOnes.length + ' boxes'}` : `live from ${liveOnes.map(r => r.n).join(', ')}, edge copy for the rest`;
+    } else skipLive = 4;   // nothing answered: use the edge copy for the next minute before trying again
   } else skipLive--;
-  if (!d) {
-    try { const file = demo ? 'workers.demo.json' : 'workers.json'; const r = await fetch(`${base}/${file}?t=${Date.now()}`, { cache: 'no-store' }); if (!r.ok) throw new Error(`http ${r.status}`); d = await r.json(); feed = demo ? 'demo document' : 'edge copy, up to 5 min behind the box'; }
-    catch (e) { $('live').className = 'live down'; $('stamp').textContent = `workers.json not reachable (${e && e.message || e})`; if (!D) { $('now').innerHTML = $('queue').innerHTML = `<div class="empty">No workers.json next to this page yet and the box does not answer. The Mac pusher writes the copy every 60 s and the fleet deploy carries it within 5 min.</div>`; } return; }
-  }
+  if (!d) { d = edge; feed = 'edge copy, up to 5 min behind the box'; }
+  if (!d) { if (!D) { $('now').innerHTML = $('queue').innerHTML = `<div class="empty">No workers.json next to this page yet and no box answers. The Mac pusher writes the copy every 60 s and the fleet deploy carries it within 5 min.</div>`; } return; }
   d._feed = feed; apply(d);
 }
 await load(); setInterval(load, 15000); setInterval(tick, 1000); setInterval(() => { if (D) { renderSources(); } }, 10000);
diff --git a/tools/workers/push.mjs b/tools/workers/push.mjs
index dcac78327..5d269434d 100644
--- a/tools/workers/push.mjs
+++ b/tools/workers/push.mjs
@@ -31,7 +31,10 @@ const cfg = n => { try { return readFileSync(join(homedir(), '.config', 'igneum'
 const iso = ms => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
 const log = (...a) => console.error(`${iso(Date.now())} push: ${a.join(' ')}`);
 const KEY = process.env.IGNEUM_BUILD_KEY || join(homedir(), '.ssh', 'igneum_ed25519');
-const HOST = (cfg('build-server') || '').split('\n')[0].trim();          // build@<ip>
+// every build box: ~/.config/igneum/build-server (build-1), build-server-2, build-server-3 ... one line build@<ip> each
+const HOSTS = [['build-server', 'igneum-build-1'], ['build-server-2', 'igneum-build-2'], ['build-server-3', 'igneum-build-3']]
+  .map(([f, n]) => ({ file: f, name: n, host: (cfg(f) || '').split('\n')[0].trim() })).filter(h => h.host);
+const HOST = HOSTS.length ? HOSTS[0].host : '';
 const DLSITE = cfg('dlsite-dir'), FLEET = cfg('fleet-path');
 const STALE_LIVE_S = 360;
 const SSH_OPTS = ['-i', KEY, '-o', 'BatchMode=yes', '-o', 'StrictHostKeyChecking=accept-new', '-o', 'ConnectTimeout=10', '-o', 'ControlMaster=auto', '-o', `ControlPath=${join(homedir(), '.ssh', 'cm', 'igneum-workers-%r@%h:%p')}`, '-o', 'ControlPersist=600'];
@@ -140,13 +143,13 @@ async function pcs() {
 }
 
 // ---- 3 and 4. the box -------------------------------------------------------------------------------------------------------
-function ssh(args, input) { return spawnSync('ssh', [...SSH_OPTS, HOST, ...args], { encoding: 'utf8', input, timeout: 30000, maxBuffer: 16 * 1024 * 1024 }); }
-function dropToBox(name, doc) {
-  const r = ssh([`cat > /srv/workers/sources/.${name}.tmp && mv /srv/workers/sources/.${name}.tmp /srv/workers/sources/${name}.json`], JSON.stringify(doc));
+function ssh(args, input, host = HOST) { return spawnSync('ssh', [...SSH_OPTS, host, ...args], { encoding: 'utf8', input, timeout: 30000, maxBuffer: 16 * 1024 * 1024 }); }
+function dropToBox(name, doc, host = HOST) {
+  const r = ssh([`cat > /srv/workers/sources/.${name}.tmp && mv /srv/workers/sources/.${name}.tmp /srv/workers/sources/${name}.json`], JSON.stringify(doc), host);
   if (r.status !== 0) throw new Error(`scp ${name}: ${(r.stderr || '').trim().slice(0, 200) || 'ssh failed'}`);
 }
-function pullFromBox() {
-  const r = ssh(['cat /srv/workers/workers.json; echo; echo __SEP__; cat /srv/workers/headline.json 2>/dev/null || echo null']);
+function pullFromBox(host = HOST) {
+  const r = ssh(['cat /srv/workers/workers.json; echo; echo __SEP__; cat /srv/workers/headline.json 2>/dev/null || echo null'], undefined, host);
   if (r.status !== 0) throw new Error((r.stderr || '').trim().slice(0, 200) || 'ssh failed');
   const [w, h] = r.stdout.split('__SEP__');
   return { workers: JSON.parse(w), headline: JSON.parse((h || 'null').trim() || 'null') };
@@ -173,17 +176,23 @@ async function main() {
   try { doc.mac = macSlots(now); doc.sources.mac = { ok: true, at: doc.mac.collected_at }; } catch (e) { doc.sources.mac = { ok: false, error: String(e.message || e) }; }
   // the PCs
   try { doc.pcs = await pcs(); doc.sources.pcs = doc.pcs.error ? { ok: false, error: doc.pcs.error } : { ok: true, at: doc.pcs.collected_at }; } catch (e) { doc.sources.pcs = { ok: false, error: `intake: ${String(e.message || e).slice(0, 160)}` }; }
-  // the box: drop ours, pull theirs
-  if (!HOST) doc.sources.box = { ok: false, error: 'no ~/.config/igneum/build-server' };
-  else {
+  // the boxes: drop ours on each, pull each one's file; boxes[] carries every box, box stays the first for older readers
+  doc.boxes = [];
+  if (!HOSTS.length) doc.sources.box = { ok: false, error: 'no ~/.config/igneum/build-server' };
+  for (const h of HOSTS) {
     try {
-      if (doc.mac) dropToBox('mac', doc.mac);
-      if (doc.pcs && !doc.pcs.error) dropToBox('pcs', doc.pcs);
-      const { workers, headline } = pullFromBox();
-      doc.box = workers.box; doc.headline = headline;
+      if (doc.mac) dropToBox('mac', doc.mac, h.host);
+      if (doc.pcs && !doc.pcs.error) dropToBox('pcs', doc.pcs, h.host);
+      const { workers, headline } = pullFromBox(h.host);
       const age = (now - Date.parse(workers.generated_at)) / 1000;
-      doc.sources.box = { ok: age < 120, at: workers.generated_at, age_s: Math.round(age), ...(age >= 120 ? { error: `the box's collector last wrote ${Math.round(age)} s ago (timer igneum-workers.timer)` } : {}) };
-    } catch (e) { doc.sources.box = { ok: false, error: `ssh to the box failed: ${String(e.message || e).slice(0, 160)}` }; }
+      const b = { ...workers.box, headline, feed: `https://${h.name === 'igneum-build-1' ? 'build' : h.name.replace('igneum-', '')}.igneum.network/workers.json`, source: { ok: age < 120, at: workers.generated_at, age_s: Math.round(age), ...(age >= 120 ? { error: `the box's collector last wrote ${Math.round(age)} s ago (timer igneum-workers.timer)` } : {}) } };
+      doc.boxes.push(b);
+      if (!doc.box) { doc.box = workers.box; doc.headline = headline; doc.sources.box = b.source; }
+    } catch (e) {
+      const src = { ok: false, error: `ssh to ${h.name} failed: ${String(e.message || e).slice(0, 160)}` };
+      doc.boxes.push({ name: h.name, source: src });
+      if (!doc.sources.box) doc.sources.box = src;
+    }
   }
   if (!doc.headline) doc.headline = null;
   const text = JSON.stringify(doc);
@@ -197,7 +206,7 @@ async function main() {
     if (age > STALE_LIVE_S) { const ok = deploy(); why = ok ? `deployed (live copy was ${age === Infinity ? 'missing' : Math.round(age) + ' s old'})` : 'DEPLOY FAILED'; }
     else why = `live copy ${Math.round(age)} s old, no deploy`;
   }
-  log(`box ${doc.sources.box.ok ? 'ok' : 'NO: ' + doc.sources.box.error}; mac ${doc.sources.mac.ok ? `${doc.mac.slots.held.length} held, ${doc.mac.queue.length} waiting` : 'NO'}; pcs ${doc.sources.pcs.ok ? doc.pcs.machines.map(m => `${m.name} ${m.running ? 'running ' + m.running.job : 'idle'} q${m.queue.length}`).join(', ') : 'NO: ' + doc.sources.pcs.error}; ${why}`);
+  log(`boxes ${doc.boxes.map(b => `${b.name} ${b.source && b.source.ok ? 'ok' : 'NO'}`).join(', ')}; mac ${doc.sources.mac.ok ? `${doc.mac.slots.held.length} held, ${doc.mac.queue.length} waiting` : 'NO'}; pcs ${doc.sources.pcs.ok ? doc.pcs.machines.map(m => `${m.name} ${m.running ? 'running ' + m.running.job : 'idle'} q${m.queue.length}`).join(', ') : 'NO: ' + doc.sources.pcs.error}; ${why}`);
 }
 
 if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {