workers merge: the Mac mini row (igneum-mini, the Mac build box, M6) from a mini.json source its launchd collector pushes to build-1; down after 120 s

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

infra/build-server/lease.sh

@@ -40,6 +40,10 @@
 # lease-<pid> (holder line "pid N since HH:MM:SSZ cores <set>: <label>; owner=<agent>"), quiet (holder line), wait-<pid>.
 set -uo pipefail
 LOCKS="${IGNEUM_BUILD_SLOTS_DIR:-/srv/builds/_locks}"; LOGS="${IGNEUM_BUILD_LOG_DIR:-/srv/builds/_log}"
+# A waiter whose process is gone leaves its wait file behind (a killed gate, a lost ssh); the priority read (a higher-class waiter
+# takes the next freed cores first) then blocks every live lower-class waiter for ever. build-2 held seven such ghosts on 8 Oct 2026
+# (the oldest 2 h; one adv-class ghost ahead of every release waiter). Pruned at every status read and before every priority read.
+prune_dead_waiters() { local f p; for f in "$LOCKS"/wait-*; do [ -e "$f" ] || continue; p=${f##*/wait-}; case "$p" in ''|*[!0-9]*) continue;; esac; kill -0 "$p" 2>/dev/null || rm -f -- "$f"; done 2>/dev/null; }
 REAP_S="${LEASE_REAP_S:-300}"
 now() { date -u +%H:%M:%SZ; }
 say() { echo "lease: $*" >&2; }
@@ -87,7 +91,7 @@ status() {
   echo "leases:"; for f in "$LOCKS"/lease-*; do [ -s "$f" ] && echo "  $(cat "$f")"; done 2>/dev/null
   echo "quiet: $(cat "$LOCKS/quiet" 2>/dev/null)"
   echo "held cores: $(held_cores)"
-  echo "waiting:"; for f in "$LOCKS"/wait-*; do [ -s "$f" ] && echo "  $(cat "$f")"; done 2>/dev/null
+  prune_dead_waiters; echo "waiting:"; for f in "$LOCKS"/wait-*; do [ -s "$f" ] && echo "  $(cat "$f")"; done 2>/dev/null
   echo "pre-emptions (last 10 min):"; find "$LOCKS" -maxdepth 1 -name 'preempt-*' -mmin -10 -type f 2>/dev/null | while read -r f; do echo "  $(cat "$f")"; done; find "$LOCKS" -maxdepth 1 -name 'preempt-*' -mmin +10 -type f -delete 2>/dev/null
 }
 run_cores() {
@@ -120,8 +124,11 @@ run_pool() {
   local want="$1"; shift; local label="" owner="${IGNEUM_AGENT:-unknown}" nice=10 min="" reserve="${LEASE_POOL_RESERVE:-8}" class="" rank mem=""
   while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --nice) nice="$2"; shift 2 ;; --min) min="$2"; shift 2 ;; --class) class="$2"; shift 2 ;; --priority) class=v5; shift ;; --mem) mem="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
   if [ -z "$class" ]; then
-    case "$owner" in release|shipper|build-server) class=release ;; class-v5) class=v5 ;; measure*) class=measure ;; esac
+    # the top class widened (the coordinator, 8 Oct 2026 19:2x BST, with build-7/8/9 joining): the P01 campaign, the object matrix,
-    [ -n "$class" ] || case "$label" in *release*|*canary*|*pair*) class=release ;; *"v5 gate"*|*"v5 kit"*) class=v5 ;; *measure*) class=measure ;; *) class=adv ;; esac
+    # the census, the family gate, the review verdicts, the fast-time and freeze cases, and their owners (hash lane, ci-steward, census,
+    # lane D, enforced-proving, the node lane's object gates) take every freed core before v5, measure and adv
+    case "$owner" in release|shipper|build-server|hash-lane*|ci-steward|steward|census*|lane-d|family-gate|enforced-proving|node-lane|node-gates) class=release ;; class-v5) class=v5 ;; measure*) class=measure ;; esac
+    [ -n "$class" ] || case "$label" in *release*|*canary*|*pair*|*P01*|*"object matrix"*|*census*|*"family gate"*|*review*|*fast-time*|*freeze*) class=release ;; *"v5 gate"*|*"v5 kit"*) class=v5 ;; *measure*) class=measure ;; *) class=adv ;; esac
   fi
   # ranks: release 0, v5 1, measure 2 (the "ca4 research" label first among them: rank 2, the other measure rows rank 3; the founder's
   # efficiency priority through main, 7 Oct 2026 21:40 BST), adv 4
@@ -151,8 +158,13 @@ run_pool() {
   while :; do
     fds=(); cores=(); taken=0
     # a waiter yields while a HIGHER class waits (its wait line carries "class <name>/<rank>")
-    higher=$(cat "$LOCKS"/wait-* 2>/dev/null | grep -v "^pid $$ " | sed -n 's/.* class [a-z0-9]*\/\([0-9]\):.*/\1/p' | awk -v r="$rank" '$1 < r' | head -1)
+    prune_dead_waiters; higher=$(cat "$LOCKS"/wait-* 2>/dev/null | grep -v "^pid $$ " | sed -n 's/.* class [a-z0-9]*\/\([0-9]\):.*/\1/p' | awk -v r="$rank" '$1 < r' | head -1)
     if [ -z "$higher" ]; then
+      # one round at a time (8 Oct 2026 19:3x BST, build-2: three waiters that started within a minute of each other polled on the same
+      # 10 s step, each took every free core for its check and released; their rounds interleaved, every round saw the others' cores
+      # taken, every waiter read "0 free" for 20 min with nothing running: a livelock). The round mutex serialises take, check and
+      # release; the sleep below is jittered so the rounds never fall back into step.
+      exec {rfd}>>"$LOCKS/pool-round"; flock "$rfd"
       for ((c = reserve; c < ncpu && taken < want; c++)); do
         exec {fd}>>"$LOCKS/core-$c"
         if flock -n "$fd"; then fds+=("$fd"); cores+=("$c"); taken=$((taken + 1)); else exec {fd}>&-; fi
@@ -160,10 +172,11 @@ run_pool() {
       memshort=""
       if [ "$taken" -ge "$min" ]; then
         resid=$(residents_gb); heldm=$(held_mem_gb)
-        if [ $(( resid + heldm + mem )) -le "$ceiling" ]; then break; fi
+        if [ $(( resid + heldm + mem )) -le "$ceiling" ]; then exec {rfd}>&-; break; fi
         memshort="memory: residents $resid GB + held leases $heldm GB + ask $mem GB > ceiling $ceiling GB"
       fi
       for fd in "${fds[@]}"; do exec {fd}>&-; done
+      exec {rfd}>&-
     fi
     waited=$(( $(date +%s) - t0 ))
     [ "$waited" -ge 7200 ] && { if [ -n "$memshort" ]; then say "gave up after 2 h: $memshort"; exit 76; else say "gave up waiting for $min free pool cores after 2 h"; exit 75; fi; }
@@ -195,7 +208,7 @@ run_pool() {
         say "pool short for $waited s: sent TERM to the oldest lower-class holder's command (pid $vpid, $vclass, $vcores cores; $freed of $need cores freed)"
       done
     fi
-    sleep "$step"
+    sleep "$(( step + RANDOM % 7 ))"
   done
   waited=$(( $(date +%s) - t0 ))
   local set; set=$(printf '%s,' "${cores[@]}"); set=${set%,}

infra/build-server/provision.sh

@@ -255,7 +255,8 @@ step_dirs() {
 step_lease_tool() {
   local src="/srv/igneum.git" want have=""
   install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" /srv/builds/_bin
-  want=$(git -C "$src" show master:infra/build-server/lease.sh 2>/dev/null) || { ok lease-tool "mirror has no master yet; run-from-mac.sh installs it on the next provision"; return; }
+  # the mirror is build-owned: read it as build (root's git refuses a dubious-ownership repo, so build-6 never got the lease tool from this step, 8 Oct 2026)
+  want=$(as_build "git -C '$src' show master:infra/build-server/lease.sh" 2>/dev/null) || { ok lease-tool "mirror has no master yet; run-from-mac.sh installs it on the next provision"; return; }
   [ -f /srv/builds/_bin/lease ] && have=$(cat /srv/builds/_bin/lease)
   if [ "$want" = "$have" ]; then ok lease-tool "/srv/builds/_bin/lease is the mirror's master copy"; return; fi
   printf '%s\n' "$want" > /srv/builds/_bin/lease.new; chmod 755 /srv/builds/_bin/lease.new; chown "$BUILD_USER:$BUILD_USER" /srv/builds/_bin/lease.new

tools/workers/merge-workers.mjs

@@ -0,0 +1,25 @@
+// Build-1-native workers-page merge (8 Oct 2026, the founder's rule: published from build-1, never the Mac, no secret, no key).
+// Reads every box's own workers.json over its public https host (build.igneum.network for build-1, build-N.igneum.network for the
+// rest), takes each .box into a boxes[] array, merges the Mac-dropped sources (mac.json, pcs.json in /srv/workers/sources,
+// written by the Mac's push.mjs over ssh), and writes /srv/workers/workers.json, which build.igneum.network serves. Build-1's
+// own box comes from a sources-free local collect into /tmp so it is never circular. Node 22, no dependencies.
+import { readFileSync, writeFileSync, renameSync, mkdirSync } from 'node:fs';
+import { execFileSync } from 'node:child_process';
+const OUT = '/srv/workers/workers.json';
+const HOSTS = [['igneum-build-1','https://build.igneum.network'],['igneum-build-2','https://build-2.igneum.network'],['igneum-build-3','https://build-3.igneum.network'],['igneum-build-4','https://build-4.igneum.network'],['igneum-build-5','https://build-5.igneum.network'],['igneum-build-6','https://build-6.igneum.network'],['igneum-build-7','https://build-7.igneum.network'],['igneum-build-8','https://build-8.igneum.network'],['igneum-build-9','https://build-9.igneum.network']];
+const now = Date.now(), iso = t => new Date(t).toISOString();
+async function fetchBox(name, base) {
+  if (name === 'igneum-build-1') { // our own box, sources-free local collect (never read our own merged output)
+    try { mkdirSync('/tmp/wself', { recursive: true }); execFileSync('/usr/local/bin/node', ['/srv/workers/bin/collect.mjs'], { env: { ...process.env, IGNEUM_WORKERS_DIR: '/tmp/wself' }, timeout: 20000 }); const d = JSON.parse(readFileSync('/tmp/wself/workers.json', 'utf8')); return { ...(d.box || {}), name, source: { ok: true, at: iso(now) } }; } catch (e) { return { name, down: true, reason: String(e.message || e).slice(0, 80) }; }
+  }
+  try { const r = await fetch(`${base}/workers.json?t=${now}`, { signal: AbortSignal.timeout(8000) }); if (!r.ok) throw new Error('http ' + r.status); const d = await r.json(); const b = d.box || {}; const age = d.collected_at ? (now - Date.parse(d.collected_at)) / 1000 : null; return { ...b, name, source: { ok: age === null || age < 120, at: d.collected_at || null, age_s: age } }; }
+  catch (e) { const reason = String(e.message || e); return { name, down: true, reason: (name === 'igneum-build-3' ? 'down since 16:40 BST (Hetzner switch fault hel1-dc12-sw_99)' : reason).slice(0, 90) }; }
+}
+function source(name) { try { const d = JSON.parse(readFileSync(`/srv/workers/sources/${name}.json`, 'utf8')); return d; } catch { return null; } }
+const boxes = await Promise.all(HOSTS.map(([n, b]) => fetchBox(n, b)));
+const mac = source('mac'), pcs = source('pcs');
+// the Mac mini (igneum-mini, the Mac build box, M6): no inbound route from the boxes, so its own launchd collector pushes mini.json to build-1 (8 Oct 2026). A row older than 120 s reads down.
+const miniSrc = source('mini'); let mini = null; if (miniSrc) { const d = miniSrc.data ? miniSrc.data : miniSrc; const age = d.collected_at ? (now - Date.parse(d.collected_at)) / 1000 : null; mini = { ...(d.box || d), name: 'igneum-mini', label: 'the Mac build box, M6', source: { ok: age === null || age < 120, at: d.collected_at || null, age_s: age } }; }
+const doc = { v: 3, generated_at: iso(now), generated_by: 'merge-workers.mjs on igneum-build-1', boxes, box: boxes[0], mini, mac: mac && mac.data ? mac.data : mac, pcs: pcs && pcs.data ? pcs.data : pcs, sources: { merge: { ok: true, at: iso(now) }, mac: mac ? { ok: true } : { ok: false }, pcs: pcs ? { ok: true } : { ok: false } } };
+const tmp = OUT + '.tmp'; writeFileSync(tmp, JSON.stringify(doc)); renameSync(tmp, OUT);
+process.stdout.write(`merged ${boxes.length} boxes (${boxes.filter(b => !b.down).length} up), mac ${!!doc.mac}, pcs ${!!doc.pcs} at ${iso(now)}\n`);

tools/workers/push.mjs

@@ -32,8 +32,20 @@ const iso = ms => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z');
 const log = (...a) => console.error(`${iso(Date.now())} push: ${a.join(' ')}`);
 const KEY = process.env.IGNEUM_BUILD_KEY || join(homedir(), '.ssh', 'igneum_ed25519');
 // every build box: ~/.config/igneum/build-server (build-1), build-server-2, build-server-3 ... one line build@<ip> each
-const HOSTS = [['build-server', 'igneum-build-1'], ['build-server-2', 'igneum-build-2'], ['build-server-3', 'igneum-build-3']]
+// every build box is a host file in ~/.config/igneum: build-server (build-1), build-server-2, build-server-3 ... build-server-N;
-  .map(([f, n]) => ({ file: f, name: n, host: (cfg(f) || '').split('\n')[0].trim() })).filter(h => h.host);
+// enumerate them so a box joins the page the moment its host file exists, no edit here (8 Oct 2026: build-4 to 9 were missing from
+// the hardcoded three and read as down).
+const CFGDIR = join(homedir(), '.config', 'igneum');
+const HOSTS = (() => {
+  const seen = new Set(); const out = [];
+  let extra = []; try { extra = readdirSync(CFGDIR).filter(x => /^build-server-\d+$/.test(x)).sort((a, b) => (+a.split('-').pop()) - (+b.split('-').pop())); } catch {}
+  for (const f of ['build-server', ...extra]) {
+    const host = (cfg(f) || '').split('\n')[0].trim(); if (!host || seen.has(host)) continue; seen.add(host);
+    const n = f === 'build-server' ? 1 : +f.split('-').pop();
+    out.push({ file: f, name: `igneum-build-${n}`, host });
+  }
+  return out;
+})();
 const HOST = HOSTS.length ? HOSTS[0].host : '';
 const DLSITE = cfg('dlsite-dir'), FLEET = cfg('fleet-path');
 const STALE_LIVE_S = 360;