From 6f5787ed71953b9a9c89dd41b0bd874838aeeb4a Mon Sep 17 00:00:00 2001
From: igneum-labs <337424239+igneum-labs@users.noreply.github.com>
Date: Wed, 7 Oct 2026 13:30:40 +0000
Subject: [PATCH 1/2] Build boxes: the measure file is retired; a pinned
 measurement leases its cores only (lease.sh cores), a whole-box quiet
 measurement is its own class (refused beside a slot or a lease, 20-minute
 cap, named owner), bounded suites never take it, every run keeps off leased
 cores, stopped holders are reaped after 5 minutes by every keeper, every
 waiter has a label file; the box-side self-tests in the gate; provision
 installs the lease tool and the headless Chromium libraries

Main, 7 October 2026, 15:07 UK: one global exclusive measure flock across unrelated measurements stalled build-1 at load 120 with
free slots (a stopped probe held it 5.5 h; an exclusive waiter queued every new shared taker) and build-2 behind a one-core VDF
bench. lease.sh (installed at /srv/builds/_bin/lease by provision.sh and by hand on both boxes) takes one flock per core for a
pinned measurement and the quiet file for a whole-box one; remote-run.sh takes quiet shared only for unbounded runs, excludes
leased cores from its set, and its keeper refreshes the holder file and calls lease reap (a STOPPED holder of a lease, quiet or a
slot for 5 minutes is killed with a line in _log/reaped.log). Keepers close the lock descriptors they inherit (an orphaned sleep
held a slot and the worktree lock 20 s past the release; the slot self-test had rotted on that since the worktree lock landed).
tools/ci/box-locks-check.sh runs lease.sh --self-test and remote-run.sh --self-test-slots on build-1. provision.sh also carries
the 16 libraries headless Chromium needs (installed by hand on both boxes at 14:5x UK) and a headless self-test step.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 docs/plans/build-server.md         |  25 +++++
 infra/build-server/capacity/lib.sh |   8 +-
 infra/build-server/lease.sh        | 155 +++++++++++++++++++++++++++++
 infra/build-server/provision.sh    |  27 +++++
 infra/build-server/remote-run.sh   | 100 +++++++++++++------
 tools/ci/box-locks-check.sh        |  21 ++++
 tools/ci/pre-push.sh               |   1 +
 7 files changed, 303 insertions(+), 34 deletions(-)
 create mode 100755 infra/build-server/lease.sh
 create mode 100755 tools/ci/box-locks-check.sh

diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md
index 24c5ca04d..a7e907d56 100644
--- a/docs/plans/build-server.md
+++ b/docs/plans/build-server.md
@@ -377,3 +377,28 @@ spill-over. Now (lib.sh `bs_route_spill`, master from this commit):
   them keeps its number; since this commit a bounded run takes the band its slot owns (slot 0 the last 32 cores, slot 1 the 32
   below, slot 2 the 32 below that), so three bounded runs never share a core. A gate still takes its slot ahead of queued suites.
 - `--box N` still pins. Self-test: tools/ci/route-spill-check.sh (thirteen cases through `BS_ROUTE_STATE_<n>`, no ssh), in the gate.
+
+## 8. The measure file is retired: per-core leases and the quiet class (7 October 2026, 15:07 UK)
+
+Main's reading at 15:07 UK: on build-1 a sync-fuzz probe (the capacity lane's, SIGSTOPped since 09:45Z, no owner) held the global
+measure flock for five and a half hours; beside it attack-f6's phase2b waited exclusive on the same file behind seven F2 solvers
+holding it shared on cores 6-11,54-59, and every new shared taker (every build) queued behind the exclusive waiter: load 120, slots
+free, nine waiting. On build-2 the era VDF bench held the file exclusive while pinned to one core and five jobs waited. One global
+exclusive lock across unrelated measurements was the wrong design. Now:
+
+- `infra/build-server/lease.sh`, installed on every box at `/srv/builds/_bin/lease` (provision.sh; by hand on build-1 and build-2
+  at 14:2x BST): `lease cores <set> --label "..." [--owner <agent>] -- <cmd>` takes a lease on THOSE CORES ONLY (one flock per
+  core, `_locks/core-<n>`, a `wait-<pid>` file with the label while it waits), runs the command under nice 10 and taskset, and
+  releases. Nothing else is excluded. `lease quiet --label "..." --owner <agent> -- <cmd>` is the whole-box class: refused (exit
+  73) while any slot or core lease is held, capped at 20 minutes, holder line with the owner in `_locks/quiet`. `lease status`,
+  `lease reap`.
+- remote-run.sh: an unbounded run (nice 0, the full set) takes `quiet` shared and waits for it; a bounded run (suites, benches,
+  everything on box 2) never takes it. Every run keeps off leased cores (its set minus the `core-<n>` flocks, said once). A run's
+  keeper refreshes its holder file's mtime every 20 s and calls `lease reap`: a holder of a lease, the quiet file or a slot whose
+  process has been STOPPED for 5 minutes is killed and its file cleared, one line each in `_log/reaped.log`. The keeper closes the
+  lock descriptors it inherits (an orphaned `sleep 20` held a slot and a worktree lock 20 s past the release). BR_MEASURE=1 is the
+  quiet class with the same refusals; it needs a named owner (IGNEUM_AGENT).
+- The lanes' own `flock -s /srv/builds/_locks/measure -c "nice -n 10 taskset -c <set> ..."` lines no longer hold anything a build
+  waits for; they become `/srv/builds/_bin/lease cores <set> --label "..." -- <cmd>`, and `flock -x .../measure` becomes
+  `lease quiet`. Self-tests: lease.sh --self-test and remote-run.sh --self-test-slots, run on build-1 by tools/ci/box-locks-check.sh
+  in the gate.
diff --git a/infra/build-server/capacity/lib.sh b/infra/build-server/capacity/lib.sh
index 7418d4b2c..a7d36d3ee 100755
--- a/infra/build-server/capacity/lib.sh
+++ b/infra/build-server/capacity/lib.sh
@@ -37,11 +37,12 @@ CAP_NODE_BRANCH="${CAP_NODE_BRANCH:-}"; CAP_REPO_SHA="${CAP_REPO_SHA:-}"; CAP_FO
 
 # ---- the build-slot and measure hold the layer yields to --------------------------------------------------------------
 # a slot or the measure file is "held" when flock -n cannot take it (another process holds the fd). The same probe the
-# collector uses (tools/workers/collect.mjs flockHeld). Returns 0 (true) when ANY build slot or the measure hold is taken.
+# collector uses (tools/workers/collect.mjs flockHeld). Returns 0 (true) when ANY build slot, the quiet hold or a core lease is taken.
 cap_build_active() {
   local slots k f
   slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1
-  if [ -e "$LOCKS/measure" ] && ! flock -n "$LOCKS/measure" true 2>/dev/null; then return 0; fi
+  if [ -e "$LOCKS/quiet" ] && ! flock -n "$LOCKS/quiet" true 2>/dev/null; then return 0; fi   # 7 Oct 2026: the quiet class replaced the measure file
+  for f in "$LOCKS"/core-*; do [ -e "$f" ] && ! flock -n "$f" true 2>/dev/null && return 0; done
   for k in $(seq 0 $((slots - 1))); do
     f="$LOCKS/build-$k"
     [ -e "$f" ] || continue
@@ -51,7 +52,8 @@ cap_build_active() {
 }
 cap_hold_reason() {
   local slots k
-  if [ -e "$LOCKS/measure" ] && ! flock -n "$LOCKS/measure" true 2>/dev/null; then echo "measure hold"; return; fi
+  if [ -e "$LOCKS/quiet" ] && ! flock -n "$LOCKS/quiet" true 2>/dev/null; then echo "quiet hold"; return; fi
+  for f in "$LOCKS"/core-*; do [ -e "$f" ] && ! flock -n "$f" true 2>/dev/null && { echo "core lease"; return; }; done
   slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1
   for k in $(seq 0 $((slots - 1))); do
     [ -e "$LOCKS/build-$k" ] || continue
diff --git a/infra/build-server/lease.sh b/infra/build-server/lease.sh
new file mode 100755
index 000000000..09baf3551
--- /dev/null
+++ b/infra/build-server/lease.sh
@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+# Measurement leases on a build box (main, 7 October 2026, 15:07 UK: one global exclusive "measure" flock across unrelated
+# measurements stalled build-1 at load 120 with free slots, an exclusive waiter queueing every new shared taker behind it, and
+# a stopped probe held the box for five and a half hours). The measure file is retired. In its place:
+#
+#   lease cores <set> --label "<text>" [--owner <agent>] [--nice N] -- <command...>
+#       A measurement that pins cores takes a lease on THOSE CORES ONLY (one flock per core, _locks/core-<n>, taken in ascending
+#       order, waited for up to 2 h with a wait-<pid> file carrying the label), runs the command under nice N (default 10) and
+#       taskset on the set, and releases. Builds and suites keep off leased cores (remote-run.sh reads the core files before it
+#       pins its own set). Nothing else is excluded: the box stays open.
+#   lease quiet --label "<text>" --owner <agent> [--cap-s N] -- <command...>
+#       A WHOLE-BOX quiet measurement: its own class, refused (exit 73) while any build slot or any core lease is held, capped at
+#       20 minutes (timeout; --cap-s at most 1200), holder line with the owner in _locks/quiet. Unbounded builds take quiet
+#       shared and wait for it; bounded suites (nice 10, a 32-core band) never take it.
+#   lease status           every lease, the quiet holder and every waiter with its label
+#   lease reap             a holder (lease, quiet or build slot) whose process has been STOPPED (state T) for 5 minutes or more is
+#                          killed and its file cleared, one line each in _log/reaped.log; remote-run.sh's keeper calls this every
+#                          20 s while any run is on the box (LEASE_REAP_S overrides the 300 s for the self-test)
+#   lease --self-test      the known cases against a scratch lock directory (in the gate: tools/ci/pre-push.sh)
+#
+# Installed on every box at /srv/builds/_bin/lease by provision.sh (and copied by hand on 7 October 2026); the lock directory is
+# IGNEUM_BUILD_SLOTS_DIR (the profile sets /srv/builds/_locks), the log directory IGNEUM_BUILD_LOG_DIR. Files: core-<n> (flock),
+# lease-<pid> (holder line "pid N since HH:MM:SSZ cores <set>: <label>; owner=<agent>"), quiet (holder line), wait-<pid>.
+set -uo pipefail
+LOCKS="${IGNEUM_BUILD_SLOTS_DIR:-/srv/builds/_locks}"; LOGS="${IGNEUM_BUILD_LOG_DIR:-/srv/builds/_log}"
+REAP_S="${LEASE_REAP_S:-300}"
+now() { date -u +%H:%M:%SZ; }
+say() { echo "lease: $*" >&2; }
+expand_set() {   # "6-11,54-59" -> "6 7 8 9 10 11 54 ..."
+  local out="" part lo hi; IFS=',' read -ra parts <<<"$1"
+  for part in "${parts[@]}"; do case "$part" in *-*) lo=${part%-*}; hi=${part#*-} ;; *) lo=$part; hi=$part ;; esac
+    [ "$lo" -le "$hi" ] 2>/dev/null || { say "bad core set '$1'"; return 2; }
+    for ((c = lo; c <= hi; c++)); do out="$out $c"; done; done
+  echo "${out# }"
+}
+held_cores() {   # the cores whose lease file is flocked right now, as a space list
+  local f c out=""
+  for f in "$LOCKS"/core-*; do [ -e "$f" ] || continue; c=${f##*/core-}
+    exec 8>>"$f"; if ! flock -n 8; then out="$out $c"; fi; exec 8>&-; done
+  echo "${out# }"
+}
+any_slot_held() { local n f k; n=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); for ((k = 0; k < n; k++)); do f="$LOCKS/build-$k"; [ -e "$f" ] || continue; exec 8>>"$f"; if ! flock -n 8; then exec 8>&-; return 0; fi; exec 8>&-; done; return 1; }
+pid_of() { sed -n 's/^pid \([0-9]*\) .*/\1/p' "$1" 2>/dev/null | head -1; }
+reap() {   # stopped holders older than REAP_S are killed, their files cleared, one line each in reaped.log
+  local f p st age line n=0
+  for f in "$LOCKS"/lease-* "$LOCKS"/quiet "$LOCKS"/build-[0-9]*; do
+    [ -s "$f" ] || continue; p=$(pid_of "$f"); [ -n "$p" ] || continue
+    st=$(ps -o stat= -p "$p" 2>/dev/null | tr -d ' '); case "$st" in T*) ;; *) continue ;; esac
+    age=$(( $(date +%s) - $(stat -c %Y "$f") ))
+    # the file's mtime is refreshed by the holder's keeper every 20 s while it runs; a stopped holder's file goes stale
+    [ "$age" -ge "$REAP_S" ] || continue
+    line="$(now) reaped pid $p (stopped $age s, state $st) holding $(basename "$f"): $(head -c 160 "$f" | tr '\n' ' ')"
+    kill -KILL "$p" 2>/dev/null; pkill -KILL -P "$p" 2>/dev/null; : > "$f"; case "$f" in */lease-*) rm -f "$f" ;; esac
+    mkdir -p "$LOGS"; echo "$line" >> "$LOGS/reaped.log"; say "$line"; n=$((n + 1))
+  done
+  echo "$n"
+}
+status() {
+  echo "leases:"; for f in "$LOCKS"/lease-*; do [ -s "$f" ] && echo "  $(cat "$f")"; done 2>/dev/null
+  echo "quiet: $(cat "$LOCKS/quiet" 2>/dev/null)"
+  echo "held cores: $(held_cores)"
+  echo "waiting:"; for f in "$LOCKS"/wait-*; do [ -s "$f" ] && echo "  $(cat "$f")"; done 2>/dev/null
+}
+run_cores() {
+  local set="$1"; shift; local label="" owner="${IGNEUM_AGENT:-unknown}" nice=10
+  while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --nice) nice="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
+  [ -n "$label" ] || { say "--label is required (the dashboard and the next lane read it)"; exit 2; }
+  [ $# -gt 0 ] || { say "no command"; exit 2; }
+  local cores c fd t0 waited=0 line waitfile="$LOCKS/wait-$$"
+  cores=$(expand_set "$set") || exit 2
+  mkdir -p "$LOCKS"; t0=$(date +%s)
+  line="pid $$ since $(now) waited 0 s: lease cores $set: $label; owner=$owner"; printf '%s\n' "$line" > "$waitfile"
+  trap 'rm -f "$waitfile" "$LOCKS/lease-$$"' EXIT
+  local fds=()
+  for c in $cores; do
+    exec {fd}>>"$LOCKS/core-$c"
+    if ! flock -n "$fd"; then say "core $c is leased ($(for f in "$LOCKS"/lease-*; do grep -l " core-$c\b\| cores [^:]*\b$c\b" "$f" 2>/dev/null; done | head -1 | xargs -r cat | cut -c1-120)); waiting"; flock -w 7200 "$fd" || { say "gave up waiting for core $c after 2 h"; exit 75; }; fi
+    fds+=("$fd")
+  done
+  waited=$(( $(date +%s) - t0 ))
+  line="pid $$ since $(now) waited $waited s: lease cores $set: $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/lease-$$"; rm -f "$waitfile"
+  say "holding cores $set (waited $waited s): $label"
+  # the keeper closes the lock descriptors first: an inherited flock would outlive the release in its orphaned sleep
+  ( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$!
+  nice -n "$nice" taskset -c "$set" "$@"; local rc=$?
+  pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
+  rm -f "$LOCKS/lease-$$"; say "released cores $set after $(( $(date +%s) - t0 )) s, exit $rc"
+  exit $rc
+}
+run_quiet() {
+  local label="" owner="${IGNEUM_AGENT:-}" cap=1200
+  while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --cap-s) cap="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
+  [ -n "$label" ] && [ -n "$owner" ] || { say "quiet needs --label and --owner (a named owner, the 5.5-hour rule)"; exit 2; }
+  [ "$cap" -le 1200 ] 2>/dev/null || cap=1200
+  [ $# -gt 0 ] || { say "no command"; exit 2; }
+  mkdir -p "$LOCKS"
+  if any_slot_held; then say "REFUSED: a build slot is held; a whole-box quiet measurement waits for an idle box (lease status)"; exit 73; fi
+  if [ -n "$(held_cores)" ]; then say "REFUSED: cores $(held_cores | tr ' ' ',') are leased; a whole-box quiet measurement waits for an idle box"; exit 73; fi
+  exec 7>>"$LOCKS/quiet"
+  if ! flock -n 7; then say "REFUSED: another quiet measurement holds the box: $(head -c 160 "$LOCKS/quiet")"; exit 73; fi
+  local t0 line; t0=$(date +%s)
+  line="pid $$ since $(now) waited 0 s: quiet (cap $cap s): $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/quiet"
+  trap ': > "$LOCKS/quiet"' EXIT
+  ( exec 7>&-; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/quiet" 2>/dev/null; [ -s "$LOCKS/quiet" ] || printf '%s\n' "$line" > "$LOCKS/quiet"; sleep 20; done ) & local keeper=$!
+  say "holding the box quiet (cap $cap s): $label"
+  timeout --signal TERM --kill-after 30 "$cap" "$@"; local rc=$?
+  pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
+  : > "$LOCKS/quiet"; [ "$rc" = 124 ] && say "the quiet measurement hit its ${cap}s cap and was ended"
+  say "released the box after $(( $(date +%s) - t0 )) s, exit $rc"; exit $rc
+}
+self_test() {
+  t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
+  export IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" LEASE_REAP_S=2; mkdir -p "$t/locks"; echo 2 > "$t/locks/slots"
+  local me="$0" fail=0
+  f() { echo "lease self-test: FAIL: $*"; fail=1; }
+  # 1. two leases on disjoint cores run together; the same core waits
+  bash "$me" cores 0-1 --label A -- bash -c "date +%s.%N > '$t/a.start'; sleep 2; date +%s.%N > '$t/a.end'" 2>/dev/null &
+  sleep 0.5; bash "$me" cores 2-3 --label B -- bash -c "date +%s.%N > '$t/b.start'" 2>/dev/null; sleep 0.2
+  python3 -c "import sys; sys.exit(0 if float(open('$t/b.start').read()) < float(open('$t/a.start').read()) + 1.5 else 1)" || f "a lease on other cores waited for an unrelated lease"
+  bash "$me" cores 1-2 --label C -- bash -c "date +%s.%N > '$t/c.start'" 2>/dev/null
+  python3 -c "import sys; sys.exit(0 if float(open('$t/c.start').read()) >= float(open('$t/a.end').read()) else 1)" || f "a lease sharing a core started before the holder released it"
+  wait
+  [ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "lease or wait files left behind: $(ls "$t/locks")"
+  # 2. quiet is refused while a slot is held, and runs on an idle box with a holder line naming the owner
+  ( exec 9>>"$t/locks/build-0"; flock 9; echo "pid $BASHPID since x waited 0 s: fake build" > "$t/locks/build-0"; sleep 2 ) &
+  sleep 0.3; bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a slot was held"
+  wait; : > "$t/locks/build-0"
+  bash "$me" quiet --label Q --owner tester -- bash -c "grep -q 'owner=tester' '$t/locks/quiet'" || f "quiet did not run on an idle box with the owner in its holder line"
+  bash "$me" quiet --label Q --owner tester --cap-s 1 -- sleep 5; [ $? = 124 ] || f "the quiet cap did not end the command"
+  # 3. a quiet is refused while a core is leased
+  bash "$me" cores 0 --label L -- sleep 2 2>/dev/null & sleep 0.5
+  bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a core was leased"
+  wait
+  # 4. reap: a stopped holder older than the window is killed and its file cleared, with a line
+  bash "$me" cores 5 --label S -- sleep 60 2>/dev/null & local lp=$!; sleep 0.6
+  local hp; hp=$(sed -n 's/^pid \([0-9]*\) .*/\1/p' "$t"/locks/lease-* | head -1); kill -STOP "$hp"; sleep 2.5
+  touch -d '-10 seconds' "$t"/locks/lease-* 2>/dev/null
+  [ "$(bash "$me" reap 2>/dev/null)" = 1 ] || f "the stopped lease holder was not reaped"
+  grep -q "reaped pid $hp" "$t/log/reaped.log" 2>/dev/null || f "no reaped line was written"
+  [ -z "$(ls "$t/locks" | grep '^lease-')" ] || f "the reaped lease file remains"
+  kill -KILL "$lp" 2>/dev/null; wait 2>/dev/null
+  # 5. a running (not stopped) holder is left alone
+  bash "$me" cores 6 --label R -- sleep 3 2>/dev/null & sleep 0.6; touch -d '-10 seconds' "$t"/locks/lease-*
+  [ "$(bash "$me" reap 2>/dev/null)" = 0 ] || f "a running holder was reaped"; wait
+  [ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept"
+  return $fail
+}
+case "${1:-}" in
+  cores) shift; run_cores "$@" ;;
+  quiet) shift; run_quiet "$@" ;;
+  status) status ;;
+  reap) reap ;;
+  --self-test) self_test ;;
+  *) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
+esac
diff --git a/infra/build-server/provision.sh b/infra/build-server/provision.sh
index 5ec888592..f2447f7df 100755
--- a/infra/build-server/provision.sh
+++ b/infra/build-server/provision.sh
@@ -146,6 +146,10 @@ APT_PACKAGES=(
   libicu74 python3-numpy
   # innoextract: tools/repro reads the shipped igneumd.exe and igneum-miner.exe out of the public Inno Setup installer
   innoextract
+  # headless Chromium (the dashboard lane's site captures): the 16 system libraries it dlopens, found missing on both boxes
+  # on 7 October 2026 (installed by hand at 14:5x UK; step_headless_check launches the shell once and prints its version)
+  libatk1.0-0t64 libatk-bridge2.0-0t64 libatspi2.0-0t64 libcairo2 libcups2t64 libgbm1 libpango-1.0-0 libx11-6 libxcb1
+  libxcomposite1 libxdamage1 libxext6 libxfixes3 libxrandr2 libasound2t64 libxkbcommon0 fonts-liberation
 )
 step_apt() {
   local need=() p
@@ -239,6 +243,27 @@ step_dirs() {
   done
   [ "$any" = 1 ] && changed dirs "/srv/builds ($(worktree_count) worktree dirs), /srv/sccache, slots=$SLOTS" || ok dirs "$(worktree_count) worktree dirs, slots=$SLOTS"
 }
+# the lease tool (infra/build-server/lease.sh: per-core measurement leases, the quiet class, the reaper; 7 October 2026) from the
+# mirror's master at /srv/builds/_bin/lease, which remote-run.sh's keeper calls; the mirror is pushed by step_mirrors' caller
+step_lease_tool() {
+  local src="/srv/igneum.git" want have=""
+  install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" /srv/builds/_bin
+  want=$(git -C "$src" show master:infra/build-server/lease.sh 2>/dev/null) || { ok lease-tool "mirror has no master yet; run-from-mac.sh installs it on the next provision"; return; }
+  [ -f /srv/builds/_bin/lease ] && have=$(cat /srv/builds/_bin/lease)
+  if [ "$want" = "$have" ]; then ok lease-tool "/srv/builds/_bin/lease is the mirror's master copy"; return; fi
+  printf '%s\n' "$want" > /srv/builds/_bin/lease.new; chmod 755 /srv/builds/_bin/lease.new; chown "$BUILD_USER:$BUILD_USER" /srv/builds/_bin/lease.new
+  mv /srv/builds/_bin/lease.new /srv/builds/_bin/lease; changed lease-tool "/srv/builds/_bin/lease installed from the mirror's master"
+}
+# headless Chromium self-test: the shell the dashboard lane's node tooling downloads (playwright or puppeteer cache of the build
+# user) launches once with --headless and prints its version; when no shell is downloaded yet the libraries are checked by ldd of
+# nothing, so the step only says so (the apt list above carries them)
+step_headless_check() {
+  local shell="" v
+  shell=$(find "/home/$BUILD_USER/.cache/ms-playwright" "/home/$BUILD_USER/.cache/puppeteer" /srv -maxdepth 6 -type f \( -name headless_shell -o -name chrome-headless-shell -o -name chrome \) 2>/dev/null | head -1)
+  [ -n "$shell" ] || { ok headless "no headless shell downloaded yet (the 16 libraries are installed; the lane's first capture downloads it)"; return; }
+  if v=$(sudo -u "$BUILD_USER" timeout 60 "$shell" --headless --no-sandbox --disable-gpu --version 2>&1 | head -1) && [ -n "$v" ]; then ok headless "$shell: $v"
+  else die "headless shell $shell does not launch: $v"; fi
+}
 
 step_mirrors() {
   local r any=0
@@ -634,6 +659,7 @@ do_provision() {
   step_docker
   step_dirs
   step_mirrors
+  step_lease_tool
   step_rustup
   step_sccache
   step_cargo_config
@@ -647,6 +673,7 @@ do_provision() {
   step_cargo_tools
   step_zig
   step_night
+  step_headless_check
   step_summary
   log "done"
 }
diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh
index 40ef449e6..955f00574 100755
--- a/infra/build-server/remote-run.sh
+++ b/infra/build-server/remote-run.sh
@@ -181,8 +181,8 @@ if [ "${1:-}" = --self-test-slots ]; then
   mkdir -p "$t/locks" "$t/log" "$t/dir"; echo 2 > "$t/locks/slots"
   fake() {   # <name> <seconds> [BR_MEASURE=1]: a fake run that records its start and end epoch and the job count it was given
     local name="$1" secs="$2" measure="${3:-0}"
-    IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" BR_MEASURE="$measure" BR_DIR="$t/dir" BR_CMD="date +%s.%N > '$t/$name.start'; echo JOBS=\${CARGO_BUILD_JOBS:-none} > '$t/$name.jobs'; sleep $secs; date +%s.%N > '$t/$name.end'" \
-      BR_LABEL="self-test $name" BR_TOOL=self-test BR_KIND=other BR_WT=t BR_CRATE=t BR_BRANCH=t BR_SHA=0 BR_AGENT=self-test BR_COMMAND="fake $name" \
+    IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" BR_MEASURE="$measure" BR_CORES="${BR_CORES:-0}" BR_NICE="${BR_NICE:-0}" BR_DIR="$t/dir" BR_CMD="date +%s.%N > '$t/$name.start'; echo JOBS=\${CARGO_BUILD_JOBS:-none} > '$t/$name.jobs'; sleep $secs; date +%s.%N > '$t/$name.end'" \
+      BR_LABEL="self-test $name" BR_TOOL=self-test BR_KIND=other BR_WT="wt-$name" BR_CRATE=t BR_BRANCH=t BR_SHA=0 BR_AGENT=self-test BR_COMMAND="fake $name" \
       bash "$me" >"$t/$name.out" 2>&1
   }
   fail() { echo "self-test-slots: FAIL: $*"; exit 1; }
@@ -193,33 +193,40 @@ if [ "${1:-}" = --self-test-slots ]; then
   # 2. one build alone: 90
   fake c 1
   [ "$(cat "$t/c.jobs")" = JOBS=90 ] || fail "a lone build got $(cat "$t/c.jobs") (want JOBS=90)"
-  # 3. a measure blocks a build: the build starts only after the measure ended
-  fake m 3 1 & sleep 0.5; fake d 1 & wait
-  after "$t/d.start" "$t/m.end" || fail "a build started while a measurement held the box (build start $(cat "$t/d.start"), measure end $(cat "$t/m.end"))"
+  # 3. a quiet measurement blocks an unbounded build (it starts only after the quiet ended) and lets a bounded suite run beside it
+  fake m 3 1 & sleep 0.5; fake d 1 & BR_CORES=1 BR_NICE=10 fake s 1 & wait
+  after "$t/d.start" "$t/m.end" || fail "an unbounded build started while a quiet measurement held the box (build start $(cat "$t/d.start"), quiet end $(cat "$t/m.end"))"
+  python3 -c "import sys; sys.exit(0 if float(open('$t/s.start').read()) < float(open('$t/m.end').read()) else 1)" || fail "a bounded suite waited for the quiet measurement"
   [ "$(cat "$t/m.jobs")" = JOBS=none ] || fail "a measurement was given a job count"
-  # 4. a build blocks a measure: the measure starts only after the build ended
-  fake e 3 & sleep 0.5; fake n 1 1 & wait
-  after "$t/n.start" "$t/e.end" || fail "a measurement started while a build ran (measure start $(cat "$t/n.start"), build end $(cat "$t/e.end"))"
+  grep -q 'owner=self-test' "$t/m.out" "$t/log/builds.jsonl" 2>/dev/null || fail "the quiet holder line lacks its owner"
+  # 4. a quiet measurement is REFUSED (exit 73) while a build holds a slot or a core is leased
+  fake e 3 & sleep 0.5; fake n 1 1; rc=$?; wait
+  [ "$rc" = 73 ] && [ ! -f "$t/n.start" ] || fail "a quiet measurement was not refused while a build ran (rc $rc)"
+  ( exec 9>>"$t/locks/core-7"; flock 9; sleep 2 ) & sleep 0.3; fake o 1 1; rc=$?; wait
+  [ "$rc" = 73 ] || fail "a quiet measurement was not refused while a core was leased (rc $rc)"
+  # 4b. a run keeps off leased cores: with core 1 leased, a 2-core bounded run on a 2-core box says so (the exclusion line)
+  ( exec 9>>"$t/locks/core-$(( $(nproc) - 1 ))"; flock 9; sleep 2 ) & sleep 0.3; BR_CORES=2 BR_NICE=10 fake p 1; wait
+  grep -q 'are leased to a measurement; this run keeps to' "$t/p.out" || fail "a run beside a leased core did not exclude it: $(cat "$t/p.out" | tail -3)"
   # 5. a probing build leaves a busy slot's holder line intact
   fake f 3 & sleep 1.2; fake g 1 & sleep 0.3
   grep -q 'self-test f' "$t/locks/build-0" || fail "the holder line of the busy slot build-0 was lost when another build probed it: '$(cat "$t/locks/build-0")'"
   wait
   # 6. the log carries the job count and the measure flag
   grep -q '"jobs":45' "$t/log/builds.jsonl" && grep -q '"measure":true' "$t/log/builds.jsonl" || fail "builds.jsonl lacks jobs or measure fields"
-  echo "self-test-slots: two concurrent builds 45 each, a lone build 90, a measure blocks a build, a build blocks a measure, a probe keeps the holder line, the log carries jobs and measure"; exit 0
+  echo "self-test-slots: two concurrent builds 45 each, a lone build 90, a quiet blocks an unbounded build and not a bounded suite, a quiet is refused beside a slot or a lease, a run keeps off leased cores, a probe keeps the holder line, the log carries jobs and measure"; exit 0
 fi
 
 # One run per worktree directory at a time (6 October 2026, 19:51:09 UK: two runs of one worktree started in the same second;
 # one found no crate directory while the other's checkout was replacing the tree, exit 2). The checkout and the run each take
 # the worktree's lock (append mode, held to exit) and wait up to 2 h for it instead of dying; the wait is said on stderr.
 wt_lock() {   # <worktree name>
-  local name="${1//\//_}" fd
+  local name="${1//\//_}"
   [ -n "$name" ] || return 0
   mkdir -p "$IGNEUM_BUILD_SLOTS_DIR" 2>/dev/null || return 0
-  exec {fd}>>"$IGNEUM_BUILD_SLOTS_DIR/wt-$name.lock" || return 0
-  if ! flock -n "$fd"; then
+  exec {WT_FD}>>"$IGNEUM_BUILD_SLOTS_DIR/wt-$name.lock" || return 0   # WT_FD is global: the keeper closes it (an orphaned sleep held it 20 s)
+  if ! flock -n "$WT_FD"; then
     echo "build-remote: another run holds worktree $1 on this box, waiting for it (up to 2 h)" >&2
-    flock -w 7200 "$fd" || { echo "build-remote: gave up waiting for worktree $1 after 2 h" >&2; exit 75; }
+    flock -w 7200 "$WT_FD" || { echo "build-remote: gave up waiting for worktree $1 after 2 h" >&2; exit 75; }
   fi
 }
 if [ "${BR_MODE:-run}" = checkout ]; then
@@ -318,25 +325,30 @@ give_up() {   # <what>
   rm -f "$waitfile"; exit 75
 }
 waitfile="$SLOTS_DIR/wait-$BR_PID"
+# The measure file is RETIRED (main, 7 October 2026, 15:07 UK: one global exclusive flock across unrelated measurements stalled
+# build-1 at load 120 with free slots). A measurement that pins cores takes a lease on those cores only (lease.sh, installed at
+# /srv/builds/_bin/lease), and this runner keeps its command off leased cores (see cores_str below). A WHOLE-BOX quiet measurement
+# (BR_MEASURE=1) is its own class: refused with exit 73 while any slot or core lease is held, capped at 20 minutes, holder line with
+# the owner in $SLOTS_DIR/quiet. An unbounded run (nice 0, the full core set) takes quiet shared and waits for it; a bounded run
+# (BR_CORES > 0: suites, benches, everything on box 2) never takes it.
+held_cores() { local f c out=""; for f in "$SLOTS_DIR"/core-*; do [ -e "$f" ] || continue; c=${f##*/core-}; exec {cfd}>>"$f"; if ! flock -n "$cfd"; then out="$out $c"; fi; exec {cfd}>&-; done; echo "${out# }"; }
+any_slot_held() { local k f; for ((k = 0; k < slots; k++)); do f="$SLOTS_DIR/build-$k"; [ -e "$f" ] || continue; exec {sfd}>>"$f"; if ! flock -n "$sfd"; then exec {sfd}>&-; return 0; fi; exec {sfd}>&-; done; return 1; }
 # append mode: opening a lock file must never truncate the holder line another run wrote into it
-exec {mfd}>>"$SLOTS_DIR/measure"
+exec {mfd}>>"$SLOTS_DIR/quiet"
 if [ "${BR_MEASURE:-0}" = 1 ]; then
-  # a measurement: the measure file exclusively; every running build holds it shared, so this waits for them and blocks new ones
-  if ! flock -n "$mfd"; then
-    echo "build-remote: measure waits for the running build(s) (up to 2 h): $(for f in "$SLOTS_DIR"/build-*; do head -c 120 "$f" 2>/dev/null; done | tr '\n' ' ')" >&2
-    holder_line 0 > "$waitfile"; trap 'rm -f "$waitfile"' EXIT
-    flock -w 7200 "$mfd" || give_up "the measure hold"
-    rm -f "$waitfile"; trap - EXIT
-  fi
+  if any_slot_held; then echo "build-remote: QUIET REFUSED: a build slot is held ($(for f in "$SLOTS_DIR"/build-*; do head -c 100 "$f" 2>/dev/null; done | tr '\n' ' ')); a whole-box measurement needs an idle box" >&2; BR_CLASS=quiet-refused jsonlog 73 0 0 "" "$(date +%s)" "" "" "" "" "" ""; exit 73; fi
+  lc=$(held_cores); if [ -n "$lc" ]; then echo "build-remote: QUIET REFUSED: cores $(echo $lc | tr ' ' ',') are leased; a whole-box measurement needs an idle box" >&2; BR_CLASS=quiet-refused jsonlog 73 0 0 "" "$(date +%s)" "" "" "" "" "" ""; exit 73; fi
+  if ! flock -n "$mfd"; then echo "build-remote: QUIET REFUSED: another quiet measurement holds the box: $(head -c 160 "$SLOTS_DIR/quiet")" >&2; BR_CLASS=quiet-refused jsonlog 73 0 0 "" "$(date +%s)" "" "" "" "" "" ""; exit 73; fi
+  [ -n "${BR_AGENT:-}" ] && [ "$BR_AGENT" != unknown ] || { echo "build-remote: QUIET REFUSED: a whole-box measurement needs a named owner (IGNEUM_AGENT)" >&2; exit 73; }
   waited=$(( $(date +%s) - BR_T0 )); got=measure
-  holder_line "$waited" > "$SLOTS_DIR/measure"
-  echo "build-remote: holding measure on $BR_HOST (waited $waited s; builds are excluded until this run ends)" >&2
+  BR_LABEL="quiet (cap 1200 s): $BR_LABEL; owner=$BR_AGENT"; holder_line "$waited" > "$SLOTS_DIR/quiet"
+  echo "build-remote: holding the box QUIET on $BR_HOST (waited $waited s; owner $BR_AGENT; cap 20 min; unbounded builds wait, bounded suites run beside it on their bands)" >&2
+  BR_CMD="timeout --signal TERM --kill-after 30 1200 bash -c $(printf '%q' "$BR_CMD")"
 else
-  # a build: the measure file shared (a running measurement blocks us), then one exclusive slot
-  if ! flock -s -n "$mfd"; then
-    echo "build-remote: a measurement holds the box, waiting (up to 2 h): $(head -c 160 "$SLOTS_DIR/measure" 2>/dev/null)" >&2
+  if [ "${BR_CORES:-0}" = 0 ] && ! flock -s -n "$mfd"; then
+    echo "build-remote: a quiet measurement holds the box, waiting (up to 20 min): $(head -c 160 "$SLOTS_DIR/quiet" 2>/dev/null)" >&2
     holder_line 0 > "$waitfile"; trap 'rm -f "$waitfile"' EXIT
-    flock -s -w 7200 "$mfd" || give_up "the measurement to end"
+    flock -s -w 1500 "$mfd" || give_up "the quiet measurement to end"
     rm -f "$waitfile"; trap - EXIT
   fi
   # scheduling (main, 7 Oct 2026): a gate announces itself (gate-pending-<pid>) and takes the next free slot; a suite, bench or
@@ -390,13 +402,16 @@ fi
 # because a run from a worktree without last night's append-mode fix still opens a busy sibling's slot file with `>` on every
 # probe). A keeper re-writes the holder line whenever it finds the file empty, every BR_KEEP_S seconds, until release_slot.
 keeper_pid=""
-keep_line() {   # <file> <waited>
+keep_line() {   # <file> <waited>; the keeper also refreshes the file's mtime (a stopped holder's file goes stale) and reaps stopped
+  # holders of any lease, quiet or slot after 5 minutes through lease.sh (/srv/builds/_bin/lease reap, one line each in reaped.log)
   local f="$1" w="$2"
-  ( while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; sleep "${BR_KEEP_S:-20}"; done ) &
+  ( exec {mfd}>&- {fd}>&- 2>/dev/null; [ -n "${WT_FD:-}" ] && exec {WT_FD}>&-; while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; touch "$f" 2>/dev/null
+      [ -x /srv/builds/_bin/lease ] && IGNEUM_BUILD_SLOTS_DIR="$SLOTS_DIR" IGNEUM_BUILD_LOG_DIR="$LOG_DIR" /srv/builds/_bin/lease reap >/dev/null 2>&1
+      sleep "${BR_KEEP_S:-20}"; done ) &
   keeper_pid=$!
 }
-if [ "$got" = measure ]; then keep_line "$SLOTS_DIR/measure" "$waited"; else keep_line "$SLOTS_DIR/build-$got" "$waited"; fi
-release_slot() { [ -n "$keeper_pid" ] && { kill "$keeper_pid" 2>/dev/null; wait "$keeper_pid" 2>/dev/null; keeper_pid=""; }; if [ "$got" = measure ]; then : > "$SLOTS_DIR/measure"; else : > "$SLOTS_DIR/build-$got"; fi; }
+if [ "$got" = measure ]; then keep_line "$SLOTS_DIR/quiet" "$waited"; else keep_line "$SLOTS_DIR/build-$got" "$waited"; fi
+release_slot() { [ -n "$keeper_pid" ] && { pkill -P "$keeper_pid" 2>/dev/null; kill "$keeper_pid" 2>/dev/null; wait "$keeper_pid" 2>/dev/null; keeper_pid=""; }; if [ "$got" = measure ]; then : > "$SLOTS_DIR/quiet"; else : > "$SLOTS_DIR/build-$got"; fi; }
 cd "$BR_DIR" || { BR_CLASS=no-dir jsonlog 2 "$got" "$waited" "" "$(date +%s)" "" "" "" "" "" ""; redlog 2 0 no-dir; release_slot; exit 2; }
 
 # PRE-FLIGHT for a cargo command (the instant-death class, 6 October 2026): the manifest parses and every -p package exists,
@@ -457,6 +472,29 @@ if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then
   lo=$((ncpu - BR_CORES * (band + 1))); [ "$lo" -ge 0 ] || lo=$((ncpu - BR_CORES))
   cores_str="$lo-$((lo + BR_CORES - 1))"
 fi
+# leased cores (lease.sh: a pinned measurement's core-<n> flocks) are taken out of this run's set; a set that would be empty keeps
+# its cores (the measurement is told by its own lease line); the exclusion is said once
+if [ "$got" != measure ]; then
+  leased=$(held_cores)
+  if [ -n "$leased" ]; then
+    kept=$(python3 -c '
+import sys
+def expand(s):
+    out=set()
+    for part in s.split(","):
+        a,_,b=part.partition("-"); a=int(a); b=int(b) if b else a; out.update(range(a,b+1))
+    return out
+mine=expand(sys.argv[1]); leased=set(int(x) for x in sys.argv[2].split()); keep=sorted(mine-leased)
+if not keep: print(sys.argv[1]); sys.exit()
+runs=[]; 
+for c in keep:
+    if runs and runs[-1][1]==c-1: runs[-1][1]=c
+    else: runs.append([c,c])
+print(",".join(f"{a}-{b}" if a!=b else f"{a}" for a,b in runs))' "$cores_str" "$leased")
+    [ "$kept" != "$cores_str" ] && echo "build-remote: cores $(echo $leased | tr ' ' ',') are leased to a measurement; this run keeps to $kept" >&2
+    cores_str="$kept"
+  fi
+fi
 ( [ "${BR_NICE:-0}" -gt 0 ] && renice -n "$BR_NICE" -p $BASHPID >/dev/null 2>&1; [ "$cores_str" != "0-$((ncpu - 1))" ] && taskset -cp "$cores_str" $BASHPID >/dev/null 2>&1; eval "$BR_CMD" ) > >(tee -a "$BR_RUN_LOG") 2> >(tee -a "$BR_RUN_LOG" >&2)
 rc=$?
 # the keeper stops BEFORE the bare `wait` (which flushes the two tees): a bare wait also waits for the keeper, and the keeper waits
diff --git a/tools/ci/box-locks-check.sh b/tools/ci/box-locks-check.sh
new file mode 100755
index 000000000..75eb34b42
--- /dev/null
+++ b/tools/ci/box-locks-check.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# The box-side lock self-tests, run on igneum-build-1 over ssh (flock, taskset and GNU timeout live there, not on the Mac; the Mac
+# gate carried no slot self-test since the worktree lock landed on 6 October 2026 and the slot test rotted unseen). Both scripts
+# are copied as they are in this tree (never the box's installed copies) into a scratch path and run against scratch lock
+# directories: infra/build-server/lease.sh --self-test (per-core leases, the quiet class, the reaper) and
+# infra/build-server/remote-run.sh --self-test-slots (slots, jobs, quiet beside builds and suites, leased cores excluded).
+#
+#   tools/ci/box-locks-check.sh         # exit 1 when either self-test fails or the box cannot be reached
+set -euo pipefail
+cd "$(dirname "$0")/../.."
+. infra/build-server/lib.sh
+bs_host 1
+stamp="ci-$$-$(date +%s)"
+scp -q "${BS_SSH_OPTS[@]}" infra/build-server/lease.sh "$BS_HOST:/tmp/lease-$stamp.sh"
+scp -q "${BS_SSH_OPTS[@]}" infra/build-server/remote-run.sh "$BS_HOST:/tmp/rr-$stamp.sh"
+rc=0
+bs_ssh "bash /tmp/lease-$stamp.sh --self-test 2>&1 | grep -E '^lease self-test'; exit \${PIPESTATUS[0]}" || rc=1
+bs_ssh "IGNEUM_REMOTE_RUN_UNDER_TEST=/tmp/rr-$stamp.sh bash /tmp/rr-$stamp.sh --self-test-slots 2>&1 | grep -E '^self-test-slots'; exit \${PIPESTATUS[0]}" || rc=1
+bs_ssh "rm -f /tmp/lease-$stamp.sh /tmp/rr-$stamp.sh" || true
+[ "$rc" = 0 ] && echo "box-locks: the lease tool and the slot runner pass their self-tests on $BS_HOST"
+exit $rc
diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh
index 016baca3b..d9f4e9c94 100755
--- a/tools/ci/pre-push.sh
+++ b/tools/ci/pre-push.sh
@@ -87,6 +87,7 @@ tree_checks() {
   run "long-running tools keep their body in one parsed block (the edited-while-running class)" bash -c 'bash tools/ci/whole-body-check.sh --self-test && bash tools/ci/whole-body-check.sh'
   run "build-remote without a priority flag bounds suites and benches (nice 10, 32 cores); a gate runs unbounded" bash tools/ci/build-kind-default-check.sh
   run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh
+  run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh)" bash tools/ci/box-locks-check.sh
   run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh'
   run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh'
   run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test

From e1c555568ff72ddd1b2b6c73a3a4bd0c6d8c53e5 Mon Sep 17 00:00:00 2001
From: igneum-labs <337424239+igneum-labs@users.noreply.github.com>
Date: Wed, 7 Oct 2026 13:46:42 +0000
Subject: [PATCH 2/2] CI and boxes: the simulators job runs on master and
 release-* pushes and pull requests into them only (a feature-branch code push
 runs the igneum-pow tests alone; tools/ci/sims-branch-check.sh); the box lock
 self-tests run in parallel in one ssh; build-2 and build-3 join the runner
 pool bounded to 32 cores; the 0.3.20 first-wave script (seeds, hands,
 RPC-filter lift; dry run by default)

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 .github/workflows/ci.yml         |   4 +-
 infra/build-server/provision.sh  |   1 +
 infra/build-server/wave1-0320.sh | 114 +++++++++++++++++++++++++++++++
 tools/ci/box-locks-check.sh      |   7 +-
 tools/ci/pre-push.sh             |   1 +
 tools/ci/sims-branch-check.sh    |  35 ++++++++++
 6 files changed, 158 insertions(+), 4 deletions(-)
 create mode 100755 infra/build-server/wave1-0320.sh
 create mode 100755 tools/ci/sims-branch-check.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4b3570203..755e799ee 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -77,7 +77,9 @@ jobs:
   sims:
     name: simulators, quick modes
     needs: changes
-    if: ${{ needs.changes.outputs.code == 'true' }}
+    # master and release-* pushes, and pull requests into them, only (main, 7 October 2026: every code push cost two box jobs and the
+    # queue read 22); a feature-branch code push runs the igneum-pow tests alone. tools/ci/sims-branch-check.sh holds this rule.
+    if: ${{ needs.changes.outputs.code == 'true' && ((github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))) || (github.event_name == 'pull_request' && (github.base_ref == 'master' || startsWith(github.base_ref, 'release-')))) }}
     runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
     steps:
       - uses: actions/checkout@v4
diff --git a/infra/build-server/provision.sh b/infra/build-server/provision.sh
index 4160f2c0b..7b098816b 100755
--- a/infra/build-server/provision.sh
+++ b/infra/build-server/provision.sh
@@ -74,6 +74,7 @@ RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1,ci-red}"     # added to the defau
                                                             # holds the red watcher's record file and poster. A second box: BOX_HOSTNAME=igneum-build-2
                                                             # RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 (register.sh --host)
 RUNNER_CPUS="${RUNNER_CPUS:-}"                              # AllowedCPUs for the runner's service when set (a second box is bounded like a suite: 32 cores, nice 10)
+case "$BOX_HOSTNAME" in *-2|*-3) [ -n "$RUNNER_CPUS" ] || RUNNER_CPUS="0-31" ;; esac   # build-2 and build-3 join the pool (label igneum-build-1) bounded to 32 cores at Nice 10 (main, 7 Oct 2026)
 RUNNER_JOBS="${RUNNER_JOBS:-48}"                            # cargo jobs for a CI job: half the box, the agents' builds keep the rest
 RUNNER_TOKEN="${RUNNER_TOKEN:-}"                            # a registration token (1 h), from infra/build-server/runner/register.sh over stdin; never logged
 RUNNER_SCCACHE_PORT="${RUNNER_SCCACHE_PORT:-4227}"          # the runner's own sccache server; 4226 is the build user's
diff --git a/infra/build-server/wave1-0320.sh b/infra/build-server/wave1-0320.sh
new file mode 100755
index 000000000..1683489bd
--- /dev/null
+++ b/infra/build-server/wave1-0320.sh
@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# The 0.3.20 FIRST wave (the shipper, 7 October 2026): the publish carries a floor-moved override file whose digest differs from
+# the live one, and a node on the old file refuses a node on the new one as a peer, so the three testnet seeds and the hands move
+# AT the publish minute, in parallel with the fleet's wave. Dry run by default (every line printed, no box touched); --go executes.
+#
+#   infra/build-server/wave1-0320.sh seeds --override <file> --igneumd <seed-class igneumd> --sha256 <hex> [--miner <seed-class igneum-miner>] \
+#                                          --digest <hex> [--commit c4459193] [--go]
+#       the three seeds in PARALLEL (seed1/2/3.testnet, root over the ops key, infra/seed-nodes/seeds-testnet.tsv): the binary put
+#       as /opt/igneum/bin/igneumd.new with its sha256 asserted on the box, the override written to /etc/igneum/override-params.json,
+#       EXTRA_ARGS in /etc/igneum/seed.env set to --override-params-file=<that>, the unit igneumd stopped, the binary swapped (the old
+#       one kept as igneumd.prev), the unit started on its kept datadir, then the read-back: commit string in the installed binary,
+#       the "Consensus params digest" line of the new run against --digest, eth_syncing over the loopback EVM RPC, peers; one line
+#       per seed, logs in the scratch directory
+#   infra/build-server/wave1-0320.sh hands --node <fork worktree on the Mac> --override-json '<object>' --digest <hex> [--go]
+#       the hands through infra/build-server/hands/move-hand.sh: binary (build + install + override), restart observer-node, restart
+#       node1, each read back by that tool (first executing line, commit string, digest, powEngine)
+#   infra/build-server/wave1-0320.sh lift-rpc-filter [--go]
+#       on seed1 only, AFTER the three read-backs: BLOCKED_UNTIL_FIXED_NODE in /opt/igneum/bin/rpc-filter.py becomes the empty set
+#       (the N7 class is fixed from 8097d600; backup kept as rpc-filter.py.bak-<stamp>), python3 -m py_compile, the unit
+#       igneum-rpc-filter restarted, read back: unit active and eth_blockNumber answered through https://rpc.testnet.igneum.network
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd); ROOT=$(cd "$HERE/../.." && pwd)
+SEEDS_TSV="$ROOT/infra/seed-nodes/seeds-testnet.tsv"; KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
+S="${WAVE_LOG_DIR:-/private/tmp/claude-501/-Users-joshm/cd75457f-4858-4f86-9634-7481ee056b7b/scratchpad/wave1-0320}"; mkdir -p "$S"
+SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15)
+now() { TZ=Europe/London date '+%H:%M:%S %Z'; }
+say() { echo "$(now) wave1: $*" >&2; }
+die() { say "ERROR: $*"; exit 1; }
+seed_ip() { awk -F'\t' -v n="$1" '$1==n {print $4}' "$SEEDS_TSV"; }
+mode="${1:-}"; [ -n "$mode" ] || { sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }; shift
+GO=0; OVERRIDE=""; BIN=""; SHA=""; MINER=""; DIGEST=""; COMMIT="c4459193"; NODE=""; OVJSON=""
+while [ $# -gt 0 ]; do case "$1" in
+  --go) GO=1; shift ;; --override) OVERRIDE="$2"; shift 2 ;; --igneumd) BIN="$2"; shift 2 ;; --sha256) SHA="$2"; shift 2 ;;
+  --miner) MINER="$2"; shift 2 ;; --digest) DIGEST="$2"; shift 2 ;; --commit) COMMIT="$2"; shift 2 ;; --node) NODE="$2"; shift 2 ;;
+  --override-json) OVJSON="$2"; shift 2 ;; *) die "unknown option $1" ;; esac; done
+
+one_seed() {   # <name>: runs in the background; prints one RESULT line at the end
+  local name="$1" ip log t0 line
+  ip=$(seed_ip "$name"); [ -n "$ip" ] || { echo "RESULT $name: no ip in $SEEDS_TSV"; return 1; }
+  log="$S/$name.log"; : > "$log"; t0=$(date +%s)
+  {
+    if [ "$GO" = 0 ]; then
+      echo "DRY: scp $BIN root@$ip:/opt/igneum/bin/igneumd.new; sha256 asserted = $SHA"
+      [ -n "$MINER" ] && echo "DRY: scp $MINER root@$ip:/opt/igneum/bin/igneum-miner.new (swapped with the node)"
+      echo "DRY: scp $OVERRIDE root@$ip:/etc/igneum/override-params.json; seed.env EXTRA_ARGS=--override-params-file=/etc/igneum/override-params.json"
+      echo "DRY: systemctl stop igneumd; mv igneumd igneumd.prev; mv igneumd.new igneumd; systemctl start igneumd"
+      echo "DRY: read back: grep -c $COMMIT in the binary; journal 'Consensus params digest' == $DIGEST; eth_syncing on 127.0.0.1:26890; peers"
+      echo "DRY RUN, nothing touched; box $ip answers as $("${SSH[@]}" "root@$ip" 'hostname; systemctl is-active igneumd; sha256sum /opt/igneum/bin/igneumd | cut -c1-12' 2>/dev/null | tr '\n' ' ')"
+    else
+    scp -q -i "$KEY" -o BatchMode=yes "$BIN" "root@$ip:/opt/igneum/bin/igneumd.new"
+    [ -n "$MINER" ] && scp -q -i "$KEY" -o BatchMode=yes "$MINER" "root@$ip:/opt/igneum/bin/igneum-miner.new"
+    scp -q -i "$KEY" -o BatchMode=yes "$OVERRIDE" "root@$ip:/etc/igneum/override-params.json"
+    "${SSH[@]}" "root@$ip" bash -s -- "$SHA" "$COMMIT" "$DIGEST" "$([ -n "$MINER" ] && echo 1 || echo 0)" <<'REMOTE'
+set -euo pipefail
+SHA="$1"; COMMIT="$2"; DIGEST="$3"; MINER="$4"; cd /opt/igneum/bin
+got=$(sha256sum igneumd.new | cut -d' ' -f1); [ "$got" = "$SHA" ] || { echo "sha256 MISMATCH on the box: $got"; exit 1; }
+python3 -c 'import json,sys; json.load(open("/etc/igneum/override-params.json"))' || { echo "override file does not parse"; exit 1; }
+chmod 755 igneumd.new; [ "$MINER" = 1 ] && chmod 755 igneum-miner.new
+grep -qE '^EXTRA_ARGS=' /etc/igneum/seed.env && sed -i -E 's#^EXTRA_ARGS=.*#EXTRA_ARGS="--override-params-file=/etc/igneum/override-params.json"#' /etc/igneum/seed.env || echo 'EXTRA_ARGS="--override-params-file=/etc/igneum/override-params.json"' >> /etc/igneum/seed.env
+t0=$(date +%s); systemctl stop igneumd
+mv -f igneumd igneumd.prev; mv -f igneumd.new igneumd; [ "$MINER" = 1 ] && { mv -f igneum-miner igneum-miner.prev 2>/dev/null || true; mv -f igneum-miner.new igneum-miner; }
+systemctl start igneumd; sleep 6
+down=$(( $(date +%s) - t0 ))
+commits=$(grep -a -c "$COMMIT" igneumd || true)
+first=$(journalctl -u igneumd --since "-40 s" --no-pager 2>/dev/null | grep -vE "Started|Stopped|Stopping|Deactivated|Consumed" | head -1 | cut -c1-120)
+dig=$(journalctl -u igneumd --since "-40 s" --no-pager 2>/dev/null | grep -oE "Consensus params digest: [0-9a-f]+" | tail -1 | awk '{print $4}')
+syncing=$(curl -s -m 5 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_syncing","params":[]}' http://127.0.0.1:26890 | cut -c1-80)
+peers=$(curl -s -m 5 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"net_peerCount","params":[]}' http://127.0.0.1:26890 | grep -oE '"result":"[^"]*"' | cut -d'"' -f4)
+dmatch=no; [ -n "$dig" ] && [ "$dig" = "$DIGEST" ] && dmatch=MATCH; [ -n "$dig" ] && [ "$dig" != "$DIGEST" ] && dmatch="MISMATCH($dig)"
+echo "unit $(systemctl is-active igneumd) down ${down}s; commit strings $commits; digest $dmatch; eth_syncing $syncing; peers $peers; first: $first"
+REMOTE
+    fi
+  } >> "$log" 2>&1; rc=$?
+  line=$(tail -1 "$log" | cut -c1-300)
+  echo "RESULT $name ($ip) rc=$rc after $(( $(date +%s) - t0 )) s: $line"
+}
+
+case "$mode" in
+  seeds)
+    [ -n "$OVERRIDE" ] && [ -n "$BIN" ] && [ -n "$SHA" ] && [ -n "$DIGEST" ] || die "seeds needs --override --igneumd --sha256 --digest"
+    [ -f "$OVERRIDE" ] && [ -f "$BIN" ] || die "override or binary file missing"
+    local_sha=$(shasum -a 256 "$BIN" | cut -d' ' -f1); [ "$local_sha" = "$SHA" ] || die "the binary's sha256 is $local_sha, not $SHA"
+    python3 -c 'import json,sys; json.load(open(sys.argv[1]))' "$OVERRIDE" || die "override file does not parse"
+    say "seeds: $( [ "$GO" = 1 ] && echo GO || echo DRY RUN ); igneumd $SHA; override $OVERRIDE ($(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))))' "$OVERRIDE") fields); digest $DIGEST; commit $COMMIT"
+    for n in seed1.testnet seed2.testnet seed3.testnet; do one_seed "$n" & done; wait
+    say "seeds done; logs in $S" ;;
+  hands)
+    [ -n "$NODE" ] && [ -n "$OVJSON" ] && [ -n "$DIGEST" ] || die "hands needs --node --override-json --digest"
+    g=""; [ "$GO" = 1 ] && g="--go"
+    say "hands: $( [ "$GO" = 1 ] && echo GO || echo DRY RUN ) through move-hand.sh"
+    "$HERE/hands/move-hand.sh" binary --node "$NODE" --override-json "$OVJSON" $g
+    "$HERE/hands/move-hand.sh" restart observer-node --digest "$DIGEST" $g
+    "$HERE/hands/move-hand.sh" restart node1 --digest "$DIGEST" $g ;;
+  lift-rpc-filter)
+    ip=$(seed_ip seed1.testnet)
+    if [ "$GO" = 0 ]; then
+      say "DRY RUN: on root@$ip: back up /opt/igneum/bin/rpc-filter.py, set BLOCKED_UNTIL_FIXED_NODE = set(), py_compile, systemctl restart igneum-rpc-filter, read back"
+      "${SSH[@]}" "root@$ip" 'echo "current: $(grep -c "^BLOCKED_UNTIL_FIXED_NODE = {" /opt/igneum/bin/rpc-filter.py) block set(s), unit $(systemctl is-active igneum-rpc-filter.service)"'; exit 0; fi
+    "${SSH[@]}" "root@$ip" bash -s <<'REMOTE'
+set -euo pipefail
+f=/opt/igneum/bin/rpc-filter.py; cp -a "$f" "$f.bak-$(date -u +%Y%m%dT%H%M%SZ)"
+python3 - <<'PY'
+import re
+p='/opt/igneum/bin/rpc-filter.py'; s=open(p).read()
+new, n = re.subn(r'BLOCKED_UNTIL_FIXED_NODE = \{.*?\n\}', 'BLOCKED_UNTIL_FIXED_NODE = set()   # lifted 7 Oct 2026: every seed runs the fixed node (N7 class fixed from 8097d600)', s, count=1, flags=re.S)
+assert n == 1, "block set not found"
+open(p,'w').write(new)
+PY
+python3 -m py_compile "$f"; systemctl restart igneum-rpc-filter.service; sleep 2
+echo "unit $(systemctl is-active igneum-rpc-filter.service); blocked set now: $(grep -E '^BLOCKED_UNTIL_FIXED_NODE' "$f" | cut -c1-60)"
+REMOTE
+    say "public read-back: $(curl -s -m 8 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' https://rpc.testnet.igneum.network | cut -c1-120)" ;;
+  *) die "unknown mode $mode" ;;
+esac
diff --git a/tools/ci/box-locks-check.sh b/tools/ci/box-locks-check.sh
index 75eb34b42..897cbafc3 100755
--- a/tools/ci/box-locks-check.sh
+++ b/tools/ci/box-locks-check.sh
@@ -14,8 +14,9 @@ stamp="ci-$$-$(date +%s)"
 scp -q "${BS_SSH_OPTS[@]}" infra/build-server/lease.sh "$BS_HOST:/tmp/lease-$stamp.sh"
 scp -q "${BS_SSH_OPTS[@]}" infra/build-server/remote-run.sh "$BS_HOST:/tmp/rr-$stamp.sh"
 rc=0
-bs_ssh "bash /tmp/lease-$stamp.sh --self-test 2>&1 | grep -E '^lease self-test'; exit \${PIPESTATUS[0]}" || rc=1
-bs_ssh "IGNEUM_REMOTE_RUN_UNDER_TEST=/tmp/rr-$stamp.sh bash /tmp/rr-$stamp.sh --self-test-slots 2>&1 | grep -E '^self-test-slots'; exit \${PIPESTATUS[0]}" || rc=1
-bs_ssh "rm -f /tmp/lease-$stamp.sh /tmp/rr-$stamp.sh" || true
+# the two self-tests run in parallel in one ssh session (each against its own scratch lock directory); their last lines are printed
+bs_ssh "set -o pipefail; (bash /tmp/lease-$stamp.sh --self-test 2>&1 | grep -E '^lease self-test'; echo lease=\${PIPESTATUS[0]} >> /tmp/locks-$stamp.rc) & (IGNEUM_REMOTE_RUN_UNDER_TEST=/tmp/rr-$stamp.sh bash /tmp/rr-$stamp.sh --self-test-slots 2>&1 | grep -E '^self-test-slots'; echo slots=\${PIPESTATUS[0]} >> /tmp/locks-$stamp.rc) & wait; cat /tmp/locks-$stamp.rc; rm -f /tmp/lease-$stamp.sh /tmp/rr-$stamp.sh /tmp/locks-$stamp.rc" | tee /tmp/box-locks-$$.out || rc=1
+grep -q '^lease=0$' /tmp/box-locks-$$.out && grep -q '^slots=0$' /tmp/box-locks-$$.out || rc=1
+rm -f /tmp/box-locks-$$.out
 [ "$rc" = 0 ] && echo "box-locks: the lease tool and the slot runner pass their self-tests on $BS_HOST"
 exit $rc
diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh
index c4f84dac8..15761bd9c 100755
--- a/tools/ci/pre-push.sh
+++ b/tools/ci/pre-push.sh
@@ -88,6 +88,7 @@ tree_checks() {
   run "build-remote without a priority flag bounds suites and benches (nice 10, 32 cores); a gate runs unbounded" bash tools/ci/build-kind-default-check.sh
   run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh
   run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh)" bash tools/ci/box-locks-check.sh
+  run "the simulators job runs on master and release-* pushes and pull requests into them only" bash tools/ci/sims-branch-check.sh
   run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh'
   run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh'
   run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test
diff --git a/tools/ci/sims-branch-check.sh b/tools/ci/sims-branch-check.sh
new file mode 100755
index 000000000..c25e12bd8
--- /dev/null
+++ b/tools/ci/sims-branch-check.sh
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# The simulators job of .github/workflows/ci.yml runs on master and release-* pushes, and on pull requests into them, ONLY (main,
+# 7 October 2026: with build-2 in the runner pool the queue still read 22 because every code push cost two box jobs; a
+# feature-branch code push now runs the igneum-pow tests alone). This check reads the job's `if:` line and evaluates the rule
+# with the expression's own shape for the known cases, so a change that lets a feature-branch push run the simulators fails it.
+#
+#   tools/ci/sims-branch-check.sh    # exit 1 with the case that resolved wrongly (the check IS its self-test)
+set -euo pipefail
+cd "$(dirname "$0")/../.."
+line=$(awk '/^  sims:/{f=1} f && /^    if:/{print; exit}' .github/workflows/ci.yml)
+[ -n "$line" ] || { echo "sims-branch: no if: line under the sims job" >&2; exit 1; }
+for want in "github.event_name == 'push'" "github.ref == 'refs/heads/master'" "startsWith(github.ref, 'refs/heads/release-')" \
+            "github.event_name == 'pull_request'" "github.base_ref == 'master'" "startsWith(github.base_ref, 'release-')" "needs.changes.outputs.code == 'true'"; do
+  case "$line" in *"$want"*) ;; *) echo "sims-branch: the sims if: line lacks \"$want\": $line" >&2; exit 1 ;; esac
+done
+# the rule, as the expression reads: code and ((push and (master or release-*)) or (pull_request and base (master or release-*)))
+rule() {   # <code> <event> <ref> <base_ref> -> run|skip
+  local code="$1" ev="$2" ref="$3" base="$4"
+  [ "$code" = true ] || { echo skip; return; }
+  if [ "$ev" = push ] && { [ "$ref" = refs/heads/master ] || [[ "$ref" == refs/heads/release-* ]]; }; then echo run; return; fi
+  if [ "$ev" = pull_request ] && { [ "$base" = master ] || [[ "$base" == release-* ]]; }; then echo run; return; fi
+  echo skip
+}
+fail=0
+expect() { local want="$1"; shift; local got; got=$(rule "$@"); if [ "$got" = "$want" ]; then echo "sims-branch: [$*] -> $got"; else echo "sims-branch: [$*] -> $got, expected $want" >&2; fail=1; fi; }
+expect run  true push refs/heads/master ""
+expect run  true push refs/heads/release-0.3.20 ""
+expect skip true push refs/heads/build-server ""
+expect skip true push refs/heads/feature/x ""
+expect run  true pull_request refs/pull/12/merge master
+expect run  true pull_request refs/pull/12/merge release-0.3.21
+expect skip true pull_request refs/pull/12/merge build-server
+expect skip false push refs/heads/master ""
+[ "$fail" = 0 ] && echo "sims-branch: the simulators run on master and release-* pushes and on pull requests into them only"
+exit $fail