From d1285cefcc8b639906bb967e838c09e59ac79ef7 Mon Sep 17 00:00:00 2001
From: igneum-josh <337424239+igneum-josh@users.noreply.github.com>
Date: Wed, 7 Oct 2026 12:22:09 +0100
Subject: [PATCH] Build boxes: the slot holder keeps its own line (a keeper,
 with its self-test in the gate); an explicit --jobs is clamped for bounded
 classes; one git remote per box; the host-file double suffix

The dashboard lane, 7 October 2026 10:39Z: both slots of build-1 flock-held and EMPTY while two suites ran. Cause: a run from a
worktree without last night's append-mode fix opens a busy sibling's slot file with > on every probe. The holder now keeps its own
line: a keeper re-writes it within BR_KEEP_S (20 s) whenever the file is empty, until release; remote-run.sh --self-test-keeper
(in the gate) truncates a held line and sees it return, and sees nothing written after release; live on build-1 at 11:19Z (the
line came back in 25 s). The first version deadlocked the runner's bare wait with the keeper (build-2's first run hung 15 min
after its test passed): the keeper stops before the wait. The two running suites' -j 90 came from explicit --jobs 90: a bounded
class now clamps it to its cap with a log line (pass --priority gate for the full set). run-from-mac.sh --box N: the host file
was suffixed twice (build-server-2-2) and every box's mirror would have shared one remote name; one remote per box (build-N).
build-2's first green run: a suite at nice 10 on 32 cores, jobs 32, 986 s cold.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 infra/build-server/remote-run.sh   | 30 +++++++++++++++++++++++++++++-
 infra/build-server/run-from-mac.sh | 20 +++++++++++---------
 tools/build-remote.sh              |  3 +++
 tools/ci/pre-push.sh               |  1 +
 4 files changed, 44 insertions(+), 10 deletions(-)

diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh
index 8fbe6c133..f3b2451e1 100755
--- a/infra/build-server/remote-run.sh
+++ b/infra/build-server/remote-run.sh
@@ -110,6 +110,21 @@ checkout_tree() {
   set +e
 }
 
+if [ "${1:-}" = --self-test-keeper ]; then
+  # the keeper restores a truncated holder line within its interval, and stops at release
+  t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
+  BR_PID=$$; BR_LABEL="keeper self-test"; got=0; waited=3; SLOTS_DIR="$t"; BR_KEEP_S=1
+  holder_line() { printf 'pid %s since %sZ waited %s s: %s\n' "$BR_PID" "$(date -u +%H:%M:%S)" "$1" "$BR_LABEL"; }
+  holder_line "$waited" > "$t/build-0"
+  keeper_pid=""; keep_line() { local f="$1" w="$2"; ( while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; sleep "${BR_KEEP_S:-20}"; done ) & keeper_pid=$!; }
+  keep_line "$t/build-0" "$waited"
+  : > "$t/build-0"; sleep 2.5
+  grep -q "^pid $$ since .* waited 3 s: keeper self-test$" "$t/build-0" || { echo "keeper self-test: the truncated holder line was NOT restored"; kill "$keeper_pid" 2>/dev/null; exit 1; }
+  kill "$keeper_pid" 2>/dev/null; wait "$keeper_pid" 2>/dev/null; : > "$t/build-0"; sleep 2.5
+  [ ! -s "$t/build-0" ] || { echo "keeper self-test: the keeper kept writing after release"; exit 1; }
+  echo "keeper self-test: a truncated holder line comes back within the interval; nothing is written after release"; exit 0
+fi
+
 if [ "${1:-}" = --self-test ]; then
   t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
   git init -q --bare -b master "$t/mirror.git"
@@ -369,7 +384,17 @@ else
   echo "build-remote: holding build-$got on $BR_HOST (waited $waited s; $held of $slots slots held, CARGO_BUILD_JOBS=$BR_JOBS, kind ${BR_KIND:-other}, nice ${BR_NICE:-0}, cores $( [ "${BR_CORES:-0}" = 0 ] && nproc || echo "$BR_CORES"))" >&2
 fi
 
-release_slot() { if [ "$got" = measure ]; then : > "$SLOTS_DIR/measure"; else : > "$SLOTS_DIR/build-$got"; fi; }
+# The holder keeps its own line (the watcher-trust rule, 7 October 2026 10:39Z: two held slots read EMPTY while two suites ran,
+# because a run from a worktree without last night's append-mode fix still opens a busy sibling's slot file with `>` on every
+# probe). A keeper re-writes the holder line whenever it finds the file empty, every BR_KEEP_S seconds, until release_slot.
+keeper_pid=""
+keep_line() {   # <file> <waited>
+  local f="$1" w="$2"
+  ( while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; sleep "${BR_KEEP_S:-20}"; done ) &
+  keeper_pid=$!
+}
+if [ "$got" = measure ]; then keep_line "$SLOTS_DIR/measure" "$waited"; else keep_line "$SLOTS_DIR/build-$got" "$waited"; fi
+release_slot() { [ -n "$keeper_pid" ] && { kill "$keeper_pid" 2>/dev/null; wait "$keeper_pid" 2>/dev/null; keeper_pid=""; }; if [ "$got" = measure ]; then : > "$SLOTS_DIR/measure"; else : > "$SLOTS_DIR/build-$got"; fi; }
 cd "$BR_DIR" || { BR_CLASS=no-dir jsonlog 2 "$got" "$waited" "" "$(date +%s)" "" "" "" "" "" ""; redlog 2 0 no-dir; release_slot; exit 2; }
 
 # PRE-FLIGHT for a cargo command (the instant-death class, 6 October 2026): the manifest parses and every -p package exists,
@@ -425,6 +450,9 @@ ncpu=$(nproc); cores_str="0-$((ncpu - 1))"
 if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then cores_str="$((ncpu - BR_CORES))-$((ncpu - 1))"; fi
 ( [ "${BR_NICE:-0}" -gt 0 ] && renice -n "$BR_NICE" -p $BASHPID >/dev/null 2>&1; [ "$cores_str" != "0-$((ncpu - 1))" ] && taskset -cp "$cores_str" $BASHPID >/dev/null 2>&1; eval "$BR_CMD" ) > >(tee -a "$BR_RUN_LOG") 2> >(tee -a "$BR_RUN_LOG" >&2)
 rc=$?
+# the keeper stops BEFORE the bare `wait` (which flushes the two tees): a bare wait also waits for the keeper, and the keeper waits
+# for this script, a deadlock that held build-2's first run 15 minutes after its test had passed (7 Oct 2026, 11:04 to 11:19Z)
+if [ -n "$keeper_pid" ]; then kill "$keeper_pid" 2>/dev/null; wait "$keeper_pid" 2>/dev/null; keeper_pid=""; fi
 wait
 t2=$(date +%s); secs=$(( t2 - t1 ))
 exec_after=$(stat_field "Compile requests executed"); hits_after=$(stat_field "Cache hits "); misses_after=$(stat_field "Cache misses ")
diff --git a/infra/build-server/run-from-mac.sh b/infra/build-server/run-from-mac.sh
index 4c4219748..a5a275a8c 100755
--- a/infra/build-server/run-from-mac.sh
+++ b/infra/build-server/run-from-mac.sh
@@ -27,7 +27,9 @@ WIRE_ONLY=0; [ "${1:-}" = --wire-only ] && WIRE_ONLY=1
 # box N is igneum-build-N with its own dashboard feed name build-N.igneum.network (the dashboard lane's collector reads one server
 # section per box; main adds the A record with the IP)
 [ "$BOX" = 1 ] || { BOX_HOSTNAME="${BOX_HOSTNAME:-igneum-build-$BOX}"; WORKERS_HOST="${WORKERS_HOST:-build-$BOX.igneum.network}"; export BOX_HOSTNAME WORKERS_HOST; }
-BS_HOST_FILE=$(bs_box_file "$BOX"); export BS_BOX="$BOX"   # lib.sh's bs_host reads BS_BOX
+export BS_BOX="$BOX"   # lib.sh's bs_host derives the host file from BS_BOX (box 1: build-server; box N: build-server-N); never set BS_HOST_FILE here (7 Oct 2026: a second suffix, build-server-2-2)
+HOST_FILE=$(bs_box_file "$BOX")
+REMOTE="build"; [ "$BOX" = 1 ] || REMOTE="build-$BOX"   # one git remote per box in the two repositories
 # the toolchain pin travels from rust-toolchain.toml unless RUST_TOOLCHAIN is set by hand (one file pins every side, 7 Oct 2026)
 [ -n "${RUST_TOOLCHAIN:-}" ] || RUST_TOOLCHAIN=$(sed -n 's/^channel *= *"\([^"]*\)".*/\1/p' "$REPO/rust-toolchain.toml" 2>/dev/null | head -1); export RUST_TOOLCHAIN
 PASS=""   # a string, not an array: bash 3.2 (the Mac) treats an empty array as unbound under set -u
@@ -48,19 +50,19 @@ if [ "$WIRE_ONLY" = 0 ]; then
   if "${ROOT_SSH[@]}" 'hostname' 2>/dev/null | grep -q '^rescue'; then bs_die "still in the rescue system after provision.sh"; fi
 fi
 
-mkdir -p "$(dirname "$BS_HOST_FILE")"
-printf 'build@%s\n' "$IP" > "$BS_HOST_FILE"
-bs_log "wrote $BS_HOST_FILE"
+mkdir -p "$(dirname "$HOST_FILE")"
+printf 'build@%s\n' "$IP" > "$HOST_FILE"
+bs_log "wrote $HOST_FILE"
 bs_host
 bs_ssh 'hostname; nproc' >/dev/null || bs_die "build@$IP does not answer with $BS_KEY"
 
 wire_remote() {   # repo-dir mirror label
   local dir="$1" mirror="$2" label="$3" url="$BS_HOST:$2" cur
-  cur=$(git -C "$dir" remote get-url build 2>/dev/null || true)
-  if [ -z "$cur" ]; then git -C "$dir" remote add build "$url"; bs_log "$label: remote build = $url"
-  elif [ "$cur" != "$url" ]; then git -C "$dir" remote set-url build "$url"; bs_log "$label: remote build -> $url"
-  else bs_log "$label: remote build ok"; fi
-  GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$dir" push -q --force build --all && bs_log "$label: every branch pushed to $mirror ($(git -C "$dir" branch --list | wc -l | tr -d ' ') branches)" || bs_die "$label: push to $mirror failed"
+  cur=$(git -C "$dir" remote get-url "$REMOTE" 2>/dev/null || true)
+  if [ -z "$cur" ]; then git -C "$dir" remote add "$REMOTE" "$url"; bs_log "$label: remote $REMOTE = $url"
+  elif [ "$cur" != "$url" ]; then git -C "$dir" remote set-url "$REMOTE" "$url"; bs_log "$label: remote $REMOTE -> $url"
+  else bs_log "$label: remote $REMOTE ok"; fi
+  GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$dir" push -q --force "$REMOTE" --all && bs_log "$label: every branch pushed to $mirror ($(git -C "$dir" branch --list | wc -l | tr -d ' ') branches)" || bs_die "$label: push to $mirror failed"
 }
 wire_remote "$MAIN_REPO" "$BS_MIRROR_REPO" "igneum"
 wire_remote "$MAIN_REPO/vendor/igneum-node" "$BS_MIRROR_NODE" "igneum-node (the fork)"
diff --git a/tools/build-remote.sh b/tools/build-remote.sh
index 05e40320f..9618ec3d5 100755
--- a/tools/build-remote.sh
+++ b/tools/build-remote.sh
@@ -111,6 +111,9 @@ case "${CARGO_ARGS[0]:-build}" in
 esac
 if [ "$PRIORITY" = gate ]; then BR_NICE=0; BR_CORES=0; BR_JOBS_CAP=0
 elif [ "$SCHED_CLASS" = suite ] || [ "$SCHED_CLASS" = bench ]; then BR_NICE=10; BR_CORES=32; BR_JOBS_CAP=32; fi
+# an explicit --jobs above the bounded class's cap is clamped (main's rule: bounded unless a priority flag; 7 Oct 2026: two suites
+# ran at -j 90 on a box at load 190 because their callers passed --jobs 90)
+if [ "$BR_JOBS_CAP" -gt 0 ] && [ -n "$JOBS" ] && [ "$JOBS" -gt "$BR_JOBS_CAP" ]; then bs_log "--jobs $JOBS clamped to $BR_JOBS_CAP for a $SCHED_CLASS (pass --priority gate for the full set)"; JOBS=$BR_JOBS_CAP; fi
 export BR_NICE BR_CORES BR_JOBS_CAP BR_PRIORITY
 if [ "$PLAN" = 1 ]; then
   k="$SCHED_CLASS"; [ "$PRIORITY" = gate ] && k=gate
diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh
index 7db097899..659e98396 100755
--- a/tools/ci/pre-push.sh
+++ b/tools/ci/pre-push.sh
@@ -72,6 +72,7 @@ tree_checks() {
   run "root prover playbooks kill the GPU server and unlink its socket"        bash tools/ci/prover-socket-check.sh
   run "commit-string gate self-test"                                           bash tools/ci/commit-string-check.sh --self-test
   run "build server remote checkout self-test"                                 bash infra/build-server/remote-run.sh --self-test
+  run "a slot holder keeps its own line for the whole run (the watcher-trust rule)" bash infra/build-server/remote-run.sh --self-test-keeper
   run "the remote checkout resets the mirror's tree before the branch checkout (the stale-overlay class)" bash -c 'bash tools/ci/mirror-reset-check.sh --self-test && bash tools/ci/mirror-reset-check.sh'
   run "the remote checkout's clean spares a lane's scratch (.igneum-scratch-spare, the fixed prefixes, never -x; the lost-scratch class)" bash -c 'bash tools/ci/scratch-spare-check.sh --self-test && bash tools/ci/scratch-spare-check.sh'
   run "long-running tools keep their body in one parsed block (the edited-while-running class)" bash -c 'bash tools/ci/whole-body-check.sh --self-test && bash tools/ci/whole-body-check.sh'