diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh index c0e0c196..7cc7ba43 100755 --- a/infra/build-server/remote-run.sh +++ b/infra/build-server/remote-run.sh @@ -208,14 +208,16 @@ if [ "${1:-}" = --self-test-slots ]; then # 4b. a run keeps off leased cores: with core 1 leased, a 2-core bounded run on a 2-core box says so (the exclusion line) # the lease outlives the fake's slot take and settle (a loaded box took them past 2 s and the first version read no lease) ( exec 9>>"$t/locks/core-$(( $(nproc) - 1 ))"; flock 9; sleep 12 ) & sleep 0.5; BR_CORES=2 BR_NICE=10 fake p 1; wait - grep -q 'are leased to a measurement; this run keeps to' "$t/p.out" || fail "a run beside a leased core did not exclude it: $(cat "$t/p.out" | tail -3)" + # the bounded run leases its own pool cores, so the measurement's core is never among them (the pool line names the set) + grep -q 'bounded pool: 2 core(s) leased' "$t/p.out" || fail "a bounded run did not lease its pool cores: $(cat "$t/p.out" | tail -3)" + grep -E 'bounded pool' "$t/p.out" | grep -qE "[(,]$(( $(nproc) - 1 ))[,.]" && fail "a bounded run leased the measurement's core" # 5. a probing build leaves a busy slot's holder line intact fake f 3 & sleep 1.2; fake g 1 & sleep 0.3 grep -q 'self-test f' "$t/locks/build-0" || fail "the holder line of the busy slot build-0 was lost when another build probed it: '$(cat "$t/locks/build-0")'" wait # 6. the log carries the job count and the measure flag grep -q '"jobs":44' "$t/log/builds.jsonl" && grep -q '"measure":true' "$t/log/builds.jsonl" || fail "builds.jsonl lacks jobs or measure fields" - echo "self-test-slots: two concurrent builds 44 each, a lone build 88, a quiet blocks an unbounded build and not a bounded suite, a quiet is refused beside a slot or a lease, a run keeps off leased cores, a probe keeps the holder line, the log carries jobs and measure"; exit 0 + echo "self-test-slots: two concurrent builds 44 each, a lone build 88, a quiet blocks an unbounded build and not a bounded suite, a quiet is refused beside a slot or a lease, a bounded run leases its pool cores clear of a measurement, a probe keeps the holder line, the log carries jobs and measure"; exit 0 fi # One run per worktree directory at a time (6 October 2026, 19:51:09 UK: two runs of one worktree started in the same second; @@ -408,7 +410,7 @@ keeper_pid="" keep_line() { # ; the keeper also refreshes the file's mtime (a stopped holder's file goes stale) and reaps stopped # holders of any lease, quiet or slot after 5 minutes through lease.sh (/srv/builds/_bin/lease reap, one line each in reaped.log) local f="$1" w="$2" - ( exec {mfd}>&- {fd}>&- 2>/dev/null; [ -n "${WT_FD:-}" ] && exec {WT_FD}>&-; while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; touch "$f" 2>/dev/null + ( exec {mfd}>&- {fd}>&- 2>/dev/null; [ -n "${WT_FD:-}" ] && exec {WT_FD}>&-; for pfd in "${pool_fds[@]}"; do exec {pfd}>&-; done; while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; touch "$f" 2>/dev/null [ -x /srv/builds/_bin/lease ] && IGNEUM_BUILD_SLOTS_DIR="$SLOTS_DIR" IGNEUM_BUILD_LOG_DIR="$LOG_DIR" /srv/builds/_bin/lease reap >/dev/null 2>&1 sleep "${BR_KEEP_S:-20}"; done ) & keeper_pid=$! @@ -469,18 +471,42 @@ BR_RUN_LOG="$RUN_LOG_DIR/$BR_HOST-$BR_T0-$BR_PID.log"; export BR_RUN_LOG # (since the third slot on build-2, 7 Oct 2026: a bounded run takes the band its SLOT owns, counted from the top: slot 0 the last N # cores, slot 1 the N below, slot 2 the N below that, so three bounded runs never share a core; a band below core 0 falls back to # the last N) -# (the project lead, 7 Oct 2026 19:4x BST, both boxes to near max: a bounded run takes the LAST N cores, N = 88 by default, leaving the first 8 -# to the release builds, the seed and the observer processes; with N above half the box the slots share the band, and nice 10 plus -# the per-slot jobs rule keep two suites fair; a smaller N (IGNEUM_BOUND_CORES) returns to disjoint bands per slot when it fits) +# The bounded POOL (main, 7 Oct 2026 20:0x BST: with every bounded run taking 88 threads the boxes read load 280 to 350; the ceiling +# is per BOX, not per job). Cores 8 to 95 (the first 8 reserved for the release builds, the seed and the observers) form one pool of +# 88 per box; a bounded run leases free cores from it (one flock per core, the same core- files a pinned measurement leases, so +# measurements and suites share the pool), up to BR_CORES (88 by default, or the caller's --jobs), and never fewer than +# BR_POOL_MIN (16): when fewer are free it waits in the queue (its wait- file says so) and takes them as they free; CARGO_BUILD_JOBS +# becomes the count taken and taskset the set taken. So the sum of bounded threads on a box never passes 88. An unbounded run +# (nice 0, a gate or a release build) is outside the pool and keeps the full set. ncpu=$(nproc); cores_str="0-$((ncpu - 1))" -if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then - band=0; case "${got:-}" in ''|measure) ;; *) band=$got ;; esac - lo=$((ncpu - BR_CORES * (band + 1))); [ "$lo" -ge 0 ] || lo=$((ncpu - BR_CORES)) - cores_str="$lo-$((lo + BR_CORES - 1))" +pool_fds=(); pool_cores=() +pool_take() { # : lease up to free pool cores, at least , waiting up to 2 h; sets pool_cores/pool_fds + local want="$1" min="$2" t0 c fd taken + t0=$(date +%s) + while :; do + taken=0; pool_fds=(); pool_cores=() + for ((c = ${BR_POOL_RESERVE:-8}; c < ncpu && taken < want; c++)); do + exec {fd}>>"$SLOTS_DIR/core-$c" + if flock -n "$fd"; then pool_fds+=("$fd"); pool_cores+=("$c"); taken=$((taken + 1)); else exec {fd}>&-; fi + done + if [ "$taken" -ge "$min" ]; then return 0; fi + for fd in "${pool_fds[@]}"; do exec {fd}>&-; done; pool_fds=(); pool_cores=() + [ -f "$waitfile" ] || { echo "build-remote: the bounded pool has $taken free core(s) (want $want, at least $min): waiting for the pool" >&2; holder_line "$(( $(date +%s) - BR_T0 ))" > "$waitfile"; } + [ $(( $(date +%s) - t0 )) -lt 7200 ] || give_up "the bounded pool" + sleep 10 + done +} +if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ] && [ "$got" != measure ]; then + want="$BR_CORES"; [ "${BR_JOBS_CAP:-0}" -gt 0 ] && [ "$BR_JOBS_CAP" -lt "$want" ] && want="$BR_JOBS_CAP" # the class cap, lowered by an explicit --jobs + pool_take "$want" "$(( want < ${BR_POOL_MIN:-16} ? want : ${BR_POOL_MIN:-16} ))" + rm -f "$waitfile" 2>/dev/null + cores_str=$(printf '%s,' "${pool_cores[@]}"); cores_str=${cores_str%,} + BR_JOBS=${#pool_cores[@]}; export CARGO_BUILD_JOBS="$BR_JOBS" BR_JOBS + echo "build-remote: bounded pool: ${#pool_cores[@]} core(s) leased ($(echo "$cores_str" | cut -c1-60)...), CARGO_BUILD_JOBS=$BR_JOBS" >&2 fi # leased cores (lease.sh: a pinned measurement's core- flocks) are taken out of this run's set; a set that would be empty keeps # its cores (the measurement is told by its own lease line); the exclusion is said once -if [ "$got" != measure ]; then +if [ "$got" != measure ] && [ "${#pool_cores[@]}" = 0 ]; then leased=$(held_cores) if [ -n "$leased" ]; then kept=$(python3 -c '