igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

Build boxes: the bounded POOL (main, 7 Oct 2026 20:0x BST: every bounded run took 88 threads and the boxes read load 280 to 527): cores 8 to 95 form one pool of 88 per box; a bounded run leases free cores from it (one flock per core, shared with the measurement leases), up to its class cap or --jobs and never fewer than 16, waiting in the queue when fewer are free; CARGO_BUILD_JOBS and taskset follow the cores taken, so the sum of bounded threads on a box never passes 88; self-test updated

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-07 19:03:35 +00:00
parent 5ec454fd63
commit 8e0aa605f7
1 changed files with 37 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
infra/build-server/remote-run.sh
View file

@ -208,14 +208,16 @@ if [ "${1:-}" = --self-test-slots ]; then
# 4b. a run keeps off leased cores: with core 1 leased, a 2-core bounded run on a 2-core box says so (the exclusion line)
# the lease outlives the fake's slot take and settle (a loaded box took them past 2 s and the first version read no lease)
( exec 9>>"$t/locks/core-$(( $(nproc) - 1 ))"; flock 9; sleep 12 ) & sleep 0.5; BR_CORES=2 BR_NICE=10 fake p 1; wait
grep -q 'are leased to a measurement; this run keeps to' "$t/p.out" || fail "a run beside a leased core did not exclude it: $(cat "$t/p.out" | tail -3)"
# the bounded run leases its own pool cores, so the measurement's core is never among them (the pool line names the set)
grep -q 'bounded pool: 2 core(s) leased' "$t/p.out" || fail "a bounded run did not lease its pool cores: $(cat "$t/p.out" | tail -3)"
grep -E 'bounded pool' "$t/p.out" | grep -qE "[(,]$(( $(nproc) - 1 ))[,.]" && fail "a bounded run leased the measurement's core"
# 5. a probing build leaves a busy slot's holder line intact
fake f 3 & sleep 1.2; fake g 1 & sleep 0.3
grep -q 'self-test f' "$t/locks/build-0" || fail "the holder line of the busy slot build-0 was lost when another build probed it: '$(cat "$t/locks/build-0")'"
wait
# 6. the log carries the job count and the measure flag
grep -q '"jobs":44' "$t/log/builds.jsonl" && grep -q '"measure":true' "$t/log/builds.jsonl" || fail "builds.jsonl lacks jobs or measure fields"
echo "self-test-slots: two concurrent builds 44 each, a lone build 88, a quiet blocks an unbounded build and not a bounded suite, a quiet is refused beside a slot or a lease, a run keeps off leased cores, a probe keeps the holder line, the log carries jobs and measure"; exit 0
echo "self-test-slots: two concurrent builds 44 each, a lone build 88, a quiet blocks an unbounded build and not a bounded suite, a quiet is refused beside a slot or a lease, a bounded run leases its pool cores clear of a measurement, a probe keeps the holder line, the log carries jobs and measure"; exit 0
fi
# One run per worktree directory at a time (6 October 2026, 19:51:09 UK: two runs of one worktree started in the same second;
@ -408,7 +410,7 @@ keeper_pid=""
keep_line() { # <file> <waited>; the keeper also refreshes the file's mtime (a stopped holder's file goes stale) and reaps stopped
# holders of any lease, quiet or slot after 5 minutes through lease.sh (/srv/builds/_bin/lease reap, one line each in reaped.log)
local f="$1" w="$2"
( exec {mfd}>&- {fd}>&- 2>/dev/null; [ -n "${WT_FD:-}" ] && exec {WT_FD}>&-; while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; touch "$f" 2>/dev/null
( exec {mfd}>&- {fd}>&- 2>/dev/null; [ -n "${WT_FD:-}" ] && exec {WT_FD}>&-; for pfd in "${pool_fds[@]}"; do exec {pfd}>&-; done; while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; touch "$f" 2>/dev/null
[ -x /srv/builds/_bin/lease ] && IGNEUM_BUILD_SLOTS_DIR="$SLOTS_DIR" IGNEUM_BUILD_LOG_DIR="$LOG_DIR" /srv/builds/_bin/lease reap >/dev/null 2>&1
sleep "${BR_KEEP_S:-20}"; done ) &
keeper_pid=$!
@ -469,18 +471,42 @@ BR_RUN_LOG="$RUN_LOG_DIR/$BR_HOST-$BR_T0-$BR_PID.log"; export BR_RUN_LOG
# (since the third slot on build-2, 7 Oct 2026: a bounded run takes the band its SLOT owns, counted from the top: slot 0 the last N
# cores, slot 1 the N below, slot 2 the N below that, so three bounded runs never share a core; a band below core 0 falls back to
# the last N)
# (the project lead, 7 Oct 2026 19:4x BST, both boxes to near max: a bounded run takes the LAST N cores, N = 88 by default, leaving the first 8
# to the release builds, the seed and the observer processes; with N above half the box the slots share the band, and nice 10 plus
# the per-slot jobs rule keep two suites fair; a smaller N (IGNEUM_BOUND_CORES) returns to disjoint bands per slot when it fits)
# The bounded POOL (main, 7 Oct 2026 20:0x BST: with every bounded run taking 88 threads the boxes read load 280 to 350; the ceiling
# is per BOX, not per job). Cores 8 to 95 (the first 8 reserved for the release builds, the seed and the observers) form one pool of
# 88 per box; a bounded run leases free cores from it (one flock per core, the same core-<n> files a pinned measurement leases, so
# measurements and suites share the pool), up to BR_CORES (88 by default, or the caller's --jobs), and never fewer than
# BR_POOL_MIN (16): when fewer are free it waits in the queue (its wait-<pid> file says so) and takes them as they free; CARGO_BUILD_JOBS
# becomes the count taken and taskset the set taken. So the sum of bounded threads on a box never passes 88. An unbounded run
# (nice 0, a gate or a release build) is outside the pool and keeps the full set.
ncpu=$(nproc); cores_str="0-$((ncpu - 1))"
if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then
band=0; case "${got:-}" in ''|measure) ;; *) band=$got ;; esac
lo=$((ncpu - BR_CORES * (band + 1))); [ "$lo" -ge 0 ] || lo=$((ncpu - BR_CORES))
cores_str="$lo-$((lo + BR_CORES - 1))"
pool_fds=(); pool_cores=()
pool_take() { # <want> <min>: lease up to <want> free pool cores, at least <min>, waiting up to 2 h; sets pool_cores/pool_fds
local want="$1" min="$2" t0 c fd taken
t0=$(date +%s)
while :; do
taken=0; pool_fds=(); pool_cores=()
for ((c = ${BR_POOL_RESERVE:-8}; c < ncpu && taken < want; c++)); do
exec {fd}>>"$SLOTS_DIR/core-$c"
if flock -n "$fd"; then pool_fds+=("$fd"); pool_cores+=("$c"); taken=$((taken + 1)); else exec {fd}>&-; fi
done
if [ "$taken" -ge "$min" ]; then return 0; fi
for fd in "${pool_fds[@]}"; do exec {fd}>&-; done; pool_fds=(); pool_cores=()
[ -f "$waitfile" ] || { echo "build-remote: the bounded pool has $taken free core(s) (want $want, at least $min): waiting for the pool" >&2; holder_line "$(( $(date +%s) - BR_T0 ))" > "$waitfile"; }
[ $(( $(date +%s) - t0 )) -lt 7200 ] || give_up "the bounded pool"
sleep 10
done
}
if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ] && [ "$got" != measure ]; then
want="$BR_CORES"; [ "${BR_JOBS_CAP:-0}" -gt 0 ] && [ "$BR_JOBS_CAP" -lt "$want" ] && want="$BR_JOBS_CAP" # the class cap, lowered by an explicit --jobs
pool_take "$want" "$(( want < ${BR_POOL_MIN:-16} ? want : ${BR_POOL_MIN:-16} ))"
rm -f "$waitfile" 2>/dev/null
cores_str=$(printf '%s,' "${pool_cores[@]}"); cores_str=${cores_str%,}
BR_JOBS=${#pool_cores[@]}; export CARGO_BUILD_JOBS="$BR_JOBS" BR_JOBS
echo "build-remote: bounded pool: ${#pool_cores[@]} core(s) leased ($(echo "$cores_str" | cut -c1-60)...), CARGO_BUILD_JOBS=$BR_JOBS" >&2
fi
# leased cores (lease.sh: a pinned measurement's core-<n> flocks) are taken out of this run's set; a set that would be empty keeps
# its cores (the measurement is told by its own lease line); the exclusion is said once
if [ "$got" != measure ]; then
if [ "$got" != measure ] && [ "${#pool_cores[@]}" = 0 ]; then
leased=$(held_cores)
if [ -n "$leased" ]; then
kept=$(python3 -c '