From 8e77476a59d1acf06cd7dd6ce02cc5305610b467 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:23:56 +0000 Subject: [PATCH] lease pool -- cmd (main, 7 Oct 2026 20:17 BST: adversarial binaries started by hand at 64 to 89 threads, several beside each other, read load 601): a sweep takes up to its thread count of FREE cores from the same 88-core bounded pool as the builds, never fewer than --min, waits while fewer are free, runs pinned at nice 10 with {cores} and {cpuset} substituted and LEASE_CORES exported; the rule is no sweep starts except through it; self-test case added; installed on both boxes Co-Authored-By: Claude Fable 5.1 --- infra/build-server/lease.sh | 59 +++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/infra/build-server/lease.sh b/infra/build-server/lease.sh index 5138474b..be7554c5 100755 --- a/infra/build-server/lease.sh +++ b/infra/build-server/lease.sh @@ -12,6 +12,14 @@ # A WHOLE-BOX quiet measurement: its own class, refused (exit 73) while any build slot or any core lease is held, capped at # 20 minutes (timeout; --cap-s at most 1200), holder line with the owner in _locks/quiet. Unbounded builds take quiet # shared and wait for it; bounded suites (nice 10, a 32-core band) never take it. +# lease pool --label "" [--owner ] [--min N] [--nice N] -- +# A SWEEP or any thread-bound job (main, 7 October 2026, 20:17 UK: adversarial binaries started by hand at --threads 64 to 89, +# several beside each other on a box, read load 601): takes up to FREE cores from the bounded pool (cores 8 to +# ncpu-1, the same per-core flocks the bounded builds lease, the first 8 cores reserved for the nodes and the box), never +# fewer than --min (default the smaller of and 16), waiting up to 2 h in 10 s steps with a wait- line while +# fewer are free; then runs the command under nice N (default 10) pinned to the cores taken, with "{cores}" in any +# argument replaced by the COUNT taken and "{cpuset}" by the set, and LEASE_CORES / LEASE_CPUSET exported. The sum of +# pooled threads on a box therefore never passes 88. Rule: no sweep starts except through `lease pool`. # lease status every lease, the quiet holder and every waiter with its label # lease reap a holder (lease, quiet or build slot) whose process has been STOPPED (state T) for 5 minutes or more is # killed and its file cleared, one line each in _log/reaped.log; remote-run.sh's keeper calls this every @@ -87,6 +95,43 @@ run_cores() { rm -f "$LOCKS/lease-$$"; say "released cores $set after $(( $(date +%s) - t0 )) s, exit $rc" exit $rc } +run_pool() { + local want="$1"; shift; local label="" owner="${IGNEUM_AGENT:-unknown}" nice=10 min="" reserve="${LEASE_POOL_RESERVE:-8}" + while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --nice) nice="$2"; shift 2 ;; --min) min="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done + [ "$want" -ge 1 ] 2>/dev/null || { say "pool needs a thread count"; exit 2; } + [ -n "$label" ] || { say "--label is required (the dashboard and the next lane read it)"; exit 2; } + [ $# -gt 0 ] || { say "no command"; exit 2; } + local ncpu; ncpu="${LEASE_POOL_NCPU:-$(nproc)}"; local size=$((ncpu - reserve)); [ "$want" -gt "$size" ] && want=$size + [ -n "$min" ] || min=$(( want < 16 ? want : 16 )); [ "$min" -gt "$want" ] && min=$want + local t0 waited=0 line waitfile="$LOCKS/wait-$$" fds=() cores=() c fd taken said=0 + mkdir -p "$LOCKS"; t0=$(date +%s) + line="pid $$ since $(now) waited 0 s: lease pool $want (min $min): $label; owner=$owner"; printf '%s\n' "$line" > "$waitfile" + trap 'rm -f "$waitfile" "$LOCKS/lease-$$"' EXIT + while :; do + fds=(); cores=(); taken=0 + for ((c = reserve; c < ncpu && taken < want; c++)); do + exec {fd}>>"$LOCKS/core-$c" + if flock -n "$fd"; then fds+=("$fd"); cores+=("$c"); taken=$((taken + 1)); else exec {fd}>&-; fi + done + [ "$taken" -ge "$min" ] && break + for fd in "${fds[@]}"; do exec {fd}>&-; done + waited=$(( $(date +%s) - t0 )); [ "$waited" -ge 7200 ] && { say "gave up waiting for $min free pool cores after 2 h"; exit 75; } + [ "$said" = 0 ] && { say "$taken of $min pool cores free ($(held_cores | wc -w) leased); waiting"; said=1; } + printf '%s\n' "pid $$ since $(now) waited $waited s: lease pool $want (min $min, $taken free): $label; owner=$owner" > "$waitfile" + sleep 10 + done + waited=$(( $(date +%s) - t0 )) + local set; set=$(printf '%s,' "${cores[@]}"); set=${set%,} + line="pid $$ since $(now) waited $waited s: lease pool $taken of $want cores $set: $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/lease-$$"; rm -f "$waitfile" + say "holding $taken pool cores ($set, waited $waited s): $label" + ( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$! + local args=() a; for a in "$@"; do a=${a//\{cores\}/$taken}; a=${a//\{cpuset\}/$set}; args+=("$a"); done + export LEASE_CORES="$taken" LEASE_CPUSET="$set" + if [ "${LEASE_NO_PIN:-0}" = 1 ]; then "${args[@]}"; else nice -n "$nice" taskset -c "$set" "${args[@]}"; fi; local rc=$? + pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null + rm -f "$LOCKS/lease-$$"; say "released $taken pool cores after $(( $(date +%s) - t0 )) s, exit $rc" + exit $rc +} run_quiet() { local label="" owner="${IGNEUM_AGENT:-}" cap=1200 while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --cap-s) cap="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done @@ -147,14 +192,24 @@ self_test() { # 5. a running (not stopped) holder is left alone bash "$me" cores 6 --label R -- sleep 6 2>/dev/null & for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done; sleep 0.3; touch -d '-10 seconds' "$t"/locks/lease-* [ "$(bash "$me" reap 2>/dev/null)" = 0 ] || f "a running holder was reaped"; wait - [ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept" + # 6. pool: with 12 cores (8 reserved, 4 in the pool) a pool run takes what is free, substitutes the count, and the next waits + export LEASE_POOL_NCPU=12 + bash "$me" pool 88 --min 2 --label P1 -- bash -c "echo \$LEASE_CORES > '$t/p1.n'; sleep 6; date +%s.%N > '$t/p1.end'" 2>/dev/null & for i in $(seq 1 60); do [ -s "$t/p1.n" ] && break; sleep 0.1; done + [ "$(cat "$t/p1.n" 2>/dev/null)" = 4 ] || f "a pool run did not take the 4 free pool cores (got '$(cat "$t/p1.n" 2>/dev/null)')" + bash "$me" pool 3 --min 1 --label P2 -- bash -c "date +%s.%N > '$t/p2.start'; echo {cores} > '$t/p2.n'" 2>/dev/null + python3 -c "import sys; sys.exit(0 if float(open('$t/p2.start').read()) >= float(open('$t/p1.end').read()) else 1)" || f "a pool run started while the pool was full instead of waiting" + [ "$(cat "$t/p2.n" 2>/dev/null)" = 3 ] || f "{cores} was not replaced by the count taken (got '$(cat "$t/p2.n" 2>/dev/null)')" + wait; unset LEASE_POOL_NCPU + [ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "pool lease or wait files left behind: $(ls "$t/locks")" + [ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept, a pool run takes the free pool cores and the next waits for them" return $fail } case "${1:-}" in cores) shift; run_cores "$@" ;; + pool) shift; run_pool "$@" ;; quiet) shift; run_quiet "$@" ;; status) status ;; reap) reap ;; --self-test) self_test ;; - *) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; + *) sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; esac