igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

lease pool <threads> -- cmd (main, 7 Oct 2026 20:17 BST: adversarial binaries started by hand at 64 to 89 threads, several beside each other, read load 601): a sweep takes up to its thread count of FREE cores from the same 88-core bounded pool as the builds, never fewer than --min, waits while fewer are free, runs pinned at nice 10 with {cores} and {cpuset} substituted and LEASE_CORES exported; the rule is no sweep starts except through it; self-test case added; installed on both boxes

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-07 19:23:56 +00:00
parent 8e0aa605f7
commit 8e77476a59
1 changed files with 57 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
infra/build-server/lease.sh
View file

@ -12,6 +12,14 @@
# A WHOLE-BOX quiet measurement: its own class, refused (exit 73) while any build slot or any core lease is held, capped at
# 20 minutes (timeout; --cap-s at most 1200), holder line with the owner in _locks/quiet. Unbounded builds take quiet
# shared and wait for it; bounded suites (nice 10, a 32-core band) never take it.
# lease pool <threads> --label "<text>" [--owner <agent>] [--min N] [--nice N] -- <command...>
# A SWEEP or any thread-bound job (main, 7 October 2026, 20:17 UK: adversarial binaries started by hand at --threads 64 to 89,
# several beside each other on a box, read load 601): takes up to <threads> FREE cores from the bounded pool (cores 8 to
# ncpu-1, the same per-core flocks the bounded builds lease, the first 8 cores reserved for the nodes and the box), never
# fewer than --min (default the smaller of <threads> and 16), waiting up to 2 h in 10 s steps with a wait-<pid> line while
# fewer are free; then runs the command under nice N (default 10) pinned to the cores taken, with "{cores}" in any
# argument replaced by the COUNT taken and "{cpuset}" by the set, and LEASE_CORES / LEASE_CPUSET exported. The sum of
# pooled threads on a box therefore never passes 88. Rule: no sweep starts except through `lease pool`.
# lease status every lease, the quiet holder and every waiter with its label
# lease reap a holder (lease, quiet or build slot) whose process has been STOPPED (state T) for 5 minutes or more is
# killed and its file cleared, one line each in _log/reaped.log; remote-run.sh's keeper calls this every
@ -87,6 +95,43 @@ run_cores() {
rm -f "$LOCKS/lease-$$"; say "released cores $set after $(( $(date +%s) - t0 )) s, exit $rc"
exit $rc
}
run_pool() {
local want="$1"; shift; local label="" owner="${IGNEUM_AGENT:-unknown}" nice=10 min="" reserve="${LEASE_POOL_RESERVE:-8}"
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --nice) nice="$2"; shift 2 ;; --min) min="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
[ "$want" -ge 1 ] 2>/dev/null || { say "pool needs a thread count"; exit 2; }
[ -n "$label" ] || { say "--label is required (the dashboard and the next lane read it)"; exit 2; }
[ $# -gt 0 ] || { say "no command"; exit 2; }
local ncpu; ncpu="${LEASE_POOL_NCPU:-$(nproc)}"; local size=$((ncpu - reserve)); [ "$want" -gt "$size" ] && want=$size
[ -n "$min" ] || min=$(( want < 16 ? want : 16 )); [ "$min" -gt "$want" ] && min=$want
local t0 waited=0 line waitfile="$LOCKS/wait-$$" fds=() cores=() c fd taken said=0
mkdir -p "$LOCKS"; t0=$(date +%s)
line="pid $$ since $(now) waited 0 s: lease pool $want (min $min): $label; owner=$owner"; printf '%s\n' "$line" > "$waitfile"
trap 'rm -f "$waitfile" "$LOCKS/lease-$$"' EXIT
while :; do
fds=(); cores=(); taken=0
for ((c = reserve; c < ncpu && taken < want; c++)); do
exec {fd}>>"$LOCKS/core-$c"
if flock -n "$fd"; then fds+=("$fd"); cores+=("$c"); taken=$((taken + 1)); else exec {fd}>&-; fi
done
[ "$taken" -ge "$min" ] && break
for fd in "${fds[@]}"; do exec {fd}>&-; done
waited=$(( $(date +%s) - t0 )); [ "$waited" -ge 7200 ] && { say "gave up waiting for $min free pool cores after 2 h"; exit 75; }
[ "$said" = 0 ] && { say "$taken of $min pool cores free ($(held_cores | wc -w) leased); waiting"; said=1; }
printf '%s\n' "pid $$ since $(now) waited $waited s: lease pool $want (min $min, $taken free): $label; owner=$owner" > "$waitfile"
sleep 10
done
waited=$(( $(date +%s) - t0 ))
local set; set=$(printf '%s,' "${cores[@]}"); set=${set%,}
line="pid $$ since $(now) waited $waited s: lease pool $taken of $want cores $set: $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/lease-$$"; rm -f "$waitfile"
say "holding $taken pool cores ($set, waited $waited s): $label"
( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$!
local args=() a; for a in "$@"; do a=${a//\{cores\}/$taken}; a=${a//\{cpuset\}/$set}; args+=("$a"); done
export LEASE_CORES="$taken" LEASE_CPUSET="$set"
if [ "${LEASE_NO_PIN:-0}" = 1 ]; then "${args[@]}"; else nice -n "$nice" taskset -c "$set" "${args[@]}"; fi; local rc=$?
pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
rm -f "$LOCKS/lease-$$"; say "released $taken pool cores after $(( $(date +%s) - t0 )) s, exit $rc"
exit $rc
}
run_quiet() {
local label="" owner="${IGNEUM_AGENT:-}" cap=1200
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --cap-s) cap="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
@ -147,14 +192,24 @@ self_test() {
# 5. a running (not stopped) holder is left alone
bash "$me" cores 6 --label R -- sleep 6 2>/dev/null & for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done; sleep 0.3; touch -d '-10 seconds' "$t"/locks/lease-*
[ "$(bash "$me" reap 2>/dev/null)" = 0 ] || f "a running holder was reaped"; wait
[ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept"
# 6. pool: with 12 cores (8 reserved, 4 in the pool) a pool run takes what is free, substitutes the count, and the next waits
export LEASE_POOL_NCPU=12
bash "$me" pool 88 --min 2 --label P1 -- bash -c "echo \$LEASE_CORES > '$t/p1.n'; sleep 6; date +%s.%N > '$t/p1.end'" 2>/dev/null & for i in $(seq 1 60); do [ -s "$t/p1.n" ] && break; sleep 0.1; done
[ "$(cat "$t/p1.n" 2>/dev/null)" = 4 ] || f "a pool run did not take the 4 free pool cores (got '$(cat "$t/p1.n" 2>/dev/null)')"
bash "$me" pool 3 --min 1 --label P2 -- bash -c "date +%s.%N > '$t/p2.start'; echo {cores} > '$t/p2.n'" 2>/dev/null
python3 -c "import sys; sys.exit(0 if float(open('$t/p2.start').read()) >= float(open('$t/p1.end').read()) else 1)" || f "a pool run started while the pool was full instead of waiting"
[ "$(cat "$t/p2.n" 2>/dev/null)" = 3 ] || f "{cores} was not replaced by the count taken (got '$(cat "$t/p2.n" 2>/dev/null)')"
wait; unset LEASE_POOL_NCPU
[ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "pool lease or wait files left behind: $(ls "$t/locks")"
[ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept, a pool run takes the free pool cores and the next waits for them"
return $fail
}
case "${1:-}" in
cores) shift; run_cores "$@" ;;
pool) shift; run_pool "$@" ;;
quiet) shift; run_quiet "$@" ;;
status) status ;;
reap) reap ;;
--self-test) self_test ;;
*) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
*) sed -n '2,32p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
esac