272 lines
23 KiB
Bash
Executable file
272 lines
23 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Measurement leases on a build box (main, 7 October 2026, 15:07 UK: one global exclusive "measure" flock across unrelated
|
|
# measurements stalled build-1 at load 120 with free slots, an exclusive waiter queueing every new shared taker behind it, and
|
|
# a stopped probe held the box for five and a half hours). The measure file is retired. In its place:
|
|
#
|
|
# lease cores <set> --label "<text>" [--owner <agent>] [--nice N] -- <command...>
|
|
# A measurement that pins cores takes a lease on THOSE CORES ONLY (one flock per core, _locks/core-<n>, taken in ascending
|
|
# order, waited for up to 2 h with a wait-<pid> file carrying the label), runs the command under nice N (default 10) and
|
|
# taskset on the set, and releases. Builds and suites keep off leased cores (remote-run.sh reads the core files before it
|
|
# pins its own set). Nothing else is excluded: the box stays open.
|
|
# lease quiet --label "<text>" --owner <agent> [--cap-s N] -- <command...>
|
|
# A WHOLE-BOX quiet measurement: its own class, refused (exit 73) while any build slot or any core lease is held, capped at
|
|
# 20 minutes (timeout; --cap-s at most 1200), holder line with the owner in _locks/quiet. Unbounded builds take quiet
|
|
# shared and wait for it; bounded suites (nice 10, a 32-core band) never take it.
|
|
# lease pool <threads> --label "<text>" [--owner <agent>] [--min N] [--nice N] -- <command...>
|
|
# A SWEEP or any thread-bound job (main, 7 October 2026, 20:17 UK: adversarial binaries started by hand at --threads 64 to 89,
|
|
# several beside each other on a box, read load 601): takes up to <threads> FREE cores from the bounded pool (cores 8 to
|
|
# ncpu-1, the same per-core flocks the bounded builds lease, the first 8 cores reserved for the nodes and the box), never
|
|
# fewer than --min (default the smaller of <threads> and 16), waiting up to 2 h in 10 s steps with a wait-<pid> line while
|
|
# fewer are free; then runs the command under nice N (default 10) pinned to the cores taken, with "{cores}" in any
|
|
# argument replaced by the COUNT taken and "{cpuset}" by the set, and LEASE_CORES / LEASE_CPUSET exported. The sum of
|
|
# pooled threads on a box therefore never passes 88. Rule: no sweep starts except through `lease pool`.
|
|
# PRIORITY CLASSES (main, 7 Oct 2026 20:37 UK: the class v5 census sat behind eleven adversarial waiters): release (release
|
|
# builds and canaries) > v5 ("v5 gate", "v5 kit") > measure (measurement rows) > adv (adversarial sweeps). The class comes
|
|
# from --class <release|v5|measure|adv>, else from the owner (release, shipper, build-server: release; class-v5: v5;
|
|
# measure*: measure) or the label ("release", "canary", "pair": release; "v5 gate", "v5 kit": v5; "measure": measure), else
|
|
# adv. A higher-class waiter takes the next freed cores before any lower-class waiter regardless of arrival order (a lower
|
|
# class yields while a higher class waits). A lower-class holder above LEASE_PREEMPT_MIN_CORES (32) threads is pre-empted
|
|
# (SIGTERM to its command, which every sweep honours at its shard boundary; the lane re-queues it) when a higher class has
|
|
# waited LEASE_PREEMPT_S (120 s), newest holder first, one per period, logged in _log/preempt.log. The class stands in
|
|
# every wait and lease line (lease status).
|
|
# lease status every lease, the quiet holder and every waiter with its label
|
|
# lease reap a holder (lease, quiet or build slot) whose process has been STOPPED (state T) for 5 minutes or more is
|
|
# killed and its file cleared, one line each in _log/reaped.log; remote-run.sh's keeper calls this every
|
|
# 20 s while any run is on the box (LEASE_REAP_S overrides the 300 s for the self-test)
|
|
# lease --self-test the known cases against a scratch lock directory (in the gate: tools/ci/pre-push.sh)
|
|
#
|
|
# Installed on every box at /srv/builds/_bin/lease by provision.sh (and copied by hand on 7 October 2026); the lock directory is
|
|
# IGNEUM_BUILD_SLOTS_DIR (the profile sets /srv/builds/_locks), the log directory IGNEUM_BUILD_LOG_DIR. Files: core-<n> (flock),
|
|
# lease-<pid> (holder line "pid N since HH:MM:SSZ cores <set>: <label>; owner=<agent>"), quiet (holder line), wait-<pid>.
|
|
set -uo pipefail
|
|
LOCKS="${IGNEUM_BUILD_SLOTS_DIR:-/srv/builds/_locks}"; LOGS="${IGNEUM_BUILD_LOG_DIR:-/srv/builds/_log}"
|
|
REAP_S="${LEASE_REAP_S:-300}"
|
|
now() { date -u +%H:%M:%SZ; }
|
|
say() { echo "lease: $*" >&2; }
|
|
expand_set() { # "6-11,54-59" -> "6 7 8 9 10 11 54 ..."
|
|
local out="" part lo hi; IFS=',' read -ra parts <<<"$1"
|
|
for part in "${parts[@]}"; do case "$part" in *-*) lo=${part%-*}; hi=${part#*-} ;; *) lo=$part; hi=$part ;; esac
|
|
[ "$lo" -le "$hi" ] 2>/dev/null || { say "bad core set '$1'"; return 2; }
|
|
for ((c = lo; c <= hi; c++)); do out="$out $c"; done; done
|
|
echo "${out# }"
|
|
}
|
|
held_cores() { # the cores whose lease file is flocked right now, as a space list
|
|
local f c out=""
|
|
for f in "$LOCKS"/core-*; do [ -e "$f" ] || continue; c=${f##*/core-}
|
|
exec 8>>"$f"; if ! flock -n 8; then out="$out $c"; fi; exec 8>&-; done
|
|
echo "${out# }"
|
|
}
|
|
any_slot_held() { local n f k; n=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); for ((k = 0; k < n; k++)); do f="$LOCKS/build-$k"; [ -e "$f" ] || continue; exec 8>>"$f"; if ! flock -n 8; then exec 8>&-; return 0; fi; exec 8>&-; done; return 1; }
|
|
pid_of() { sed -n 's/^pid \([0-9]*\) .*/\1/p' "$1" 2>/dev/null | head -1; }
|
|
reap() { # stopped holders older than REAP_S are killed, their files cleared, one line each in reaped.log
|
|
local f p st age line n=0
|
|
for f in "$LOCKS"/lease-* "$LOCKS"/quiet "$LOCKS"/build-[0-9]*; do
|
|
[ -s "$f" ] || continue; p=$(pid_of "$f"); [ -n "$p" ] || continue
|
|
st=$(ps -o stat= -p "$p" 2>/dev/null | tr -d ' '); case "$st" in T*) ;; *) continue ;; esac
|
|
age=$(( $(date +%s) - $(stat -c %Y "$f") ))
|
|
# the file's mtime is refreshed by the holder's keeper every 20 s while it runs; a stopped holder's file goes stale
|
|
[ "$age" -ge "$REAP_S" ] || continue
|
|
line="$(now) reaped pid $p (stopped $age s, state $st) holding $(basename "$f"): $(head -c 160 "$f" | tr '\n' ' ')"
|
|
kill -KILL "$p" 2>/dev/null; pkill -KILL -P "$p" 2>/dev/null; : > "$f"; case "$f" in */lease-*) rm -f "$f" ;; esac
|
|
mkdir -p "$LOGS"; echo "$line" >> "$LOGS/reaped.log"; say "$line"; n=$((n + 1))
|
|
done
|
|
echo "$n"
|
|
}
|
|
status() {
|
|
echo "leases:"; for f in "$LOCKS"/lease-*; do [ -s "$f" ] && echo " $(cat "$f")"; done 2>/dev/null
|
|
echo "quiet: $(cat "$LOCKS/quiet" 2>/dev/null)"
|
|
echo "held cores: $(held_cores)"
|
|
echo "waiting:"; for f in "$LOCKS"/wait-*; do [ -s "$f" ] && echo " $(cat "$f")"; done 2>/dev/null
|
|
}
|
|
run_cores() {
|
|
local set="$1"; shift; local label="" owner="${IGNEUM_AGENT:-unknown}" nice=10
|
|
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --nice) nice="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
|
|
[ -n "$label" ] || { say "--label is required (the dashboard and the next lane read it)"; exit 2; }
|
|
[ $# -gt 0 ] || { say "no command"; exit 2; }
|
|
local cores c fd t0 waited=0 line waitfile="$LOCKS/wait-$$"
|
|
cores=$(expand_set "$set") || exit 2
|
|
mkdir -p "$LOCKS"; t0=$(date +%s)
|
|
line="pid $$ since $(now) waited 0 s: lease cores $set: $label; owner=$owner"; printf '%s\n' "$line" > "$waitfile"
|
|
trap 'rm -f "$waitfile" "$LOCKS/lease-$$"' EXIT
|
|
local fds=()
|
|
for c in $cores; do
|
|
exec {fd}>>"$LOCKS/core-$c"
|
|
if ! flock -n "$fd"; then say "core $c is leased ($(for f in "$LOCKS"/lease-*; do grep -l " core-$c\b\| cores [^:]*\b$c\b" "$f" 2>/dev/null; done | head -1 | xargs -r cat | cut -c1-120)); waiting"; flock -w 7200 "$fd" || { say "gave up waiting for core $c after 2 h"; exit 75; }; fi
|
|
fds+=("$fd")
|
|
done
|
|
waited=$(( $(date +%s) - t0 ))
|
|
line="pid $$ since $(now) waited $waited s: lease cores $set: $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/lease-$$"; rm -f "$waitfile"
|
|
say "holding cores $set (waited $waited s): $label"
|
|
# the keeper closes the lock descriptors first: an inherited flock would outlive the release in its orphaned sleep
|
|
( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$!
|
|
if [ "${LEASE_NO_PIN:-0}" = 1 ]; then "$@"; else nice -n "$nice" taskset -c "$set" "$@"; fi; local rc=$? # LEASE_NO_PIN: the self-test (lock semantics only; a pinned fixture on a loaded box ran seconds late)
|
|
pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
|
|
rm -f "$LOCKS/lease-$$"; say "released cores $set after $(( $(date +%s) - t0 )) s, exit $rc"
|
|
exit $rc
|
|
}
|
|
run_pool() {
|
|
local want="$1"; shift; local label="" owner="${IGNEUM_AGENT:-unknown}" nice=10 min="" reserve="${LEASE_POOL_RESERVE:-8}" class="" rank
|
|
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --nice) nice="$2"; shift 2 ;; --min) min="$2"; shift 2 ;; --class) class="$2"; shift 2 ;; --priority) class=v5; shift ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
|
|
if [ -z "$class" ]; then
|
|
case "$owner" in release|shipper|build-server) class=release ;; class-v5) class=v5 ;; measure*) class=measure ;; esac
|
|
[ -n "$class" ] || case "$label" in *release*|*canary*|*pair*) class=release ;; *"v5 gate"*|*"v5 kit"*) class=v5 ;; *measure*) class=measure ;; *) class=adv ;; esac
|
|
fi
|
|
case "$class" in release) rank=0 ;; v5) rank=1 ;; measure) rank=2 ;; adv) rank=3 ;; *) say "unknown class $class (release, v5, measure, adv)"; exit 2 ;; esac
|
|
[ "$want" -ge 1 ] 2>/dev/null || { say "pool needs a thread count"; exit 2; }
|
|
[ -n "$label" ] || { say "--label is required (the dashboard and the next lane read it)"; exit 2; }
|
|
[ $# -gt 0 ] || { say "no command"; exit 2; }
|
|
local ncpu; ncpu="${LEASE_POOL_NCPU:-$(nproc)}"; local size=$((ncpu - reserve)); [ "$want" -gt "$size" ] && want=$size
|
|
[ -n "$min" ] || min=$(( want < 16 ? want : 16 )); [ "$min" -gt "$want" ] && min=$want
|
|
local t0 waited=0 line waitfile="$LOCKS/wait-$$" fds=() cores=() c fd taken said=0
|
|
mkdir -p "$LOCKS"; t0=$(date +%s)
|
|
line="pid $$ since $(now) waited 0 s: lease pool $want (min $min) class $class/$rank: $label; owner=$owner"; printf '%s\n' "$line" > "$waitfile"
|
|
trap 'rm -f "$waitfile" "$LOCKS/lease-$$"' EXIT
|
|
local last_preempt=0 preempt_s="${LEASE_PREEMPT_S:-120}" preempt_min="${LEASE_PREEMPT_MIN_CORES:-32}" step="${LEASE_POOL_STEP_S:-10}" higher
|
|
while :; do
|
|
fds=(); cores=(); taken=0
|
|
# a waiter yields while a HIGHER class waits (its wait line carries "class <name>/<rank>")
|
|
higher=$(cat "$LOCKS"/wait-* 2>/dev/null | grep -v "^pid $$ " | sed -n 's/.* class [a-z0-9]*\/\([0-9]\):.*/\1/p' | awk -v r="$rank" '$1 < r' | head -1)
|
|
if [ -z "$higher" ]; then
|
|
for ((c = reserve; c < ncpu && taken < want; c++)); do
|
|
exec {fd}>>"$LOCKS/core-$c"
|
|
if flock -n "$fd"; then fds+=("$fd"); cores+=("$c"); taken=$((taken + 1)); else exec {fd}>&-; fi
|
|
done
|
|
[ "$taken" -ge "$min" ] && break
|
|
for fd in "${fds[@]}"; do exec {fd}>&-; done
|
|
fi
|
|
waited=$(( $(date +%s) - t0 )); [ "$waited" -ge 7200 ] && { say "gave up waiting for $min free pool cores after 2 h"; exit 75; }
|
|
[ "$said" = 0 ] && { say "$taken of $min pool cores free ($(held_cores | wc -w) leased)$([ -n "$higher" ] && echo ', a higher class waits ahead'); waiting"; said=1; }
|
|
printf '%s\n' "pid $$ since $(now) waited $waited s: lease pool $want (min $min, $taken free) class $class/$rank: $label; owner=$owner" > "$waitfile"
|
|
# pre-emption: this class has waited preempt_s with the pool short: TERM to the NEWEST lower-class holder above preempt_min cores
|
|
if [ -z "$higher" ] && [ "$waited" -ge "$preempt_s" ] && [ $(( $(date +%s) - last_preempt )) -ge "$preempt_s" ]; then
|
|
local vf vpid vrank vcores
|
|
for vf in $(ls -t "$LOCKS"/lease-* 2>/dev/null); do
|
|
vrank=$(sed -n 's/.* class [a-z0-9]*\/\([0-9]\) cmd .*/\1/p' "$vf" | head -1); [ -n "$vrank" ] || continue
|
|
vcores=$(sed -n 's/.* lease pool \([0-9]*\) of .*/\1/p' "$vf" | head -1)
|
|
[ "$vrank" -gt "$rank" ] && [ "${vcores:-0}" -gt "$preempt_min" ] || continue
|
|
vpid=$(sed -n 's/.* cmd \([0-9]*\);.*/\1/p' "$vf" | head -1)
|
|
[ -n "$vpid" ] && kill -0 "$vpid" 2>/dev/null || continue
|
|
kill -TERM "$vpid" 2>/dev/null; last_preempt=$(date +%s); mkdir -p "$LOGS"
|
|
echo "$(now) class $class waiter pid $$ ($label; owner=$owner) waited $waited s: TERM to cmd $vpid of $(head -c 200 "$vf" | tr '\n' ' ')" >> "$LOGS/preempt.log"
|
|
say "pool short for $waited s: sent TERM to the newest lower-class holder's command (pid $vpid, $vcores cores)"
|
|
break
|
|
done
|
|
fi
|
|
sleep "$step"
|
|
done
|
|
waited=$(( $(date +%s) - t0 ))
|
|
local set; set=$(printf '%s,' "${cores[@]}"); set=${set%,}
|
|
line="pid $$ since $(now) waited $waited s: lease pool $taken of $want cores $set class $class/$rank: $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/lease-$$"; rm -f "$waitfile"
|
|
say "holding $taken pool cores ($set, waited $waited s, class $class): $label"
|
|
local args=() a; for a in "$@"; do a=${a//\{cores\}/$taken}; a=${a//\{cpuset\}/$set}; args+=("$a"); done
|
|
export LEASE_CORES="$taken" LEASE_CPUSET="$set"
|
|
local cmdpid
|
|
if [ "${LEASE_NO_PIN:-0}" = 1 ]; then "${args[@]}" & else nice -n "$nice" taskset -c "$set" "${args[@]}" & fi; cmdpid=$!
|
|
# the command's pid in the holder line (a rank P waiter's TERM goes to it, never to the lease or its keeper)
|
|
line="pid $$ since $(now) waited $waited s: lease pool $taken of $want cores $set class $class/$rank cmd $cmdpid; $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/lease-$$"
|
|
( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$!
|
|
trap 'kill -TERM "$cmdpid" 2>/dev/null' TERM INT
|
|
wait "$cmdpid"; local rc=$?; trap - TERM INT
|
|
pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
|
|
rm -f "$LOCKS/lease-$$"; say "released $taken pool cores after $(( $(date +%s) - t0 )) s, exit $rc"
|
|
exit $rc
|
|
}
|
|
run_quiet() {
|
|
local label="" owner="${IGNEUM_AGENT:-}" cap=1200
|
|
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --cap-s) cap="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
|
|
[ -n "$label" ] && [ -n "$owner" ] || { say "quiet needs --label and --owner (a named owner, the 5.5-hour rule)"; exit 2; }
|
|
[ "$cap" -le 1200 ] 2>/dev/null || cap=1200
|
|
[ $# -gt 0 ] || { say "no command"; exit 2; }
|
|
mkdir -p "$LOCKS"
|
|
if any_slot_held; then say "REFUSED: a build slot is held; a whole-box quiet measurement waits for an idle box (lease status)"; exit 73; fi
|
|
if [ -n "$(held_cores)" ]; then say "REFUSED: cores $(held_cores | tr ' ' ',') are leased; a whole-box quiet measurement waits for an idle box"; exit 73; fi
|
|
exec 7>>"$LOCKS/quiet"
|
|
if ! flock -n 7; then say "REFUSED: another quiet measurement holds the box: $(head -c 160 "$LOCKS/quiet")"; exit 73; fi
|
|
local t0 line; t0=$(date +%s)
|
|
line="pid $$ since $(now) waited 0 s: quiet (cap $cap s): $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/quiet"
|
|
trap ': > "$LOCKS/quiet"' EXIT
|
|
( exec 7>&-; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/quiet" 2>/dev/null; [ -s "$LOCKS/quiet" ] || printf '%s\n' "$line" > "$LOCKS/quiet"; sleep 20; done ) & local keeper=$!
|
|
say "holding the box quiet (cap $cap s): $label"
|
|
timeout --signal TERM --kill-after 30 "$cap" "$@"; local rc=$?
|
|
pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
|
|
: > "$LOCKS/quiet"; [ "$rc" = 124 ] && say "the quiet measurement hit its ${cap}s cap and was ended"
|
|
say "released the box after $(( $(date +%s) - t0 )) s, exit $rc"; exit $rc
|
|
}
|
|
self_test() {
|
|
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
|
|
# a PRIVATE lock directory (never the live _locks) and no pinning: the test is about the locks, and it must read the same on an
|
|
# idle box and on one at load 120 (the horizon lane, 7 Oct 2026: one red on a full gate, green a minute later)
|
|
export IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" LEASE_REAP_S=2 LEASE_NO_PIN=1; mkdir -p "$t/locks"; echo 2 > "$t/locks/slots"
|
|
local me="$0" fail=0
|
|
f() { echo "lease self-test: FAIL: $*"; fail=1; }
|
|
# 1. two leases on disjoint cores run together; the same core waits
|
|
bash "$me" cores 0-1 --label A -- bash -c "date +%s.%N > '$t/a.start'; sleep 6; date +%s.%N > '$t/a.end'" 2>/dev/null &
|
|
for i in $(seq 1 50); do [ -f "$t/a.start" ] && break; sleep 0.1; done
|
|
bash "$me" cores 2-3 --label B -- bash -c "date +%s.%N > '$t/b.start'" 2>/dev/null; sleep 0.2
|
|
# B ran while A still held its cores (A sleeps 6 s): B's start is before A's end
|
|
python3 -c "import sys; sys.exit(0 if float(open('$t/b.start').read()) < float(open('$t/a.start').read()) + 5.5 else 1)" || f "a lease on other cores waited for an unrelated lease"
|
|
bash "$me" cores 1-2 --label C -- bash -c "date +%s.%N > '$t/c.start'" 2>/dev/null
|
|
python3 -c "import sys; sys.exit(0 if float(open('$t/c.start').read()) >= float(open('$t/a.end').read()) else 1)" || f "a lease sharing a core started before the holder released it"
|
|
wait
|
|
[ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "lease or wait files left behind: $(ls "$t/locks")"
|
|
# 2. quiet is refused while a slot is held, and runs on an idle box with a holder line naming the owner
|
|
( exec 9>>"$t/locks/build-0"; flock 9; echo "pid $BASHPID since x waited 0 s: fake build" > "$t/locks/build-0"; sleep 6 ) &
|
|
for i in $(seq 1 50); do [ -s "$t/locks/build-0" ] && break; sleep 0.1; done
|
|
bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a slot was held"
|
|
wait; : > "$t/locks/build-0"
|
|
bash "$me" quiet --label Q --owner tester -- bash -c "grep -q 'owner=tester' '$t/locks/quiet'" || f "quiet did not run on an idle box with the owner in its holder line"
|
|
bash "$me" quiet --label Q --owner tester --cap-s 1 -- sleep 5; [ $? = 124 ] || f "the quiet cap did not end the command"
|
|
# 3. a quiet is refused while a core is leased
|
|
bash "$me" cores 0 --label L -- sleep 6 2>/dev/null & for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done
|
|
bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a core was leased"
|
|
wait
|
|
# 4. reap: a stopped holder older than the window is killed and its file cleared, with a line
|
|
bash "$me" cores 5 --label S -- sleep 60 2>/dev/null & local lp=$!; for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done; sleep 0.3
|
|
local hp; hp=$(sed -n 's/^pid \([0-9]*\) .*/\1/p' "$t"/locks/lease-* | head -1); kill -STOP "$hp"; sleep 2.5
|
|
touch -d '-10 seconds' "$t"/locks/lease-* 2>/dev/null
|
|
[ "$(bash "$me" reap 2>/dev/null)" = 1 ] || f "the stopped lease holder was not reaped"
|
|
grep -q "reaped pid $hp" "$t/log/reaped.log" 2>/dev/null || f "no reaped line was written"
|
|
[ -z "$(ls "$t/locks" | grep '^lease-')" ] || f "the reaped lease file remains"
|
|
kill -KILL "$lp" 2>/dev/null; wait 2>/dev/null
|
|
# 5. a running (not stopped) holder is left alone
|
|
bash "$me" cores 6 --label R -- sleep 6 2>/dev/null & for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done; sleep 0.3; touch -d '-10 seconds' "$t"/locks/lease-*
|
|
[ "$(bash "$me" reap 2>/dev/null)" = 0 ] || f "a running holder was reaped"; wait
|
|
# 6. pool: with 12 cores (8 reserved, 4 in the pool) a pool run takes what is free, substitutes the count, and the next waits
|
|
export LEASE_POOL_NCPU=12
|
|
bash "$me" pool 88 --min 2 --label P1 -- bash -c "echo \$LEASE_CORES > '$t/p1.n'; sleep 6; date +%s.%N > '$t/p1.end'" 2>/dev/null & for i in $(seq 1 60); do [ -s "$t/p1.n" ] && break; sleep 0.1; done
|
|
[ "$(cat "$t/p1.n" 2>/dev/null)" = 4 ] || f "a pool run did not take the 4 free pool cores (got '$(cat "$t/p1.n" 2>/dev/null)')"
|
|
bash "$me" pool 3 --min 1 --label P2 -- bash -c "date +%s.%N > '$t/p2.start'; echo {cores} > '$t/p2.n'" 2>/dev/null
|
|
python3 -c "import sys; sys.exit(0 if float(open('$t/p2.start').read()) >= float(open('$t/p1.end').read()) else 1)" || f "a pool run started while the pool was full instead of waiting"
|
|
[ "$(cat "$t/p2.n" 2>/dev/null)" = 3 ] || f "{cores} was not replaced by the count taken (got '$(cat "$t/p2.n" 2>/dev/null)')"
|
|
wait
|
|
# 7. CLASS ORDER (known-failed first): an adv holder fills the pool; a v5 waiter and then an adv waiter queue; when the holder
|
|
# ends, the v5 waiter runs first and the adv waiter only after it, whatever the arrival order
|
|
export LEASE_PREEMPT_S=100 LEASE_POOL_STEP_S=1
|
|
bash "$me" pool 4 --min 4 --label HOLD --owner adv-test -- bash -c "date +%s > '$t/h.start'; sleep 4" 2>/dev/null & for i in $(seq 1 60); do [ -s "$t/h.start" ] && break; sleep 0.1; done; sleep 0.3
|
|
bash "$me" pool 4 --min 4 --label "v5 gate test" --owner class-v5 -- bash -c "date +%s.%N > '$t/v.start'; sleep 2; date +%s.%N > '$t/v.end'" 2>/dev/null & sleep 1.2
|
|
bash "$me" pool 4 --min 2 --label SWEEP --owner adv-test -- bash -c "date +%s.%N > '$t/s.start'" 2>/dev/null &
|
|
wait
|
|
[ -s "$t/v.start" ] && [ -s "$t/s.start" ] || f "class-order fixture did not run both waiters"
|
|
python3 -c "import sys; sys.exit(0 if float(open('$t/s.start').read()) >= float(open('$t/v.end').read()) else 1)" || f "a sweep took cores a waiting v5 gate asked for"
|
|
# 8. PRE-EMPTION: an adv holder above the core floor fills the pool; a release waiter pre-empts it (TERM to its command) after
|
|
# LEASE_PREEMPT_S and runs
|
|
export LEASE_PREEMPT_S=1 LEASE_PREEMPT_MIN_CORES=1
|
|
bash "$me" pool 4 --min 4 --label ADV --owner adv-test -- bash -c "trap 'echo termed > \"$t/adv.termed\"; exit 0' TERM; date +%s > '$t/adv.start'; sleep 30 & wait" 2>/dev/null & for i in $(seq 1 60); do [ -s "$t/adv.start" ] && break; sleep 0.1; done; sleep 0.3
|
|
bash "$me" pool 4 --min 2 --label "release canary test" --owner release -- bash -c "date +%s > '$t/p.start'" 2>/dev/null
|
|
[ -s "$t/adv.termed" ] || f "the release waiter did not pre-empt the adv holder"
|
|
[ -s "$t/p.start" ] || f "the release waiter did not run after the pre-emption"
|
|
wait; unset LEASE_PREEMPT_S LEASE_POOL_STEP_S LEASE_POOL_NCPU LEASE_PREEMPT_MIN_CORES
|
|
grep -q "class release waiter" "$t/log/preempt.log" 2>/dev/null || f "no preempt.log line"
|
|
[ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "pool lease or wait files left behind: $(ls "$t/locks")"
|
|
[ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept, a pool run takes the free pool cores and the next waits for them, a higher class is served before a lower one whatever the arrival order, a release waiter pre-empts an adv holder above the floor"
|
|
return $fail
|
|
}
|
|
case "${1:-}" in
|
|
cores) shift; run_cores "$@" ;;
|
|
pool) shift; run_pool "$@" ;;
|
|
quiet) shift; run_quiet "$@" ;;
|
|
status) status ;;
|
|
reap) reap ;;
|
|
--self-test) self_test ;;
|
|
*) sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
|
|
esac
|