6f5787ed71
Main, 7 October 2026, 15:07 UK: one global exclusive measure flock across unrelated measurements stalled build-1 at load 120 with free slots (a stopped probe held it 5.5 h; an exclusive waiter queued every new shared taker) and build-2 behind a one-core VDF bench. lease.sh (installed at /srv/builds/_bin/lease by provision.sh and by hand on both boxes) takes one flock per core for a pinned measurement and the quiet file for a whole-box one; remote-run.sh takes quiet shared only for unbounded runs, excludes leased cores from its set, and its keeper refreshes the holder file and calls lease reap (a STOPPED holder of a lease, quiet or a slot for 5 minutes is killed with a line in _log/reaped.log). Keepers close the lock descriptors they inherit (an orphaned sleep held a slot and the worktree lock 20 s past the release; the slot self-test had rotted on that since the worktree lock landed). tools/ci/box-locks-check.sh runs lease.sh --self-test and remote-run.sh --self-test-slots on build-1. provision.sh also carries the 16 libraries headless Chromium needs (installed by hand on both boxes at 14:5x UK) and a headless self-test step. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
155 lines
12 KiB
Bash
Executable file
155 lines
12 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Measurement leases on a build box (main, 7 October 2026, 15:07 UK: one global exclusive "measure" flock across unrelated
|
|
# measurements stalled build-1 at load 120 with free slots, an exclusive waiter queueing every new shared taker behind it, and
|
|
# a stopped probe held the box for five and a half hours). The measure file is retired. In its place:
|
|
#
|
|
# lease cores <set> --label "<text>" [--owner <agent>] [--nice N] -- <command...>
|
|
# A measurement that pins cores takes a lease on THOSE CORES ONLY (one flock per core, _locks/core-<n>, taken in ascending
|
|
# order, waited for up to 2 h with a wait-<pid> file carrying the label), runs the command under nice N (default 10) and
|
|
# taskset on the set, and releases. Builds and suites keep off leased cores (remote-run.sh reads the core files before it
|
|
# pins its own set). Nothing else is excluded: the box stays open.
|
|
# lease quiet --label "<text>" --owner <agent> [--cap-s N] -- <command...>
|
|
# A WHOLE-BOX quiet measurement: its own class, refused (exit 73) while any build slot or any core lease is held, capped at
|
|
# 20 minutes (timeout; --cap-s at most 1200), holder line with the owner in _locks/quiet. Unbounded builds take quiet
|
|
# shared and wait for it; bounded suites (nice 10, a 32-core band) never take it.
|
|
# lease status every lease, the quiet holder and every waiter with its label
|
|
# lease reap a holder (lease, quiet or build slot) whose process has been STOPPED (state T) for 5 minutes or more is
|
|
# killed and its file cleared, one line each in _log/reaped.log; remote-run.sh's keeper calls this every
|
|
# 20 s while any run is on the box (LEASE_REAP_S overrides the 300 s for the self-test)
|
|
# lease --self-test the known cases against a scratch lock directory (in the gate: tools/ci/pre-push.sh)
|
|
#
|
|
# Installed on every box at /srv/builds/_bin/lease by provision.sh (and copied by hand on 7 October 2026); the lock directory is
|
|
# IGNEUM_BUILD_SLOTS_DIR (the profile sets /srv/builds/_locks), the log directory IGNEUM_BUILD_LOG_DIR. Files: core-<n> (flock),
|
|
# lease-<pid> (holder line "pid N since HH:MM:SSZ cores <set>: <label>; owner=<agent>"), quiet (holder line), wait-<pid>.
|
|
set -uo pipefail
|
|
LOCKS="${IGNEUM_BUILD_SLOTS_DIR:-/srv/builds/_locks}"; LOGS="${IGNEUM_BUILD_LOG_DIR:-/srv/builds/_log}"
|
|
REAP_S="${LEASE_REAP_S:-300}"
|
|
now() { date -u +%H:%M:%SZ; }
|
|
say() { echo "lease: $*" >&2; }
|
|
expand_set() { # "6-11,54-59" -> "6 7 8 9 10 11 54 ..."
|
|
local out="" part lo hi; IFS=',' read -ra parts <<<"$1"
|
|
for part in "${parts[@]}"; do case "$part" in *-*) lo=${part%-*}; hi=${part#*-} ;; *) lo=$part; hi=$part ;; esac
|
|
[ "$lo" -le "$hi" ] 2>/dev/null || { say "bad core set '$1'"; return 2; }
|
|
for ((c = lo; c <= hi; c++)); do out="$out $c"; done; done
|
|
echo "${out# }"
|
|
}
|
|
held_cores() { # the cores whose lease file is flocked right now, as a space list
|
|
local f c out=""
|
|
for f in "$LOCKS"/core-*; do [ -e "$f" ] || continue; c=${f##*/core-}
|
|
exec 8>>"$f"; if ! flock -n 8; then out="$out $c"; fi; exec 8>&-; done
|
|
echo "${out# }"
|
|
}
|
|
any_slot_held() { local n f k; n=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); for ((k = 0; k < n; k++)); do f="$LOCKS/build-$k"; [ -e "$f" ] || continue; exec 8>>"$f"; if ! flock -n 8; then exec 8>&-; return 0; fi; exec 8>&-; done; return 1; }
|
|
pid_of() { sed -n 's/^pid \([0-9]*\) .*/\1/p' "$1" 2>/dev/null | head -1; }
|
|
reap() { # stopped holders older than REAP_S are killed, their files cleared, one line each in reaped.log
|
|
local f p st age line n=0
|
|
for f in "$LOCKS"/lease-* "$LOCKS"/quiet "$LOCKS"/build-[0-9]*; do
|
|
[ -s "$f" ] || continue; p=$(pid_of "$f"); [ -n "$p" ] || continue
|
|
st=$(ps -o stat= -p "$p" 2>/dev/null | tr -d ' '); case "$st" in T*) ;; *) continue ;; esac
|
|
age=$(( $(date +%s) - $(stat -c %Y "$f") ))
|
|
# the file's mtime is refreshed by the holder's keeper every 20 s while it runs; a stopped holder's file goes stale
|
|
[ "$age" -ge "$REAP_S" ] || continue
|
|
line="$(now) reaped pid $p (stopped $age s, state $st) holding $(basename "$f"): $(head -c 160 "$f" | tr '\n' ' ')"
|
|
kill -KILL "$p" 2>/dev/null; pkill -KILL -P "$p" 2>/dev/null; : > "$f"; case "$f" in */lease-*) rm -f "$f" ;; esac
|
|
mkdir -p "$LOGS"; echo "$line" >> "$LOGS/reaped.log"; say "$line"; n=$((n + 1))
|
|
done
|
|
echo "$n"
|
|
}
|
|
status() {
|
|
echo "leases:"; for f in "$LOCKS"/lease-*; do [ -s "$f" ] && echo " $(cat "$f")"; done 2>/dev/null
|
|
echo "quiet: $(cat "$LOCKS/quiet" 2>/dev/null)"
|
|
echo "held cores: $(held_cores)"
|
|
echo "waiting:"; for f in "$LOCKS"/wait-*; do [ -s "$f" ] && echo " $(cat "$f")"; done 2>/dev/null
|
|
}
|
|
run_cores() {
|
|
local set="$1"; shift; local label="" owner="${IGNEUM_AGENT:-unknown}" nice=10
|
|
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --nice) nice="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
|
|
[ -n "$label" ] || { say "--label is required (the dashboard and the next lane read it)"; exit 2; }
|
|
[ $# -gt 0 ] || { say "no command"; exit 2; }
|
|
local cores c fd t0 waited=0 line waitfile="$LOCKS/wait-$$"
|
|
cores=$(expand_set "$set") || exit 2
|
|
mkdir -p "$LOCKS"; t0=$(date +%s)
|
|
line="pid $$ since $(now) waited 0 s: lease cores $set: $label; owner=$owner"; printf '%s\n' "$line" > "$waitfile"
|
|
trap 'rm -f "$waitfile" "$LOCKS/lease-$$"' EXIT
|
|
local fds=()
|
|
for c in $cores; do
|
|
exec {fd}>>"$LOCKS/core-$c"
|
|
if ! flock -n "$fd"; then say "core $c is leased ($(for f in "$LOCKS"/lease-*; do grep -l " core-$c\b\| cores [^:]*\b$c\b" "$f" 2>/dev/null; done | head -1 | xargs -r cat | cut -c1-120)); waiting"; flock -w 7200 "$fd" || { say "gave up waiting for core $c after 2 h"; exit 75; }; fi
|
|
fds+=("$fd")
|
|
done
|
|
waited=$(( $(date +%s) - t0 ))
|
|
line="pid $$ since $(now) waited $waited s: lease cores $set: $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/lease-$$"; rm -f "$waitfile"
|
|
say "holding cores $set (waited $waited s): $label"
|
|
# the keeper closes the lock descriptors first: an inherited flock would outlive the release in its orphaned sleep
|
|
( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$!
|
|
nice -n "$nice" taskset -c "$set" "$@"; local rc=$?
|
|
pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
|
|
rm -f "$LOCKS/lease-$$"; say "released cores $set after $(( $(date +%s) - t0 )) s, exit $rc"
|
|
exit $rc
|
|
}
|
|
run_quiet() {
|
|
local label="" owner="${IGNEUM_AGENT:-}" cap=1200
|
|
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --cap-s) cap="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
|
|
[ -n "$label" ] && [ -n "$owner" ] || { say "quiet needs --label and --owner (a named owner, the 5.5-hour rule)"; exit 2; }
|
|
[ "$cap" -le 1200 ] 2>/dev/null || cap=1200
|
|
[ $# -gt 0 ] || { say "no command"; exit 2; }
|
|
mkdir -p "$LOCKS"
|
|
if any_slot_held; then say "REFUSED: a build slot is held; a whole-box quiet measurement waits for an idle box (lease status)"; exit 73; fi
|
|
if [ -n "$(held_cores)" ]; then say "REFUSED: cores $(held_cores | tr ' ' ',') are leased; a whole-box quiet measurement waits for an idle box"; exit 73; fi
|
|
exec 7>>"$LOCKS/quiet"
|
|
if ! flock -n 7; then say "REFUSED: another quiet measurement holds the box: $(head -c 160 "$LOCKS/quiet")"; exit 73; fi
|
|
local t0 line; t0=$(date +%s)
|
|
line="pid $$ since $(now) waited 0 s: quiet (cap $cap s): $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/quiet"
|
|
trap ': > "$LOCKS/quiet"' EXIT
|
|
( exec 7>&-; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/quiet" 2>/dev/null; [ -s "$LOCKS/quiet" ] || printf '%s\n' "$line" > "$LOCKS/quiet"; sleep 20; done ) & local keeper=$!
|
|
say "holding the box quiet (cap $cap s): $label"
|
|
timeout --signal TERM --kill-after 30 "$cap" "$@"; local rc=$?
|
|
pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
|
|
: > "$LOCKS/quiet"; [ "$rc" = 124 ] && say "the quiet measurement hit its ${cap}s cap and was ended"
|
|
say "released the box after $(( $(date +%s) - t0 )) s, exit $rc"; exit $rc
|
|
}
|
|
self_test() {
|
|
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
|
|
export IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" LEASE_REAP_S=2; mkdir -p "$t/locks"; echo 2 > "$t/locks/slots"
|
|
local me="$0" fail=0
|
|
f() { echo "lease self-test: FAIL: $*"; fail=1; }
|
|
# 1. two leases on disjoint cores run together; the same core waits
|
|
bash "$me" cores 0-1 --label A -- bash -c "date +%s.%N > '$t/a.start'; sleep 2; date +%s.%N > '$t/a.end'" 2>/dev/null &
|
|
sleep 0.5; bash "$me" cores 2-3 --label B -- bash -c "date +%s.%N > '$t/b.start'" 2>/dev/null; sleep 0.2
|
|
python3 -c "import sys; sys.exit(0 if float(open('$t/b.start').read()) < float(open('$t/a.start').read()) + 1.5 else 1)" || f "a lease on other cores waited for an unrelated lease"
|
|
bash "$me" cores 1-2 --label C -- bash -c "date +%s.%N > '$t/c.start'" 2>/dev/null
|
|
python3 -c "import sys; sys.exit(0 if float(open('$t/c.start').read()) >= float(open('$t/a.end').read()) else 1)" || f "a lease sharing a core started before the holder released it"
|
|
wait
|
|
[ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "lease or wait files left behind: $(ls "$t/locks")"
|
|
# 2. quiet is refused while a slot is held, and runs on an idle box with a holder line naming the owner
|
|
( exec 9>>"$t/locks/build-0"; flock 9; echo "pid $BASHPID since x waited 0 s: fake build" > "$t/locks/build-0"; sleep 2 ) &
|
|
sleep 0.3; bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a slot was held"
|
|
wait; : > "$t/locks/build-0"
|
|
bash "$me" quiet --label Q --owner tester -- bash -c "grep -q 'owner=tester' '$t/locks/quiet'" || f "quiet did not run on an idle box with the owner in its holder line"
|
|
bash "$me" quiet --label Q --owner tester --cap-s 1 -- sleep 5; [ $? = 124 ] || f "the quiet cap did not end the command"
|
|
# 3. a quiet is refused while a core is leased
|
|
bash "$me" cores 0 --label L -- sleep 2 2>/dev/null & sleep 0.5
|
|
bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a core was leased"
|
|
wait
|
|
# 4. reap: a stopped holder older than the window is killed and its file cleared, with a line
|
|
bash "$me" cores 5 --label S -- sleep 60 2>/dev/null & local lp=$!; sleep 0.6
|
|
local hp; hp=$(sed -n 's/^pid \([0-9]*\) .*/\1/p' "$t"/locks/lease-* | head -1); kill -STOP "$hp"; sleep 2.5
|
|
touch -d '-10 seconds' "$t"/locks/lease-* 2>/dev/null
|
|
[ "$(bash "$me" reap 2>/dev/null)" = 1 ] || f "the stopped lease holder was not reaped"
|
|
grep -q "reaped pid $hp" "$t/log/reaped.log" 2>/dev/null || f "no reaped line was written"
|
|
[ -z "$(ls "$t/locks" | grep '^lease-')" ] || f "the reaped lease file remains"
|
|
kill -KILL "$lp" 2>/dev/null; wait 2>/dev/null
|
|
# 5. a running (not stopped) holder is left alone
|
|
bash "$me" cores 6 --label R -- sleep 3 2>/dev/null & sleep 0.6; touch -d '-10 seconds' "$t"/locks/lease-*
|
|
[ "$(bash "$me" reap 2>/dev/null)" = 0 ] || f "a running holder was reaped"; wait
|
|
[ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept"
|
|
return $fail
|
|
}
|
|
case "${1:-}" in
|
|
cores) shift; run_cores "$@" ;;
|
|
quiet) shift; run_quiet "$@" ;;
|
|
status) status ;;
|
|
reap) reap ;;
|
|
--self-test) self_test ;;
|
|
*) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
|
|
esac
|