igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

lease: prune dead waiters at every status read and before the priority read (build-2 held seven ghost waiters for up to 2 h on 8 Oct 2026, one adv-class ghost ahead of every live release waiter, so the queue never drained)

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-08 18:06:47 +00:00
parent 873414eaac
commit fe37c51e69
1 changed files with 6 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
infra/build-server/lease.sh
View file

@ -40,6 +40,10 @@
# lease-<pid> (holder line "pid N since HH:MM:SSZ cores <set>: <label>; owner=<agent>"), quiet (holder line), wait-<pid>.
set -uo pipefail
LOCKS="${IGNEUM_BUILD_SLOTS_DIR:-/srv/builds/_locks}"; LOGS="${IGNEUM_BUILD_LOG_DIR:-/srv/builds/_log}"
# A waiter whose process is gone leaves its wait file behind (a killed gate, a lost ssh); the priority read (a higher-class waiter
# takes the next freed cores first) then blocks every live lower-class waiter for ever. build-2 held seven such ghosts on 8 Oct 2026
# (the oldest 2 h; one adv-class ghost ahead of every release waiter). Pruned at every status read and before every priority read.
prune_dead_waiters() { local f p; for f in "$LOCKS"/wait-*; do [ -e "$f" ] || continue; p=${f##*/wait-}; case "$p" in ''|*[!0-9]*) continue;; esac; kill -0 "$p" 2>/dev/null || rm -f -- "$f"; done 2>/dev/null; }
REAP_S="${LEASE_REAP_S:-300}"
now() { date -u +%H:%M:%SZ; }
say() { echo "lease: $*" >&2; }
@ -87,7 +91,7 @@ status() {
echo "leases:"; for f in "$LOCKS"/lease-*; do [ -s "$f" ] && echo " $(cat "$f")"; done 2>/dev/null
echo "quiet: $(cat "$LOCKS/quiet" 2>/dev/null)"
echo "held cores: $(held_cores)"
echo "waiting:"; for f in "$LOCKS"/wait-*; do [ -s "$f" ] && echo " $(cat "$f")"; done 2>/dev/null
prune_dead_waiters; echo "waiting:"; for f in "$LOCKS"/wait-*; do [ -s "$f" ] && echo " $(cat "$f")"; done 2>/dev/null
echo "pre-emptions (last 10 min):"; find "$LOCKS" -maxdepth 1 -name 'preempt-*' -mmin -10 -type f 2>/dev/null | while read -r f; do echo " $(cat "$f")"; done; find "$LOCKS" -maxdepth 1 -name 'preempt-*' -mmin +10 -type f -delete 2>/dev/null
}
run_cores() {
@ -151,7 +155,7 @@ run_pool() {
while :; do
fds=(); cores=(); taken=0
# a waiter yields while a HIGHER class waits (its wait line carries "class <name>/<rank>")
higher=$(cat "$LOCKS"/wait-* 2>/dev/null | grep -v "^pid $$ " | sed -n 's/.* class [a-z0-9]*\/\([0-9]\):.*/\1/p' | awk -v r="$rank" '$1 < r' | head -1)
prune_dead_waiters; higher=$(cat "$LOCKS"/wait-* 2>/dev/null | grep -v "^pid $$ " | sed -n 's/.* class [a-z0-9]*\/\([0-9]\):.*/\1/p' | awk -v r="$rank" '$1 < r' | head -1)
if [ -z "$higher" ]; then
for ((c = reserve; c < ncpu && taken < want; c++)); do
exec {fd}>>"$LOCKS/core-$c"