igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

Gate: the box lock self-tests read the same on an idle box and on one at load 120 (private lock directories, no pinning in the fixture, file-driven waits, a 3 s settle, a quiet that outlives the slot settle); the check runs them one at a time with one retry each (the horizon lane, 7 Oct 2026: one red on a full gate, green a minute later)

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-07 15:05:56 +00:00
parent 84f34a48c8
commit 649ea43a49
3 changed files with 36 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
infra/build-server/lease.sh
View file

@ -82,7 +82,7 @@ run_cores() {
say "holding cores $set (waited $waited s): $label" say "holding cores $set (waited $waited s): $label"
# the keeper closes the lock descriptors first: an inherited flock would outlive the release in its orphaned sleep # the keeper closes the lock descriptors first: an inherited flock would outlive the release in its orphaned sleep
( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$! ( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$!
nice -n "$nice" taskset -c "$set" "$@"; local rc=$? if [ "${LEASE_NO_PIN:-0}" = 1 ]; then "$@"; else nice -n "$nice" taskset -c "$set" "$@"; fi; local rc=$? # LEASE_NO_PIN: the self-test (lock semantics only; a pinned fixture on a loaded box ran seconds late)
pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
rm -f "$LOCKS/lease-$$"; say "released cores $set after $(( $(date +%s) - t0 )) s, exit $rc" rm -f "$LOCKS/lease-$$"; say "released cores $set after $(( $(date +%s) - t0 )) s, exit $rc"
exit $rc exit $rc
@ -110,29 +110,34 @@ run_quiet() {
} }
self_test() { self_test() {
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
export IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" LEASE_REAP_S=2; mkdir -p "$t/locks"; echo 2 > "$t/locks/slots" # a PRIVATE lock directory (never the live _locks) and no pinning: the test is about the locks, and it must read the same on an
# idle box and on one at load 120 (the horizon lane, 7 Oct 2026: one red on a full gate, green a minute later)
export IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" LEASE_REAP_S=2 LEASE_NO_PIN=1; mkdir -p "$t/locks"; echo 2 > "$t/locks/slots"
local me="$0" fail=0 local me="$0" fail=0
f() { echo "lease self-test: FAIL: $*"; fail=1; } f() { echo "lease self-test: FAIL: $*"; fail=1; }
# 1. two leases on disjoint cores run together; the same core waits # 1. two leases on disjoint cores run together; the same core waits
bash "$me" cores 0-1 --label A -- bash -c "date +%s.%N > '$t/a.start'; sleep 2; date +%s.%N > '$t/a.end'" 2>/dev/null & bash "$me" cores 0-1 --label A -- bash -c "date +%s.%N > '$t/a.start'; sleep 6; date +%s.%N > '$t/a.end'" 2>/dev/null &
sleep 0.5; bash "$me" cores 2-3 --label B -- bash -c "date +%s.%N > '$t/b.start'" 2>/dev/null; sleep 0.2 for i in $(seq 1 50); do [ -f "$t/a.start" ] && break; sleep 0.1; done
python3 -c "import sys; sys.exit(0 if float(open('$t/b.start').read()) < float(open('$t/a.start').read()) + 1.5 else 1)" || f "a lease on other cores waited for an unrelated lease" bash "$me" cores 2-3 --label B -- bash -c "date +%s.%N > '$t/b.start'" 2>/dev/null; sleep 0.2
# B ran while A still held its cores (A sleeps 6 s): B's start is before A's end
python3 -c "import sys; sys.exit(0 if float(open('$t/b.start').read()) < float(open('$t/a.start').read()) + 5.5 else 1)" || f "a lease on other cores waited for an unrelated lease"
bash "$me" cores 1-2 --label C -- bash -c "date +%s.%N > '$t/c.start'" 2>/dev/null bash "$me" cores 1-2 --label C -- bash -c "date +%s.%N > '$t/c.start'" 2>/dev/null
python3 -c "import sys; sys.exit(0 if float(open('$t/c.start').read()) >= float(open('$t/a.end').read()) else 1)" || f "a lease sharing a core started before the holder released it" python3 -c "import sys; sys.exit(0 if float(open('$t/c.start').read()) >= float(open('$t/a.end').read()) else 1)" || f "a lease sharing a core started before the holder released it"
wait wait
[ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "lease or wait files left behind: $(ls "$t/locks")" [ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "lease or wait files left behind: $(ls "$t/locks")"
# 2. quiet is refused while a slot is held, and runs on an idle box with a holder line naming the owner # 2. quiet is refused while a slot is held, and runs on an idle box with a holder line naming the owner
( exec 9>>"$t/locks/build-0"; flock 9; echo "pid $BASHPID since x waited 0 s: fake build" > "$t/locks/build-0"; sleep 2 ) & ( exec 9>>"$t/locks/build-0"; flock 9; echo "pid $BASHPID since x waited 0 s: fake build" > "$t/locks/build-0"; sleep 6 ) &
sleep 0.3; bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a slot was held" for i in $(seq 1 50); do [ -s "$t/locks/build-0" ] && break; sleep 0.1; done
bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a slot was held"
wait; : > "$t/locks/build-0" wait; : > "$t/locks/build-0"
bash "$me" quiet --label Q --owner tester -- bash -c "grep -q 'owner=tester' '$t/locks/quiet'" || f "quiet did not run on an idle box with the owner in its holder line" bash "$me" quiet --label Q --owner tester -- bash -c "grep -q 'owner=tester' '$t/locks/quiet'" || f "quiet did not run on an idle box with the owner in its holder line"
bash "$me" quiet --label Q --owner tester --cap-s 1 -- sleep 5; [ $? = 124 ] || f "the quiet cap did not end the command" bash "$me" quiet --label Q --owner tester --cap-s 1 -- sleep 5; [ $? = 124 ] || f "the quiet cap did not end the command"
# 3. a quiet is refused while a core is leased # 3. a quiet is refused while a core is leased
bash "$me" cores 0 --label L -- sleep 2 2>/dev/null & sleep 0.5 bash "$me" cores 0 --label L -- sleep 6 2>/dev/null & for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done
bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a core was leased" bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a core was leased"
wait wait
# 4. reap: a stopped holder older than the window is killed and its file cleared, with a line # 4. reap: a stopped holder older than the window is killed and its file cleared, with a line
bash "$me" cores 5 --label S -- sleep 60 2>/dev/null & local lp=$!; sleep 0.6 bash "$me" cores 5 --label S -- sleep 60 2>/dev/null & local lp=$!; for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done; sleep 0.3
local hp; hp=$(sed -n 's/^pid \([0-9]*\) .*/\1/p' "$t"/locks/lease-* | head -1); kill -STOP "$hp"; sleep 2.5 local hp; hp=$(sed -n 's/^pid \([0-9]*\) .*/\1/p' "$t"/locks/lease-* | head -1); kill -STOP "$hp"; sleep 2.5
touch -d '-10 seconds' "$t"/locks/lease-* 2>/dev/null touch -d '-10 seconds' "$t"/locks/lease-* 2>/dev/null
[ "$(bash "$me" reap 2>/dev/null)" = 1 ] || f "the stopped lease holder was not reaped" [ "$(bash "$me" reap 2>/dev/null)" = 1 ] || f "the stopped lease holder was not reaped"
@ -140,7 +145,7 @@ self_test() {
[ -z "$(ls "$t/locks" | grep '^lease-')" ] || f "the reaped lease file remains" [ -z "$(ls "$t/locks" | grep '^lease-')" ] || f "the reaped lease file remains"
kill -KILL "$lp" 2>/dev/null; wait 2>/dev/null kill -KILL "$lp" 2>/dev/null; wait 2>/dev/null
# 5. a running (not stopped) holder is left alone # 5. a running (not stopped) holder is left alone
bash "$me" cores 6 --label R -- sleep 3 2>/dev/null & sleep 0.6; touch -d '-10 seconds' "$t"/locks/lease-* bash "$me" cores 6 --label R -- sleep 6 2>/dev/null & for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done; sleep 0.3; touch -d '-10 seconds' "$t"/locks/lease-*
[ "$(bash "$me" reap 2>/dev/null)" = 0 ] || f "a running holder was reaped"; wait [ "$(bash "$me" reap 2>/dev/null)" = 0 ] || f "a running holder was reaped"; wait
[ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept" [ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept"
return $fail return $fail

14
infra/build-server/remote-run.sh
View file

@ -178,7 +178,8 @@ fi
if [ "${1:-}" = --self-test-slots ]; then if [ "${1:-}" = --self-test-slots ]; then
me="${IGNEUM_REMOTE_RUN_UNDER_TEST:-$0}" me="${IGNEUM_REMOTE_RUN_UNDER_TEST:-$0}"
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
mkdir -p "$t/locks" "$t/log" "$t/dir"; echo 2 > "$t/locks/slots" # a PRIVATE lock directory, never the live _locks; a 3 s settle so two fakes that start seconds apart on a loaded box still see each other
mkdir -p "$t/locks" "$t/log" "$t/dir"; echo 2 > "$t/locks/slots"; export BR_SETTLE_S=3
fake() { # <name> <seconds> [BR_MEASURE=1]: a fake run that records its start and end epoch and the job count it was given fake() { # <name> <seconds> [BR_MEASURE=1]: a fake run that records its start and end epoch and the job count it was given
local name="$1" secs="$2" measure="${3:-0}" local name="$1" secs="$2" measure="${3:-0}"
IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" BR_MEASURE="$measure" BR_CORES="${BR_CORES:-0}" BR_NICE="${BR_NICE:-0}" BR_DIR="$t/dir" BR_CMD="date +%s.%N > '$t/$name.start'; echo JOBS=\${CARGO_BUILD_JOBS:-none} > '$t/$name.jobs'; sleep $secs; date +%s.%N > '$t/$name.end'" \ IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" BR_MEASURE="$measure" BR_CORES="${BR_CORES:-0}" BR_NICE="${BR_NICE:-0}" BR_DIR="$t/dir" BR_CMD="date +%s.%N > '$t/$name.start'; echo JOBS=\${CARGO_BUILD_JOBS:-none} > '$t/$name.jobs'; sleep $secs; date +%s.%N > '$t/$name.end'" \
@ -188,19 +189,19 @@ if [ "${1:-}" = --self-test-slots ]; then
fail() { echo "self-test-slots: FAIL: $*"; exit 1; } fail() { echo "self-test-slots: FAIL: $*"; exit 1; }
after() { python3 -c "import sys; sys.exit(0 if float(open(sys.argv[1]).read()) >= float(open(sys.argv[2]).read()) else 1)" "$1" "$2"; } after() { python3 -c "import sys; sys.exit(0 if float(open(sys.argv[1]).read()) >= float(open(sys.argv[2]).read()) else 1)" "$1" "$2"; }
# 1. two concurrent builds: 45 jobs each # 1. two concurrent builds: 45 jobs each
fake a 3 & fake b 3 & wait fake a 6 & fake b 6 & wait
[ "$(cat "$t/a.jobs")" = JOBS=45 ] && [ "$(cat "$t/b.jobs")" = JOBS=45 ] || fail "two concurrent builds got $(cat "$t/a.jobs" "$t/b.jobs" | tr '\n' ' ') (want JOBS=45 JOBS=45)" [ "$(cat "$t/a.jobs")" = JOBS=45 ] && [ "$(cat "$t/b.jobs")" = JOBS=45 ] || fail "two concurrent builds got $(cat "$t/a.jobs" "$t/b.jobs" | tr '\n' ' ') (want JOBS=45 JOBS=45)"
# 2. one build alone: 90 # 2. one build alone: 90
fake c 1 fake c 1
[ "$(cat "$t/c.jobs")" = JOBS=90 ] || fail "a lone build got $(cat "$t/c.jobs") (want JOBS=90)" [ "$(cat "$t/c.jobs")" = JOBS=90 ] || fail "a lone build got $(cat "$t/c.jobs") (want JOBS=90)"
# 3. a quiet measurement blocks an unbounded build (it starts only after the quiet ended) and lets a bounded suite run beside it # 3. a quiet measurement blocks an unbounded build (it starts only after the quiet ended) and lets a bounded suite run beside it
fake m 3 1 & sleep 0.5; fake d 1 & BR_CORES=1 BR_NICE=10 fake s 1 & wait fake m 9 1 & sleep 0.5; fake d 1 & BR_CORES=1 BR_NICE=10 fake s 1 & wait # the quiet holds 9 s: longer than a slot take plus the 3 s settle
after "$t/d.start" "$t/m.end" || fail "an unbounded build started while a quiet measurement held the box (build start $(cat "$t/d.start"), quiet end $(cat "$t/m.end"))" after "$t/d.start" "$t/m.end" || fail "an unbounded build started while a quiet measurement held the box (build start $(cat "$t/d.start"), quiet end $(cat "$t/m.end"))"
python3 -c "import sys; sys.exit(0 if float(open('$t/s.start').read()) < float(open('$t/m.end').read()) else 1)" || fail "a bounded suite waited for the quiet measurement" python3 -c "import sys; sys.exit(0 if float(open('$t/s.start').read()) < float(open('$t/m.end').read()) else 1)" || fail "a bounded suite waited for the quiet measurement"
[ "$(cat "$t/m.jobs")" = JOBS=none ] || fail "a measurement was given a job count" [ "$(cat "$t/m.jobs")" = JOBS=none ] || fail "a measurement was given a job count"
grep -q 'owner=self-test' "$t/m.out" "$t/log/builds.jsonl" 2>/dev/null || fail "the quiet holder line lacks its owner" grep -q 'owner=self-test' "$t/m.out" "$t/log/builds.jsonl" 2>/dev/null || fail "the quiet holder line lacks its owner"
# 4. a quiet measurement is REFUSED (exit 73) while a build holds a slot or a core is leased # 4. a quiet measurement is REFUSED (exit 73) while a build holds a slot or a core is leased
fake e 3 & sleep 0.5; fake n 1 1; rc=$?; wait fake e 6 & sleep 0.5; fake n 1 1; rc=$?; wait
[ "$rc" = 73 ] && [ ! -f "$t/n.start" ] || fail "a quiet measurement was not refused while a build ran (rc $rc)" [ "$rc" = 73 ] && [ ! -f "$t/n.start" ] || fail "a quiet measurement was not refused while a build ran (rc $rc)"
( exec 9>>"$t/locks/core-7"; flock 9; sleep 6 ) & sleep 0.5; fake o 1 1; rc=$?; wait ( exec 9>>"$t/locks/core-7"; flock 9; sleep 6 ) & sleep 0.5; fake o 1 1; rc=$?; wait
[ "$rc" = 73 ] || fail "a quiet measurement was not refused while a core was leased (rc $rc)" [ "$rc" = 73 ] || fail "a quiet measurement was not refused while a core was leased (rc $rc)"
@ -382,8 +383,9 @@ else
fi fi
waited=$(( $(date +%s) - BR_T0 )) waited=$(( $(date +%s) - BR_T0 ))
holder_line "$waited" > "$SLOTS_DIR/build-$got" holder_line "$waited" > "$SLOTS_DIR/build-$got"
# the job count: let a build that started in the same second take its slot, then count the slots held (this one included) # the job count: let a build that started in the same second take its slot, then count the slots held (this one included);
sleep 1 # BR_SETTLE_S widens it for the self-test on a loaded box
sleep "${BR_SETTLE_S:-1}"
held=0 held=0
for k in $(seq 0 $((slots - 1))); do for k in $(seq 0 $((slots - 1))); do
if [ "$k" = "$got" ]; then held=$((held + 1)); continue; fi if [ "$k" = "$got" ]; then held=$((held + 1)); continue; fi

17
tools/ci/box-locks-check.sh
View file

@ -19,9 +19,18 @@ stamp="ci-$$-$(date +%s)"
scp -q "${BS_SSH_OPTS[@]}" infra/build-server/lease.sh "$BS_HOST:/tmp/lease-$stamp.sh" scp -q "${BS_SSH_OPTS[@]}" infra/build-server/lease.sh "$BS_HOST:/tmp/lease-$stamp.sh"
scp -q "${BS_SSH_OPTS[@]}" infra/build-server/remote-run.sh "$BS_HOST:/tmp/rr-$stamp.sh" scp -q "${BS_SSH_OPTS[@]}" infra/build-server/remote-run.sh "$BS_HOST:/tmp/rr-$stamp.sh"
rc=0 rc=0
# the two self-tests run in parallel in one ssh session (each against its own scratch lock directory); their last lines are printed # one at a time (two fixtures at once on a box at load 120 ran seconds late), each retried once on a failure (a transient ssh or
bs_ssh "set -o pipefail; (bash /tmp/lease-$stamp.sh --self-test 2>&1 | grep -E '^lease self-test'; echo lease=\${PIPESTATUS[0]} >> /tmp/locks-$stamp.rc) & (IGNEUM_REMOTE_RUN_UNDER_TEST=/tmp/rr-$stamp.sh bash /tmp/rr-$stamp.sh --self-test-slots 2>&1 | grep -E '^self-test-slots'; echo slots=\${PIPESTATUS[0]} >> /tmp/locks-$stamp.rc) & wait; cat /tmp/locks-$stamp.rc; rm -f /tmp/lease-$stamp.sh /tmp/rr-$stamp.sh /tmp/locks-$stamp.rc" | tee /tmp/box-locks-$$.out || rc=1 # a momentary stall; the fixtures use private lock directories, never the live _locks, so the box's state is not in the result)
grep -q '^lease=0$' /tmp/box-locks-$$.out && grep -q '^slots=0$' /tmp/box-locks-$$.out || rc=1 try_twice() { # <label> <remote command>
rm -f /tmp/box-locks-$$.out local label="$1" cmd="$2" out i
for i in 1 2; do
if out=$(bs_ssh "$cmd" 2>&1); then echo "$out" | grep -E "^(lease self-test|self-test-slots)" | tail -1; return 0; fi
echo "box-locks: $label failed on try $i: $(echo "$out" | grep -E 'FAIL|rror' | tail -2 | tr '\n' ' ')" >&2; sleep 3
done
return 1
}
try_twice lease "bash /tmp/lease-$stamp.sh --self-test 2>&1" || rc=1
try_twice slots "IGNEUM_REMOTE_RUN_UNDER_TEST=/tmp/rr-$stamp.sh bash /tmp/rr-$stamp.sh --self-test-slots 2>&1" || rc=1
bs_ssh "rm -f /tmp/lease-$stamp.sh /tmp/rr-$stamp.sh" >/dev/null 2>&1 || true
[ "$rc" = 0 ] && echo "box-locks: the lease tool and the slot runner pass their self-tests on $BS_HOST" [ "$rc" = 0 ] && echo "box-locks: the lease tool and the slot runner pass their self-tests on $BS_HOST"
exit $rc exit $rc