diff --git a/infra/build-server/lease.sh b/infra/build-server/lease.sh index 09baf355..5138474b 100755 --- a/infra/build-server/lease.sh +++ b/infra/build-server/lease.sh @@ -82,7 +82,7 @@ run_cores() { say "holding cores $set (waited $waited s): $label" # the keeper closes the lock descriptors first: an inherited flock would outlive the release in its orphaned sleep ( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$! - nice -n "$nice" taskset -c "$set" "$@"; local rc=$? + if [ "${LEASE_NO_PIN:-0}" = 1 ]; then "$@"; else nice -n "$nice" taskset -c "$set" "$@"; fi; local rc=$? # LEASE_NO_PIN: the self-test (lock semantics only; a pinned fixture on a loaded box ran seconds late) pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null rm -f "$LOCKS/lease-$$"; say "released cores $set after $(( $(date +%s) - t0 )) s, exit $rc" exit $rc @@ -110,29 +110,34 @@ run_quiet() { } self_test() { t=$(mktemp -d); trap 'rm -rf "$t"' EXIT - export IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" LEASE_REAP_S=2; mkdir -p "$t/locks"; echo 2 > "$t/locks/slots" + # a PRIVATE lock directory (never the live _locks) and no pinning: the test is about the locks, and it must read the same on an + # idle box and on one at load 120 (the horizon lane, 7 Oct 2026: one red on a full gate, green a minute later) + export IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" LEASE_REAP_S=2 LEASE_NO_PIN=1; mkdir -p "$t/locks"; echo 2 > "$t/locks/slots" local me="$0" fail=0 f() { echo "lease self-test: FAIL: $*"; fail=1; } # 1. two leases on disjoint cores run together; the same core waits - bash "$me" cores 0-1 --label A -- bash -c "date +%s.%N > '$t/a.start'; sleep 2; date +%s.%N > '$t/a.end'" 2>/dev/null & - sleep 0.5; bash "$me" cores 2-3 --label B -- bash -c "date +%s.%N > '$t/b.start'" 2>/dev/null; sleep 0.2 - python3 -c "import sys; sys.exit(0 if float(open('$t/b.start').read()) < float(open('$t/a.start').read()) + 1.5 else 1)" || f "a lease on other cores waited for an unrelated lease" + bash "$me" cores 0-1 --label A -- bash -c "date +%s.%N > '$t/a.start'; sleep 6; date +%s.%N > '$t/a.end'" 2>/dev/null & + for i in $(seq 1 50); do [ -f "$t/a.start" ] && break; sleep 0.1; done + bash "$me" cores 2-3 --label B -- bash -c "date +%s.%N > '$t/b.start'" 2>/dev/null; sleep 0.2 + # B ran while A still held its cores (A sleeps 6 s): B's start is before A's end + python3 -c "import sys; sys.exit(0 if float(open('$t/b.start').read()) < float(open('$t/a.start').read()) + 5.5 else 1)" || f "a lease on other cores waited for an unrelated lease" bash "$me" cores 1-2 --label C -- bash -c "date +%s.%N > '$t/c.start'" 2>/dev/null python3 -c "import sys; sys.exit(0 if float(open('$t/c.start').read()) >= float(open('$t/a.end').read()) else 1)" || f "a lease sharing a core started before the holder released it" wait [ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "lease or wait files left behind: $(ls "$t/locks")" # 2. quiet is refused while a slot is held, and runs on an idle box with a holder line naming the owner - ( exec 9>>"$t/locks/build-0"; flock 9; echo "pid $BASHPID since x waited 0 s: fake build" > "$t/locks/build-0"; sleep 2 ) & - sleep 0.3; bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a slot was held" + ( exec 9>>"$t/locks/build-0"; flock 9; echo "pid $BASHPID since x waited 0 s: fake build" > "$t/locks/build-0"; sleep 6 ) & + for i in $(seq 1 50); do [ -s "$t/locks/build-0" ] && break; sleep 0.1; done + bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a slot was held" wait; : > "$t/locks/build-0" bash "$me" quiet --label Q --owner tester -- bash -c "grep -q 'owner=tester' '$t/locks/quiet'" || f "quiet did not run on an idle box with the owner in its holder line" bash "$me" quiet --label Q --owner tester --cap-s 1 -- sleep 5; [ $? = 124 ] || f "the quiet cap did not end the command" # 3. a quiet is refused while a core is leased - bash "$me" cores 0 --label L -- sleep 2 2>/dev/null & sleep 0.5 + bash "$me" cores 0 --label L -- sleep 6 2>/dev/null & for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a core was leased" wait # 4. reap: a stopped holder older than the window is killed and its file cleared, with a line - bash "$me" cores 5 --label S -- sleep 60 2>/dev/null & local lp=$!; sleep 0.6 + bash "$me" cores 5 --label S -- sleep 60 2>/dev/null & local lp=$!; for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done; sleep 0.3 local hp; hp=$(sed -n 's/^pid \([0-9]*\) .*/\1/p' "$t"/locks/lease-* | head -1); kill -STOP "$hp"; sleep 2.5 touch -d '-10 seconds' "$t"/locks/lease-* 2>/dev/null [ "$(bash "$me" reap 2>/dev/null)" = 1 ] || f "the stopped lease holder was not reaped" @@ -140,7 +145,7 @@ self_test() { [ -z "$(ls "$t/locks" | grep '^lease-')" ] || f "the reaped lease file remains" kill -KILL "$lp" 2>/dev/null; wait 2>/dev/null # 5. a running (not stopped) holder is left alone - bash "$me" cores 6 --label R -- sleep 3 2>/dev/null & sleep 0.6; touch -d '-10 seconds' "$t"/locks/lease-* + bash "$me" cores 6 --label R -- sleep 6 2>/dev/null & for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done; sleep 0.3; touch -d '-10 seconds' "$t"/locks/lease-* [ "$(bash "$me" reap 2>/dev/null)" = 0 ] || f "a running holder was reaped"; wait [ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept" return $fail diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh index 5bf2a547..88b2b4d5 100755 --- a/infra/build-server/remote-run.sh +++ b/infra/build-server/remote-run.sh @@ -178,7 +178,8 @@ fi if [ "${1:-}" = --self-test-slots ]; then me="${IGNEUM_REMOTE_RUN_UNDER_TEST:-$0}" t=$(mktemp -d); trap 'rm -rf "$t"' EXIT - mkdir -p "$t/locks" "$t/log" "$t/dir"; echo 2 > "$t/locks/slots" + # a PRIVATE lock directory, never the live _locks; a 3 s settle so two fakes that start seconds apart on a loaded box still see each other + mkdir -p "$t/locks" "$t/log" "$t/dir"; echo 2 > "$t/locks/slots"; export BR_SETTLE_S=3 fake() { # [BR_MEASURE=1]: a fake run that records its start and end epoch and the job count it was given local name="$1" secs="$2" measure="${3:-0}" IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" BR_MEASURE="$measure" BR_CORES="${BR_CORES:-0}" BR_NICE="${BR_NICE:-0}" BR_DIR="$t/dir" BR_CMD="date +%s.%N > '$t/$name.start'; echo JOBS=\${CARGO_BUILD_JOBS:-none} > '$t/$name.jobs'; sleep $secs; date +%s.%N > '$t/$name.end'" \ @@ -188,19 +189,19 @@ if [ "${1:-}" = --self-test-slots ]; then fail() { echo "self-test-slots: FAIL: $*"; exit 1; } after() { python3 -c "import sys; sys.exit(0 if float(open(sys.argv[1]).read()) >= float(open(sys.argv[2]).read()) else 1)" "$1" "$2"; } # 1. two concurrent builds: 45 jobs each - fake a 3 & fake b 3 & wait + fake a 6 & fake b 6 & wait [ "$(cat "$t/a.jobs")" = JOBS=45 ] && [ "$(cat "$t/b.jobs")" = JOBS=45 ] || fail "two concurrent builds got $(cat "$t/a.jobs" "$t/b.jobs" | tr '\n' ' ') (want JOBS=45 JOBS=45)" # 2. one build alone: 90 fake c 1 [ "$(cat "$t/c.jobs")" = JOBS=90 ] || fail "a lone build got $(cat "$t/c.jobs") (want JOBS=90)" # 3. a quiet measurement blocks an unbounded build (it starts only after the quiet ended) and lets a bounded suite run beside it - fake m 3 1 & sleep 0.5; fake d 1 & BR_CORES=1 BR_NICE=10 fake s 1 & wait + fake m 9 1 & sleep 0.5; fake d 1 & BR_CORES=1 BR_NICE=10 fake s 1 & wait # the quiet holds 9 s: longer than a slot take plus the 3 s settle after "$t/d.start" "$t/m.end" || fail "an unbounded build started while a quiet measurement held the box (build start $(cat "$t/d.start"), quiet end $(cat "$t/m.end"))" python3 -c "import sys; sys.exit(0 if float(open('$t/s.start').read()) < float(open('$t/m.end').read()) else 1)" || fail "a bounded suite waited for the quiet measurement" [ "$(cat "$t/m.jobs")" = JOBS=none ] || fail "a measurement was given a job count" grep -q 'owner=self-test' "$t/m.out" "$t/log/builds.jsonl" 2>/dev/null || fail "the quiet holder line lacks its owner" # 4. a quiet measurement is REFUSED (exit 73) while a build holds a slot or a core is leased - fake e 3 & sleep 0.5; fake n 1 1; rc=$?; wait + fake e 6 & sleep 0.5; fake n 1 1; rc=$?; wait [ "$rc" = 73 ] && [ ! -f "$t/n.start" ] || fail "a quiet measurement was not refused while a build ran (rc $rc)" ( exec 9>>"$t/locks/core-7"; flock 9; sleep 6 ) & sleep 0.5; fake o 1 1; rc=$?; wait [ "$rc" = 73 ] || fail "a quiet measurement was not refused while a core was leased (rc $rc)" @@ -382,8 +383,9 @@ else fi waited=$(( $(date +%s) - BR_T0 )) holder_line "$waited" > "$SLOTS_DIR/build-$got" - # the job count: let a build that started in the same second take its slot, then count the slots held (this one included) - sleep 1 + # the job count: let a build that started in the same second take its slot, then count the slots held (this one included); + # BR_SETTLE_S widens it for the self-test on a loaded box + sleep "${BR_SETTLE_S:-1}" held=0 for k in $(seq 0 $((slots - 1))); do if [ "$k" = "$got" ]; then held=$((held + 1)); continue; fi diff --git a/tools/ci/box-locks-check.sh b/tools/ci/box-locks-check.sh index 227695c9..a9ea41c3 100755 --- a/tools/ci/box-locks-check.sh +++ b/tools/ci/box-locks-check.sh @@ -19,9 +19,18 @@ stamp="ci-$$-$(date +%s)" scp -q "${BS_SSH_OPTS[@]}" infra/build-server/lease.sh "$BS_HOST:/tmp/lease-$stamp.sh" scp -q "${BS_SSH_OPTS[@]}" infra/build-server/remote-run.sh "$BS_HOST:/tmp/rr-$stamp.sh" rc=0 -# the two self-tests run in parallel in one ssh session (each against its own scratch lock directory); their last lines are printed -bs_ssh "set -o pipefail; (bash /tmp/lease-$stamp.sh --self-test 2>&1 | grep -E '^lease self-test'; echo lease=\${PIPESTATUS[0]} >> /tmp/locks-$stamp.rc) & (IGNEUM_REMOTE_RUN_UNDER_TEST=/tmp/rr-$stamp.sh bash /tmp/rr-$stamp.sh --self-test-slots 2>&1 | grep -E '^self-test-slots'; echo slots=\${PIPESTATUS[0]} >> /tmp/locks-$stamp.rc) & wait; cat /tmp/locks-$stamp.rc; rm -f /tmp/lease-$stamp.sh /tmp/rr-$stamp.sh /tmp/locks-$stamp.rc" | tee /tmp/box-locks-$$.out || rc=1 -grep -q '^lease=0$' /tmp/box-locks-$$.out && grep -q '^slots=0$' /tmp/box-locks-$$.out || rc=1 -rm -f /tmp/box-locks-$$.out +# one at a time (two fixtures at once on a box at load 120 ran seconds late), each retried once on a failure (a transient ssh or +# a momentary stall; the fixtures use private lock directories, never the live _locks, so the box's state is not in the result) +try_twice() { #