igneum/tools/ci/box-locks-check.sh

36 lines
2.7 KiB
Bash
Executable file

#!/usr/bin/env bash
# The box-side lock self-tests, run on igneum-build-1 over ssh (flock, taskset and GNU timeout live there, not on the Mac; the Mac
# gate carried no slot self-test since the worktree lock landed on 6 October 2026 and the slot test rotted unseen). Both scripts
# are copied as they are in this tree (never the box's installed copies) into a scratch path and run against scratch lock
# directories: infra/build-server/lease.sh --self-test (per-core leases, the quiet class, the reaper) and
# infra/build-server/remote-run.sh --self-test-slots (slots, jobs, quiet beside builds and suites, leased cores excluded).
#
# tools/ci/box-locks-check.sh # exit 1 when either self-test fails or the box cannot be reached
# tools/ci/box-locks-check.sh --ci # the same, but a runner with no box (no BUILD_HOST, no ~/.config/igneum/build-server) prints
# # "ok (no box from this runner)" and exits 0: GitHub's hosted runner cannot reach the box
# # (master went red on 7 Oct 2026 the moment this check landed); the Mac hook and the
# # self-hosted runners that carry the box file run it live
set -euo pipefail
cd "$(dirname "$0")/../.."
. infra/build-server/lib.sh
if [ "${1:-}" = --ci ] && [ -z "${BUILD_HOST:-}" ] && [ ! -s "$BS_HOST_FILE" ]; then echo "box-locks: ok (no box from this runner: no BUILD_HOST and no $BS_HOST_FILE; the Mac hook and the box runners run the lock self-tests)"; exit 0; fi
bs_host 1
stamp="ci-$$-$(date +%s)"
scp -q "${BS_SSH_OPTS[@]}" infra/build-server/lease.sh "$BS_HOST:/tmp/lease-$stamp.sh"
scp -q "${BS_SSH_OPTS[@]}" infra/build-server/remote-run.sh "$BS_HOST:/tmp/rr-$stamp.sh"
rc=0
# one at a time (two fixtures at once on a box at load 120 ran seconds late), each retried once on a failure (a transient ssh or
# a momentary stall; the fixtures use private lock directories, never the live _locks, so the box's state is not in the result)
try_twice() { # <label> <remote command>
local label="$1" cmd="$2" out i
for i in 1 2; do
if out=$(bs_ssh "$cmd" 2>&1); then echo "$out" | grep -E "^(lease self-test|self-test-slots)" | tail -1; return 0; fi
echo "box-locks: $label failed on try $i: $(echo "$out" | grep -E 'FAIL|rror' | tail -2 | tr '\n' ' ')" >&2; sleep 3
done
return 1
}
try_twice lease "bash /tmp/lease-$stamp.sh --self-test 2>&1" || rc=1
try_twice slots "IGNEUM_REMOTE_RUN_UNDER_TEST=/tmp/rr-$stamp.sh bash /tmp/rr-$stamp.sh --self-test-slots 2>&1" || rc=1
bs_ssh "rm -f /tmp/lease-$stamp.sh /tmp/rr-$stamp.sh" >/dev/null 2>&1 || true
[ "$rc" = 0 ] && echo "box-locks: the lease tool and the slot runner pass their self-tests on $BS_HOST"
exit $rc