From caff65659338a6b42cbd773ec383c17b79c6efa7 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 13:16:39 +0000 Subject: [PATCH] Build boxes: the box probe carries its own ssh options (bash 3.2 under set -u refused the unset BS_SSH_OPTS array before bs_host built it; every unpinned route on the Mac died, the pool lane 7 Oct 2026); the router check runs the ssh path under /bin/bash Co-Authored-By: Claude Fable 5.1 --- infra/build-server/lib.sh | 4 +++- tools/ci/route-spill-check.sh | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/infra/build-server/lib.sh b/infra/build-server/lib.sh index de1e7d25c..223d89568 100755 --- a/infra/build-server/lib.sh +++ b/infra/build-server/lib.sh @@ -45,7 +45,9 @@ bs_box_state() { # -> "free= slots= load1=" | "absent" | "down" v=$(eval "printf '%s' \"\${BS_ROUTE_STATE_$b:-}\""); if [ -n "$v" ]; then printf '%s' "$v"; return 0; fi f=$(bs_box_file "$b"); [ -s "$f" ] || { printf 'absent'; return 0; } h="$(head -1 "$f" | tr -d '[:space:]')" - ssh "${BS_SSH_OPTS[@]}" -o ConnectTimeout=8 "$h" 'd=/srv/builds/_locks; n=$(cat $d/slots 2>/dev/null || echo 1); free=0; k=0; while [ $k -lt $n ]; do exec 9>>$d/build-$k; if flock -n 9; then free=$((free+1)); fi; exec 9>&-; k=$((k+1)); done; printf "free=%s slots=%s load1=%s" $free $n "$(cut -d" " -f1 /proc/loadavg)"' 2>/dev/null || printf 'down' + # the probe runs BEFORE bs_host builds BS_SSH_OPTS (the pool lane, 7 Oct 2026 15:1x UK: bash 3.2 under set -u refuses an + # unset array), so it carries its own options: the ops key, batch mode, a short connect timeout, no control socket + ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 "$h" 'd=/srv/builds/_locks; n=$(cat $d/slots 2>/dev/null || echo 1); free=0; k=0; while [ $k -lt $n ]; do exec 9>>$d/build-$k; if flock -n 9; then free=$((free+1)); fi; exec 9>&-; k=$((k+1)); done; printf "free=%s slots=%s load1=%s" $free $n "$(cut -d" " -f1 /proc/loadavg)"' 2>/dev/null || printf 'down' } bs_state_ok() { # -> 0 when the box can take a job now (a free slot, load1 at or under the line) local st="$1" free load diff --git a/tools/ci/route-spill-check.sh b/tools/ci/route-spill-check.sh index 4c8ef4f6a..0da8b3d03 100755 --- a/tools/ci/route-spill-check.sh +++ b/tools/ci/route-spill-check.sh @@ -33,5 +33,12 @@ BS_ROUTE_STATE_1="down" BS_ROUTE_STATE_2="free=3 slots=3 load BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a proving class with no box 3 prefers box 1" prove 1 0 BS_ROUTE_STATE_1="free=0 slots=2 load1=5" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a proving class spills to box 2 when box 1 is held" prove 2 1 BS_ROUTE_STATE_1="free=1 slots=2 load1=64" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "load exactly 64 is under the line" build 1 0 +# the ssh path itself under the hook's shell (/bin/bash is 3.2 on the Mac; the pool lane found `BS_SSH_OPTS[@]: unbound variable` +# at the first unpinned route, 7 Oct 2026): a host file pointing at a port nothing answers on must read "down" in a few seconds, +# not die on an unset array +printf 'build@127.0.0.1\n' > "$t/hosts" +out=$(IGNEUM_BUILD_KEY="$t/no-such-key" /bin/bash -c '. infra/build-server/lib.sh; bs_box_state 1' 2>&1 || true) +if [ "$out" = down ]; then echo "route-spill: the ssh probe under /bin/bash $(/bin/bash -c 'echo $BASH_VERSION') reads an unreachable box as down" +else echo "route-spill: the ssh probe under /bin/bash failed: '$out' (expected 'down')" >&2; fail=1; fi [ "$fail" = 0 ] && echo "route-spill: the class is a preference; a full or overloaded box hands the job to the other one" exit $fail