#!/usr/bin/env bash # The spill-over router of tools/build-remote.sh (lib.sh bs_route_spill; the founder, 7 October 2026, 15:02 UK: build-1 at load 139 with a # queue of 1 h 40 min while build-2 sat at 4.5 with free slots). The class is a preference: a job goes to its class's box unless # that box has no free slot or its 1-minute load is above 80 (was 64), in which case it goes to the other box when that one qualifies, else # it queues on its own box. This check feeds the router fixed box states through BS_ROUTE_STATE_ (no ssh) and host files in a # scratch directory, and compares the chosen box and the spilled flag for the known cases: a held build-1 selects build-2, a free # build-1 selects build-1, an overloaded build-1 spills, a held build-2 sends a suite to build-1, two full boxes queue on the # class's own box, a box without a host file is never chosen, a build-1 that is down spills. # # tools/ci/route-spill-check.sh # exit 1 with the case that resolved wrongly (the check IS its self-test) set -euo pipefail cd "$(dirname "$0")/../.." t=$(mktemp -d); trap 'rm -rf "$t"' EXIT printf 'build@10.0.0.1\n' > "$t/hosts"; printf 'build@10.0.0.2\n' > "$t/hosts-2" # box 3 absent on purpose export IGNEUM_BUILD_HOST_FILE="$t/hosts" fail=0 expect() { # ; the states come from the environment local name="$1" class="$2" wbox="$3" wsp="$4" out out=$(bash -c '. infra/build-server/lib.sh; bs_route_spill "$1" >/dev/null 2>&1; printf "%s %s" "$BS_ROUTE_BOX" "$BS_ROUTE_SPILLED"' _ "$class" 2>/dev/null) if [ "$out" = "$wbox $wsp" ]; then echo "route-spill: $name: $class -> box $wbox spilled=$wsp" else echo "route-spill: $name: $class resolved to '$out', expected 'box $wbox spilled=$wsp'" >&2; fail=1; fi } BS_ROUTE_STATE_1="free=0 slots=2 load1=30" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a held build-1 selects build-2" build 2 1 BS_ROUTE_STATE_1="free=1 slots=2 load1=20" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a free build-1 selects build-1" build 1 0 BS_ROUTE_STATE_1="free=2 slots=2 load1=95" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "an overloaded build-1 spills" build 2 1 BS_ROUTE_STATE_1="free=0 slots=2 load1=30" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a gate spills like a build" gate 2 1 BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=0 slots=3 load1=10" expect "a held build-2 sends a suite to build-1" suite 1 1 BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=1 slots=3 load1=120" expect "an overloaded build-2 sends a bench away" bench 1 1 BS_ROUTE_STATE_1="free=2 slots=2 load1=5" BS_ROUTE_STATE_2="free=2 slots=3 load1=10" expect "a free build-2 keeps its suite" suite 2 0 BS_ROUTE_STATE_1="free=0 slots=2 load1=80" BS_ROUTE_STATE_2="free=0 slots=3 load1=90" expect "two full boxes queue a build on build-1" build 1 0 BS_ROUTE_STATE_1="free=0 slots=2 load1=80" BS_ROUTE_STATE_2="free=0 slots=3 load1=90" expect "two full boxes queue a suite on build-2" suite 2 0 BS_ROUTE_STATE_1="down" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a build-1 that is down spills" build 2 1 BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a proving class with no box 3 prefers box 1" prove 1 0 BS_ROUTE_STATE_1="free=0 slots=2 load1=5" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a proving class spills to box 2 when box 1 is held" prove 2 1 BS_ROUTE_STATE_1="free=1 slots=2 load1=80" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "load exactly 80 is under the line" build 1 0 BS_ROUTE_STATE_1="free=2 slots=2 load1=70" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "load 70 stays on build-1 (the line is 80)" build 1 0 # the ssh path itself under the hook's shell (/bin/bash is 3.2 on the Mac; the pool lane found `BS_SSH_OPTS[@]: unbound variable` # at the first unpinned route, 7 Oct 2026): a host file pointing at a port nothing answers on must read "down" in a few seconds, # not die on an unset array printf 'build@127.0.0.1\n' > "$t/hosts" out=$(IGNEUM_BUILD_KEY="$t/no-such-key" /bin/bash -c '. infra/build-server/lib.sh; bs_box_state 1' 2>&1 || true) if [ "$out" = down ]; then echo "route-spill: the ssh probe under /bin/bash $(/bin/bash -c 'echo $BASH_VERSION') reads an unreachable box as down" else echo "route-spill: the ssh probe under /bin/bash failed: '$out' (expected 'down')" >&2; fail=1; fi [ "$fail" = 0 ] && echo "route-spill: the class is a preference; a full or overloaded box hands the job to the other one" exit $fail