diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index d32940a3e..ee4f8d6c1 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -357,3 +357,23 @@ service. The layer tracks the repo branch `master`; until this work merges, the tree, so install writes a drop-in pinning `CAP_REPO_BRANCH=box-capacity` and removes it once `master` carries `infra/build-server/capacity/run.sh` (self-healing after the merge). `--no-start` enables without starting; `--smoke ` installs then runs one job's 10-minute smoke and prints the summary. + +## 7. Spill-over between the boxes (7 October 2026, 15:0x UK) + +Josh's reading at 15:02 UK: build-1 at load 139 / 114 / 90 with both slots held and a queue (the horizon lane waited 1 h 40 min) +while build-2 read 4.5 / 27 / 46 with both slots free. The class router (section 6) pinned each class to its box with no +spill-over. Now (lib.sh `bs_route_spill`, master from this commit): + +- The class is a PREFERENCE: a build, check or gate prefers box 1, a suite, bench or attack row box 2, a proving crate box 3. +- Before a run, the preferred box is read with one ssh (free slots of its slot count, 1-minute load). It takes the job when it + has a free slot and its load is at or under 64 (`BS_SPILL_LOAD`). Otherwise the other box (1 and 2 swap; 3 falls to 1) takes it + when THAT one qualifies; when neither does, the job queues on its own box. A box without a host file is never chosen; an + unreachable box reads as "down" and is skipped. +- The decision is the first route line of the run ("route: class suite prefers box 2; box 2 (free=0 slots=3 load1=70) is full or + over load 64: spilled to box 1 (free=1 slots=2 load1=20)"), the slot label carries "; spilled from box N", and the JSONL row + carries `"route": {"preferred", "box", "spilled", "reason"}` for the dashboard's job card. +- build-2 has THREE slots (its `slots` file reads 3 since 14:0x UK; provision.sh defaults a `-2` hostname to 3). Everything that + lands on box 2 runs at the bounded class, nice 10 on a 32-core band with -j 32, builds and gates included, so a suite beside + them keeps its number; since this commit a bounded run takes the band its slot owns (slot 0 the last 32 cores, slot 1 the 32 + below, slot 2 the 32 below that), so three bounded runs never share a core. A gate still takes its slot ahead of queued suites. +- `--box N` still pins. Self-test: tools/ci/route-spill-check.sh (thirteen cases through `BS_ROUTE_STATE_`, no ssh), in the gate. diff --git a/infra/build-server/lib.sh b/infra/build-server/lib.sh index bb754b796..70e9158ea 100755 --- a/infra/build-server/lib.sh +++ b/infra/build-server/lib.sh @@ -22,13 +22,55 @@ BS_HOST_FILE="${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" # # caller passes --box: gates, builds, checks, cross-builds, the workers, the hands and the observer stay on box 1; suites, benches and # the attack rows go to box 2; proving and aggregation CPU work, the second prover's shadow runner and the pool's fast-time NETWORK go # to box 3. A class whose box has no host file yet falls back to box 1, and the log line says so. No box mines, ever (README.md). +# Since 7 October 2026 15:xx UK the class is a preference with spill-over (bs_route_spill below): a full or overloaded box hands +# the job to the other one. bs_box_file() { case "${1:-1}" in 1) echo "$BS_HOST_FILE" ;; *) echo "${BS_HOST_FILE}-$1" ;; esac; } -bs_route() { # -> the box number, falling back to 1 +bs_route() { # -> the PREFERRED box number, falling back to 1 local want=1 case "$1" in suite|bench|attack) want=2 ;; prove|shadow|fasttime) want=3 ;; esac - if [ "$want" != 1 ] && [ ! -s "$(bs_box_file "$want")" ]; then bs_log "class $1 routes to box $want, which has no host file yet ($(bs_box_file "$want")): box 1"; want=1; fi + if [ "$want" != 1 ] && [ ! -s "$(bs_box_file "$want")" ]; then bs_log "class $1 prefers box $want, which has no host file yet ($(bs_box_file "$want")): box 1"; want=1; fi echo "$want" } +# Spill-over (Josh, 7 October 2026, 15:02 UK: build-1 at load 139 with a queue of 1 h 40 min while build-2 read 4.5 with both +# slots free). The class is a PREFERENCE, not a pin: a job goes to its class's box unless that box has no free slot or its 1-minute +# load is above BS_SPILL_LOAD (64), in which case it goes to the other box when THAT one has a free slot under the same load +# line; when neither qualifies it queues on its own box. The alternate of box 1 is box 2, of box 2 box 1, of box 3 box 1; a box +# without a host file is never chosen. The decision is one line on the Mac (bs_log) and travels to the box in BR_ROUTE_* for the +# JSONL row ("route": preferred, box, spilled, reason), so the dashboard shows it per job. A box is read with one ssh +# (bs_box_state: free slots of the slot count, load1); BS_ROUTE_STATE_ in the environment replaces the ssh for the self-test +# (tools/ci/route-spill-check.sh), "down" standing for an unreachable box. +BS_SPILL_LOAD="${BS_SPILL_LOAD:-64}" +bs_box_state() { # -> "free= slots= load1=" | "absent" | "down" + local b="$1" v f h + v=$(eval "printf '%s' \"\${BS_ROUTE_STATE_$b:-}\""); if [ -n "$v" ]; then printf '%s' "$v"; return 0; fi + f=$(bs_box_file "$b"); [ -s "$f" ] || { printf 'absent'; return 0; } + h="$(head -1 "$f" | tr -d '[:space:]')" + ssh "${BS_SSH_OPTS[@]}" -o ConnectTimeout=8 "$h" 'd=/srv/builds/_locks; n=$(cat $d/slots 2>/dev/null || echo 1); free=0; k=0; while [ $k -lt $n ]; do exec 9>>$d/build-$k; if flock -n 9; then free=$((free+1)); fi; exec 9>&-; k=$((k+1)); done; printf "free=%s slots=%s load1=%s" $free $n "$(cut -d" " -f1 /proc/loadavg)"' 2>/dev/null || printf 'down' +} +bs_state_ok() { # -> 0 when the box can take a job now (a free slot, load1 at or under the line) + local st="$1" free load + case "$st" in free=*) ;; *) return 1 ;; esac + free=${st#free=}; free=${free%% *}; load=${st##*load1=} + [ "$free" -gt 0 ] 2>/dev/null || return 1 + awk -v l="$load" -v m="$BS_SPILL_LOAD" 'BEGIN { exit !(l + 0 <= m + 0) }' +} +bs_route_spill() { # [priority] -> the box number; sets BS_ROUTE_PREF, BS_ROUTE_BOX, BS_ROUTE_SPILLED, BS_ROUTE_REASON + local class="$1" pref alt ps as + pref=$(bs_route "$class" 2>/dev/null) + case "$pref" in 1) alt=2 ;; 2) alt=1 ;; *) alt=1 ;; esac + BS_ROUTE_PREF=$pref; BS_ROUTE_BOX=$pref; BS_ROUTE_SPILLED=0 + ps=$(bs_box_state "$pref") + if bs_state_ok "$ps"; then BS_ROUTE_REASON="box $pref ($ps) takes it" + else + as=$(bs_box_state "$alt") + if bs_state_ok "$as"; then BS_ROUTE_BOX=$alt; BS_ROUTE_SPILLED=1; BS_ROUTE_REASON="box $pref ($ps) is full or over load $BS_SPILL_LOAD: spilled to box $alt ($as)" + else BS_ROUTE_REASON="box $pref ($ps) and box $alt ($as) are both full or over load $BS_SPILL_LOAD: queued on box $pref"; fi + fi + bs_log "route: class $class prefers box $pref; $BS_ROUTE_REASON" + BR_ROUTE_PREF=$BS_ROUTE_PREF BR_ROUTE_BOX=$BS_ROUTE_BOX BR_ROUTE_SPILLED=$BS_ROUTE_SPILLED BR_ROUTE_REASON=$BS_ROUTE_REASON + export BR_ROUTE_PREF BR_ROUTE_BOX BR_ROUTE_SPILLED BR_ROUTE_REASON + echo "$BS_ROUTE_BOX" +} BS_ROOT_REMOTE=/srv/builds BS_MIRROR_REPO=/srv/igneum.git BS_MIRROR_NODE=/srv/igneum-node.git @@ -296,6 +338,7 @@ bs_remote_run() { BR_BRANCH="$BS_BRANCH" BR_SHA="$BS_SHA" BR_AGENT="$agent" BR_KIND="${BR_KIND:-other}" BR_COMMAND="${BR_COMMAND:-}" \ BR_TARGET="${BR_TARGET:-}" BR_ARTEFACTS="${BR_ARTEFACTS:-}" BR_SDE="${BR_SDE:-}" BR_PAIRS_WITH="${BR_PAIRS_WITH:-}" \ BR_NICE="${BR_NICE:-0}" BR_CORES="${BR_CORES:-0}" BR_JOBS_CAP="${BR_JOBS_CAP:-0}" BR_PRIORITY="${BR_PRIORITY:-normal}" \ + BR_ROUTE_PREF="${BR_ROUTE_PREF:-}" BR_ROUTE_BOX="${BR_ROUTE_BOX:-}" BR_ROUTE_SPILLED="${BR_ROUTE_SPILLED:-0}" BR_ROUTE_REASON="${BR_ROUTE_REASON:-}" \ bash -c ' for v in BR_DIR BR_LABEL BR_CMD BR_TOOL BR_WT BR_CRATE BR_BRANCH BR_SHA BR_AGENT BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS BR_SDE BR_PAIRS_WITH BR_NICE BR_CORES BR_JOBS_CAP BR_PRIORITY; do printf "export %s=%q\n" "$v" "${!v}" diff --git a/infra/build-server/provision.sh b/infra/build-server/provision.sh index 7ad923adb..754851298 100755 --- a/infra/build-server/provision.sh +++ b/infra/build-server/provision.sh @@ -53,9 +53,10 @@ SCCACHE_GB="${SCCACHE_GB:-100}" SCCACHE_VERSION="${SCCACHE_VERSION:-}" NODE_MAJOR="${NODE_MAJOR:-22}" WORKTREES="${WORKTREES:-}" -SLOTS="${SLOTS:-2}" # 2 since main's ruling of 6 October 2026 (20:3x UK); remote-run.sh gives 90 jobs alone, 45 beside another +SLOTS_GIVEN="${SLOTS:-}"; SLOTS="${SLOTS:-2}" # 2 since main's ruling of 6 October 2026 (20:3x UK); remote-run.sh gives 90 jobs alone, 45 beside another P2P_PORTS="${P2P_PORTS:-26611 26811}" BOX_HOSTNAME="${BOX_HOSTNAME:-igneum-build-1}" +case "$BOX_HOSTNAME" in *-2) [ -n "$SLOTS_GIVEN" ] || SLOTS=3 ;; esac # build-2: three slots (main, 7 Oct 2026; the bounded runs take one 32-core band each) WORKERS_HOST="${WORKERS_HOST:-build.igneum.network}" # the dashboard feed's HTTPS name (A record in deSEC, 6 Oct 2026) SSH_PUBKEY="${SSH_PUBKEY:-}" BUILD_USER=build diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh index f3b2451e1..1a8c72fe9 100755 --- a/infra/build-server/remote-run.sh +++ b/infra/build-server/remote-run.sh @@ -255,6 +255,8 @@ d = { "source_date_epoch": num(e.get('BR_SDE')), "pairs_with": e.get('BR_PAIRS_WITH') or None, "nice": num(e.get('BR_NICE')) or 0, "cores": (num(e.get('BR_CORES')) or 0) or os.cpu_count(), "priority": e.get('BR_PRIORITY') or "normal", "class": e.get('BR_CLASS') or None, "run_log": e.get('BR_RUN_LOG') or None, + "route": ({"preferred": num(e.get('BR_ROUTE_PREF')), "box": num(e.get('BR_ROUTE_BOX')), "spilled": e.get('BR_ROUTE_SPILLED') == '1', + "reason": e.get('BR_ROUTE_REASON') or ""} if e.get('BR_ROUTE_BOX') else None), } sc = {k: num(e[v]) for k, v in (("hits", "BR_HITS"), ("misses", "BR_MISSES"), ("hits_total", "BR_HITS_T"), ("misses_total", "BR_MISSES_T"))} sc = {k: v for k, v in sc.items() if v is not None} @@ -446,8 +448,15 @@ RUN_LOG_DIR="$LOG_DIR/runs"; mkdir -p "$RUN_LOG_DIR" BR_RUN_LOG="$RUN_LOG_DIR/$BR_HOST-$BR_T0-$BR_PID.log"; export BR_RUN_LOG # the class's nice and core set apply to the command's subshell and everything it starts (renice and taskset on the subshell's own # pid, BASHPID; cores are the LAST N of the box's set, so gates and builds keep the first ones to themselves) +# (since the third slot on build-2, 7 Oct 2026: a bounded run takes the band its SLOT owns, counted from the top: slot 0 the last N +# cores, slot 1 the N below, slot 2 the N below that, so three bounded runs never share a core; a band below core 0 falls back to +# the last N) ncpu=$(nproc); cores_str="0-$((ncpu - 1))" -if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then cores_str="$((ncpu - BR_CORES))-$((ncpu - 1))"; fi +if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then + band=0; case "${got:-}" in ''|measure) ;; *) band=$got ;; esac + lo=$((ncpu - BR_CORES * (band + 1))); [ "$lo" -ge 0 ] || lo=$((ncpu - BR_CORES)) + cores_str="$lo-$((lo + BR_CORES - 1))" +fi ( [ "${BR_NICE:-0}" -gt 0 ] && renice -n "$BR_NICE" -p $BASHPID >/dev/null 2>&1; [ "$cores_str" != "0-$((ncpu - 1))" ] && taskset -cp "$cores_str" $BASHPID >/dev/null 2>&1; eval "$BR_CMD" ) > >(tee -a "$BR_RUN_LOG") 2> >(tee -a "$BR_RUN_LOG" >&2) rc=$? # the keeper stops BEFORE the bare `wait` (which flushes the two tees): a bare wait also waits for the keeper, and the keeper waits diff --git a/tools/build-remote.sh b/tools/build-remote.sh index 9618ec3d5..fc5e4ea75 100755 --- a/tools/build-remote.sh +++ b/tools/build-remote.sh @@ -121,15 +121,20 @@ if [ "$PLAN" = 1 ]; then exit 0 fi -# the box: --box N, else the route by class (suite and bench to box 2, prove to box 3, the rest to box 1; lib.sh bs_route) -if [ -z "$BOX" ]; then - case "$SCHED_CLASS:$PRIORITY" in *:gate) BOX=1 ;; suite:*|bench:*) BOX=$(bs_route "$SCHED_CLASS") ;; *) BOX=1 ;; esac -fi +# the box: --box N pins it; else the class's PREFERRED box with spill-over (lib.sh bs_route_spill, Josh 7 Oct 2026: a build or gate +# prefers box 1, a suite or bench box 2, a proving crate box 3; a preferred box with no free slot or a 1-minute load above 64 hands +# the job to the other box when that one qualifies; the decision line is printed here and lands in the JSONL row as "route") +ROUTE_CLASS="$SCHED_CLASS"; [ "$PRIORITY" = gate ] && ROUTE_CLASS=gate +if [ -z "$BOX" ]; then bs_route_spill "$ROUTE_CLASS"; BOX=$BS_ROUTE_BOX; else BR_ROUTE_PREF=$BOX BR_ROUTE_BOX=$BOX BR_ROUTE_SPILLED=0 BR_ROUTE_REASON="box $BOX by --box"; export BR_ROUTE_PREF BR_ROUTE_BOX BR_ROUTE_SPILLED BR_ROUTE_REASON; fi bs_host "$BOX" bs_context -# a proving crate routes to box 3 unless the caller chose (its builds and suites alike) -if [ "$BS_CRATE_REL" = proving/igneum-prove ] && [ -z "${BOX_GIVEN:-}" ] && [ "$PRIORITY" != gate ]; then b=$(bs_route prove); [ "$b" != "$BOX" ] && { BOX=$b; bs_host "$BOX"; }; fi -bs_log "box $BOX ($BS_HOST) for class $SCHED_CLASS, priority $PRIORITY" +# a proving crate prefers box 3 unless the caller chose (its builds and suites alike; the same spill-over) +if [ "$BS_CRATE_REL" = proving/igneum-prove ] && [ -z "${BOX_GIVEN:-}" ] && [ "$PRIORITY" != gate ]; then bs_route_spill prove; [ "$BS_ROUTE_BOX" != "$BOX" ] && { BOX=$BS_ROUTE_BOX; bs_host "$BOX"; }; fi +# box 2 keeps the suites' numbers: everything that lands there runs at the bounded class (nice 10, a 32-core band, -j 32), builds +# and gates included (main, 7 Oct 2026); a gate still takes its slot ahead of queued suites +if [ "$BOX" = 2 ] && [ "$BR_CORES" = 0 ]; then BR_NICE=10; BR_CORES=32; BR_JOBS_CAP=32; export BR_NICE BR_CORES BR_JOBS_CAP; bs_log "bounded on box 2 (nice 10, a 32-core band, -j 32) so a suite beside it keeps its number"; fi +if [ "$BR_JOBS_CAP" -gt 0 ] && [ -n "$JOBS" ] && [ "$JOBS" -gt "$BR_JOBS_CAP" ]; then bs_log "--jobs $JOBS clamped to $BR_JOBS_CAP on box $BOX"; JOBS=$BR_JOBS_CAP; fi +bs_log "box $BOX ($BS_HOST) for class $SCHED_CLASS, priority $PRIORITY$( [ "${BR_ROUTE_SPILLED:-0}" = 1 ] && echo ", SPILLED from box $BR_ROUTE_PREF")" if [ "$SELFTEST" = 1 ]; then [ "$BS_KIND" = node ] || bs_die "--self-test-repro runs from a fork worktree (igneum-miner and kaspad live there)" @@ -223,6 +228,7 @@ BR_KIND=$(bs_kind build-remote "$( [ "${CARGO_ARGS[0]}" = zigbuild ] && echo bui [ "$SCHED_CLASS" = bench ] && BR_KIND=bench [ "$PRIORITY" = gate ] && BR_KIND=gate label="$label; kind=$BR_KIND nice=$BR_NICE cores=$( [ "$BR_CORES" = 0 ] && echo 96 || echo "$BR_CORES")" +[ "${BR_ROUTE_SPILLED:-0}" = 1 ] && label="$label; spilled from box $BR_ROUTE_PREF" for ((i = 0; i < ${#CARGO_ARGS[@]}; i++)); do [ "${CARGO_ARGS[$i]}" = --target ] && BR_TARGET="${CARGO_ARGS[$((i + 1))]:-}"; done BR_ARTEFACTS=""; [ "$FETCH" = 1 ] && BR_ARTEFACTS="$ARTEFACTS" export BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh index 659e98396..58ab6ce7c 100755 --- a/tools/ci/pre-push.sh +++ b/tools/ci/pre-push.sh @@ -77,6 +77,7 @@ tree_checks() { run "the remote checkout's clean spares a lane's scratch (.igneum-scratch-spare, the fixed prefixes, never -x; the lost-scratch class)" bash -c 'bash tools/ci/scratch-spare-check.sh --self-test && bash tools/ci/scratch-spare-check.sh' run "long-running tools keep their body in one parsed block (the edited-while-running class)" bash -c 'bash tools/ci/whole-body-check.sh --self-test && bash tools/ci/whole-body-check.sh' run "build-remote without a priority flag bounds suites and benches (nice 10, 32 cores); a gate runs unbounded" bash tools/ci/build-kind-default-check.sh + run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh' run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh' run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test diff --git a/tools/ci/route-spill-check.sh b/tools/ci/route-spill-check.sh new file mode 100755 index 000000000..480b5d217 --- /dev/null +++ b/tools/ci/route-spill-check.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# The spill-over router of tools/build-remote.sh (lib.sh bs_route_spill; Josh, 7 October 2026, 15:02 UK: build-1 at load 139 with a +# queue of 1 h 40 min while build-2 sat at 4.5 with free slots). The class is a preference: a job goes to its class's box unless +# that box has no free slot or its 1-minute load is above 64, in which case it goes to the other box when that one qualifies, else +# it queues on its own box. This check feeds the router fixed box states through BS_ROUTE_STATE_ (no ssh) and host files in a +# scratch directory, and compares the chosen box and the spilled flag for the known cases: a held build-1 selects build-2, a free +# build-1 selects build-1, an overloaded build-1 spills, a held build-2 sends a suite to build-1, two full boxes queue on the +# class's own box, a box without a host file is never chosen, a build-1 that is down spills. +# +# tools/ci/route-spill-check.sh # exit 1 with the case that resolved wrongly (the check IS its self-test) +set -euo pipefail +cd "$(dirname "$0")/../.." +t=$(mktemp -d); trap 'rm -rf "$t"' EXIT +printf 'build@10.0.0.1\n' > "$t/hosts"; printf 'build@10.0.0.2\n' > "$t/hosts-2" # box 3 absent on purpose +export IGNEUM_BUILD_HOST_FILE="$t/hosts" +fail=0 +expect() { # ; the states come from the environment + local name="$1" class="$2" wbox="$3" wsp="$4" out + out=$(bash -c '. infra/build-server/lib.sh; bs_route_spill "$1" >/dev/null 2>&1; printf "%s %s" "$BS_ROUTE_BOX" "$BS_ROUTE_SPILLED"' _ "$class" 2>/dev/null) + if [ "$out" = "$wbox $wsp" ]; then echo "route-spill: $name: $class -> box $wbox spilled=$wsp" + else echo "route-spill: $name: $class resolved to '$out', expected 'box $wbox spilled=$wsp'" >&2; fail=1; fi +} +BS_ROUTE_STATE_1="free=0 slots=2 load1=30" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a held build-1 selects build-2" build 2 1 +BS_ROUTE_STATE_1="free=1 slots=2 load1=20" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a free build-1 selects build-1" build 1 0 +BS_ROUTE_STATE_1="free=2 slots=2 load1=70" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "an overloaded build-1 spills" build 2 1 +BS_ROUTE_STATE_1="free=0 slots=2 load1=30" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a gate spills like a build" gate 2 1 +BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=0 slots=3 load1=10" expect "a held build-2 sends a suite to build-1" suite 1 1 +BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=1 slots=3 load1=90" expect "an overloaded build-2 sends a bench away" bench 1 1 +BS_ROUTE_STATE_1="free=2 slots=2 load1=5" BS_ROUTE_STATE_2="free=2 slots=3 load1=10" expect "a free build-2 keeps its suite" suite 2 0 +BS_ROUTE_STATE_1="free=0 slots=2 load1=80" BS_ROUTE_STATE_2="free=0 slots=3 load1=90" expect "two full boxes queue a build on build-1" build 1 0 +BS_ROUTE_STATE_1="free=0 slots=2 load1=80" BS_ROUTE_STATE_2="free=0 slots=3 load1=90" expect "two full boxes queue a suite on build-2" suite 2 0 +BS_ROUTE_STATE_1="down" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a build-1 that is down spills" build 2 1 +BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a proving class with no box 3 prefers box 1" prove 1 0 +BS_ROUTE_STATE_1="free=0 slots=2 load1=5" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a proving class spills to box 2 when box 1 is held" prove 2 1 +BS_ROUTE_STATE_1="free=1 slots=2 load1=64" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "load exactly 64 is under the line" build 1 0 +[ "$fail" = 0 ] && echo "route-spill: the class is a preference; a full or overloaded box hands the job to the other one" +exit $fail