diff --git a/infra/build-server/lib.sh b/infra/build-server/lib.sh index ef05dffb..04bf429c 100755 --- a/infra/build-server/lib.sh +++ b/infra/build-server/lib.sh @@ -281,8 +281,9 @@ bs_remote_run() { BR_DIR="$dir" BR_LABEL="$label" BR_CMD="$cmd" BR_TOOL="${BS_TOOL:-build-remote}" BR_WT="$BS_WT" BR_CRATE="$BS_CRATE_REL" \ BR_BRANCH="$BS_BRANCH" BR_SHA="$BS_SHA" BR_AGENT="$agent" BR_KIND="${BR_KIND:-other}" BR_COMMAND="${BR_COMMAND:-}" \ BR_TARGET="${BR_TARGET:-}" BR_ARTEFACTS="${BR_ARTEFACTS:-}" BR_SDE="${BR_SDE:-}" BR_PAIRS_WITH="${BR_PAIRS_WITH:-}" \ + BR_NICE="${BR_NICE:-0}" BR_CORES="${BR_CORES:-0}" BR_JOBS_CAP="${BR_JOBS_CAP:-0}" BR_PRIORITY="${BR_PRIORITY:-normal}" \ bash -c ' - for v in BR_DIR BR_LABEL BR_CMD BR_TOOL BR_WT BR_CRATE BR_BRANCH BR_SHA BR_AGENT BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS BR_SDE BR_PAIRS_WITH; do + for v in BR_DIR BR_LABEL BR_CMD BR_TOOL BR_WT BR_CRATE BR_BRANCH BR_SHA BR_AGENT BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS BR_SDE BR_PAIRS_WITH BR_NICE BR_CORES BR_JOBS_CAP BR_PRIORITY; do printf "export %s=%q\n" "$v" "${!v}" done cat "$0"' "$(dirname "${BASH_SOURCE[0]}")/remote-run.sh" | bs_ssh 'bash -s' diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh index 1948858b..0190a31a 100755 --- a/infra/build-server/remote-run.sh +++ b/infra/build-server/remote-run.sh @@ -238,6 +238,7 @@ d = { "start": iso(e['BR_START']), "end": iso(e['BR_END']), "secs": num(e['BR_SECS']), "exit": num(e['BR_EXIT']), "compiles": num(e['BR_COMPILES']), "jobs": num(e.get('BR_JOBS')), "measure": (e.get('BR_MEASURE') == '1') or None, "source_date_epoch": num(e.get('BR_SDE')), "pairs_with": e.get('BR_PAIRS_WITH') or None, + "nice": num(e.get('BR_NICE')) or 0, "cores": (num(e.get('BR_CORES')) or 0) or os.cpu_count(), "priority": e.get('BR_PRIORITY') or "normal", "class": e.get('BR_CLASS') or None, "run_log": e.get('BR_RUN_LOG') or None, } sc = {k: num(e[v]) for k, v in (("hits", "BR_HITS"), ("misses", "BR_MISSES"), ("hits_total", "BR_HITS_T"), ("misses_total", "BR_MISSES_T"))} @@ -321,6 +322,18 @@ else flock -s -w 7200 "$mfd" || give_up "the measurement to end" rm -f "$waitfile"; trap - EXIT fi + # scheduling (main, 7 Oct 2026): a gate announces itself (gate-pending-) and takes the next free slot; a suite, bench or + # other non-gate run that has not taken a slot yet yields while any gate-pending marker younger than 15 min exists + gatefile="" + if [ "${BR_PRIORITY:-normal}" = gate ]; then gatefile="$SLOTS_DIR/gate-pending-$BR_PID"; holder_line 0 > "$gatefile"; trap 'rm -f "$gatefile"' EXIT + else + yt0=$(date +%s) + while pending=$(find "$SLOTS_DIR" -maxdepth 1 -name 'gate-pending-*' -mmin -15 2>/dev/null | head -1) && [ -n "$pending" ]; do + [ -f "$waitfile" ] || { echo "build-remote: a gate is queued ($(head -c 120 "$pending")), this ${BR_KIND:-run} yields" >&2; holder_line 0 > "$waitfile"; trap 'rm -f "$waitfile"' EXIT; } + [ $(( $(date +%s) - yt0 )) -lt 7200 ] || give_up "the queued gate" + sleep 5 + done + fi got="" for k in $(seq 0 $((slots - 1))); do exec {fd}>>"$SLOTS_DIR/build-$k" @@ -349,8 +362,11 @@ else exec {tfd}>&- done if [ "$held" -gt 1 ]; then BR_JOBS=$JOBS_SHARED; else BR_JOBS=$JOBS_ALONE; fi + # the bounded classes cap the jobs (suite and bench: 32 unless the caller passed --priority gate) + if [ "${BR_JOBS_CAP:-0}" -gt 0 ] && [ "$BR_JOBS" -gt "$BR_JOBS_CAP" ]; then BR_JOBS=$BR_JOBS_CAP; fi export CARGO_BUILD_JOBS="$BR_JOBS" BR_JOBS - echo "build-remote: holding build-$got on $BR_HOST (waited $waited s; $held of $slots slots held, CARGO_BUILD_JOBS=$BR_JOBS)" >&2 + [ -n "$gatefile" ] && { rm -f "$gatefile"; trap - EXIT; } + echo "build-remote: holding build-$got on $BR_HOST (waited $waited s; $held of $slots slots held, CARGO_BUILD_JOBS=$BR_JOBS, kind ${BR_KIND:-other}, nice ${BR_NICE:-0}, cores $( [ "${BR_CORES:-0}" = 0 ] && nproc || echo "$BR_CORES"))" >&2 fi release_slot() { if [ "$got" = measure ]; then : > "$SLOTS_DIR/measure"; else : > "$SLOTS_DIR/build-$got"; fi; } @@ -403,7 +419,11 @@ t1=$(date +%s) # streams stay where they were for the Mac (stdout to stdout, stderr to stderr), each teed into the run log RUN_LOG_DIR="$LOG_DIR/runs"; mkdir -p "$RUN_LOG_DIR" BR_RUN_LOG="$RUN_LOG_DIR/$BR_HOST-$BR_T0-$BR_PID.log"; export BR_RUN_LOG -( eval "$BR_CMD" ) > >(tee -a "$BR_RUN_LOG") 2> >(tee -a "$BR_RUN_LOG" >&2) +# the class's nice and core set apply to the command's subshell and everything it starts (renice and taskset on the subshell's own +# pid, BASHPID; cores are the LAST N of the box's set, so gates and builds keep the first ones to themselves) +ncpu=$(nproc); cores_str="0-$((ncpu - 1))" +if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then cores_str="$((ncpu - BR_CORES))-$((ncpu - 1))"; fi +( [ "${BR_NICE:-0}" -gt 0 ] && renice -n "$BR_NICE" -p $BASHPID >/dev/null 2>&1; [ "$cores_str" != "0-$((ncpu - 1))" ] && taskset -cp "$cores_str" $BASHPID >/dev/null 2>&1; eval "$BR_CMD" ) > >(tee -a "$BR_RUN_LOG") 2> >(tee -a "$BR_RUN_LOG" >&2) rc=$? wait t2=$(date +%s); secs=$(( t2 - t1 )) @@ -426,8 +446,8 @@ elif [[ "$BR_CMD" == cargo\ test* ]] && { [[ "$BR_CMD" == *" -- "* ]] || grep -q echo "build-remote: REFUSED after the run: the test filter matched no test in any binary (every 'running 0 tests'); check the name" >&2 fi export BR_CLASS -printf 'build-remote: RESULT rc=%s secs=%s compiles=%s sccache_hits=%s sccache_misses=%s sccache_hits_total=%s sccache_misses_total=%s jobs=%s load=%s class=%s\n' \ - "$rc" "$secs" "$compiles" "$hits" "$misses" "${hits_after:-?}" "${misses_after:-?}" "${BR_JOBS:-measure}" "$(cut -d' ' -f1-3 /proc/loadavg)" "${BR_CLASS:-ok}" +printf 'build-remote: RESULT rc=%s secs=%s compiles=%s sccache_hits=%s sccache_misses=%s sccache_hits_total=%s sccache_misses_total=%s jobs=%s nice=%s cores=%s load=%s class=%s\n' \ + "$rc" "$secs" "$compiles" "$hits" "$misses" "${hits_after:-?}" "${misses_after:-?}" "${BR_JOBS:-measure}" "${BR_NICE:-0}" "$( [ "${BR_CORES:-0}" = 0 ] && nproc || echo "$BR_CORES")" "$(cut -d' ' -f1-3 /proc/loadavg)" "${BR_CLASS:-ok}" jsonlog "$rc" "$([ "$got" = measure ] && echo "" || echo "$got")" "$waited" "$t1" "$t2" "$secs" "$compiles" "$hits" "$misses" "${hits_after:-}" "${misses_after:-}" [ "$rc" = 0 ] || redlog "$rc" "$secs" "$BR_CLASS" release_slot diff --git a/tools/build-remote.sh b/tools/build-remote.sh index c8e72f1f..75d6b004 100755 --- a/tools/build-remote.sh +++ b/tools/build-remote.sh @@ -24,6 +24,18 @@ # A plain build is native glibc 2.39: the box, the fleet's # Ubuntu 24.04 hosts, never a seed or a rig (7 Oct 2026: # a seed took 14 restarts on a 2.39 binary). +# tools/build-remote.sh --priority gate -- test ... a RELEASE GATE (the app gate, the canary cut, the pre-push +# self-tests): nice 0, the full core set, a slot ahead of +# queued suites and benches +# tools/build-remote.sh --plan [--priority gate] -- print the resolved class (kind, nice, cores, jobs) and stop; +# no box, no crate needed (the CI check uses it) +# +# Scheduling (main, 7 October 2026, after a load of 190 on 96 threads: a release join bench and the 0.3.19 app gate starving each +# other and "builds" of 16 minutes): every SUITE (cargo test) and BENCH (cargo bench) runs under nice 10 on the last 32 cores with +# -j 32 unless the caller passes --priority gate; builds keep the box's own jobs rule (90 alone, 45 beside another slot holder); a +# gate runs at nice 0 on the full set and takes a slot ahead of queued suites and benches (remote-run.sh gate-pending marker). The +# class travels in the slot label ("; kind=suite nice=10 cores=32") and the JSONL line, so the dashboard's job card shows why a +# job is slow. A call without a priority flag defaults to the bounded class for suites and benches: tools/ci/build-kind-default-check.sh. # tools/build-remote.sh --self-test-repro [--full] from a fork worktree: igneum-miner built twice a minute # apart without sccache into one target dir must give one # sha256, and a per-run target path must not (prost's @@ -63,7 +75,7 @@ BS_TOOL=build-remote # flight cannot reach the running copy (7 Oct 2026: build-remote.sh was edited mid-run and died on shifted bytes after a 4-min build) # JOBS empty = the box decides: 90 alone, 45 beside another slot holder (remote-run.sh, main's ruling 6 Oct 2026) -JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0; SHIP=0; SHIP_CLASS="${SHIP_CLASS:-seed}"; GLIBC="${GLIBC:-}" +JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0; SHIP=0; SHIP_CLASS="${SHIP_CLASS:-seed}"; PRIORITY="${PRIORITY:-normal}"; PLAN=0; GLIBC="${GLIBC:-}" while [ $# -gt 0 ]; do case "$1" in --jobs) JOBS="$2"; shift 2 ;; @@ -73,6 +85,9 @@ while [ $# -gt 0 ]; do --no-fetch) FETCH=0; shift ;; --self-test-repro) SELFTEST=1; shift ;; --ship) SHIP=1; case "${2:-}" in hive|rig|seed|linux|native) SHIP_CLASS="$2"; shift 2 ;; *) shift ;; esac ;; + --priority) PRIORITY="$2"; shift 2 ;; + --gate) PRIORITY=gate; shift ;; + --plan) PLAN=1; shift ;; --glibc) GLIBC="$2"; shift 2 ;; --full) FULL=1; shift ;; --) shift; CARGO_ARGS=("$@"); break ;; @@ -83,6 +98,25 @@ done [ "${CARGO_ARGS[0]:-}" = cargo ] && CARGO_ARGS=("${CARGO_ARGS[@]:1}") CARGO_ARGS_GIVEN=""; [ -n "${CARGO_ARGS[*]:-}" ] && CARGO_ARGS_GIVEN=1 +# the scheduling class, from the cargo subcommand and the priority flag alone (no box, no crate): BR_NICE, BR_CORES (the last N of the +# box's 96; 0 = the full set), BR_JOBS_CAP (0 = the box's rule), BR_PRIORITY; the kind for the log is refined after bs_context +BR_NICE=0; BR_CORES=0; BR_JOBS_CAP=0; BR_PRIORITY="$PRIORITY"; SCHED_CLASS=build +case "$PRIORITY" in gate|normal) ;; *) bs_die "--priority takes gate or normal, not '$PRIORITY'" ;; esac +case "${CARGO_ARGS[0]:-build}" in + test) SCHED_CLASS=suite ;; + bench) SCHED_CLASS=bench ;; + check|clippy) SCHED_CLASS=check ;; + *) SCHED_CLASS=build ;; +esac +if [ "$PRIORITY" = gate ]; then BR_NICE=0; BR_CORES=0; BR_JOBS_CAP=0 +elif [ "$SCHED_CLASS" = suite ] || [ "$SCHED_CLASS" = bench ]; then BR_NICE=10; BR_CORES=32; BR_JOBS_CAP=32; fi +export BR_NICE BR_CORES BR_JOBS_CAP BR_PRIORITY +if [ "$PLAN" = 1 ]; then + k="$SCHED_CLASS"; [ "$PRIORITY" = gate ] && k=gate + printf 'kind=%s nice=%s cores=%s jobs=%s priority=%s\n' "$k" "$BR_NICE" "$( [ "$BR_CORES" = 0 ] && echo 96 || echo "$BR_CORES")" "$( [ "$BR_JOBS_CAP" = 0 ] && echo box || echo "$BR_JOBS_CAP")" "$PRIORITY" + exit 0 +fi + bs_host bs_context @@ -175,6 +209,9 @@ fi cmd="$(bs_repro_env)${pre}CARGO_TARGET_DIR='$TARGET_DIR' cargo $(printf '%q ' "${CARGO_ARGS[@]}")${JOBS:+-j $JOBS} 2>&1 | tee -a '$BS_REMOTE_WT/.build-remote.log'; rc=\${PIPESTATUS[0]}; [ \$rc = 0 ] && echo '$BS_SHA' > '.build-remote-sha-$TARGET_DIR'; ( exit \$rc )" # a subshell exit: the runner reads \$? and still prints its RESULT line label="$BS_WT/$BS_CRATE_REL cargo ${CARGO_ARGS[*]}" BR_KIND=$(bs_kind build-remote "$( [ "${CARGO_ARGS[0]}" = zigbuild ] && echo build || echo "${CARGO_ARGS[0]}")"); BR_COMMAND="cargo ${CARGO_ARGS[*]}"; BR_TARGET="${BR_TARGET_SHIP:-x86_64-unknown-linux-gnu}" +[ "$SCHED_CLASS" = bench ] && BR_KIND=bench +[ "$PRIORITY" = gate ] && BR_KIND=gate +label="$label; kind=$BR_KIND nice=$BR_NICE cores=$( [ "$BR_CORES" = 0 ] && echo 96 || echo "$BR_CORES")" for ((i = 0; i < ${#CARGO_ARGS[@]}; i++)); do [ "${CARGO_ARGS[$i]}" = --target ] && BR_TARGET="${CARGO_ARGS[$((i + 1))]:-}"; done BR_ARTEFACTS=""; [ "$FETCH" = 1 ] && BR_ARTEFACTS="$ARTEFACTS" export BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS diff --git a/tools/ci/build-kind-default-check.sh b/tools/ci/build-kind-default-check.sh new file mode 100755 index 00000000..7a448c5a --- /dev/null +++ b/tools/ci/build-kind-default-check.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# The scheduling classes of tools/build-remote.sh (main, 7 October 2026, the load-190 night): a build-remote call WITHOUT a priority +# flag must put every suite (cargo test) and bench (cargo bench) into the bounded class, nice 10 on 32 cores with -j 32, while a +# build keeps the box's own jobs rule and --priority gate gives nice 0 on the full set. This check runs the tool's --plan mode (no box, +# no crate) for the four shapes and compares the resolved class; a change that lets a bare `cargo test` run unbounded again fails it. +# +# tools/ci/build-kind-default-check.sh # exit 1 with the shape that resolved wrongly +# tools/ci/build-kind-default-check.sh --self-test # the same four shapes (the check IS its self-test: every expectation is a known case) +set -euo pipefail +cd "$(dirname "$0")/../.." +expect() { # + local want="$1"; shift + local got; got=$(bash tools/build-remote.sh --plan "$@" 2>/dev/null || true) + if [ "$got" = "$want" ]; then echo "build-kind: [$*] -> $got"; else echo "build-kind: [$*] resolved to '$got', expected '$want'" >&2; return 1; fi +} +fail=0 +expect 'kind=suite nice=10 cores=32 jobs=32 priority=normal' -- test -p kaspa-consensus-core --lib || fail=1 +expect 'kind=bench nice=10 cores=32 jobs=32 priority=normal' -- bench -p igneum-pow || fail=1 +expect 'kind=build nice=0 cores=96 jobs=box priority=normal' -- build --release -p kaspad || fail=1 +expect 'kind=gate nice=0 cores=96 jobs=box priority=gate' --priority gate -- test -p igneum-app || fail=1 +expect 'kind=check nice=0 cores=96 jobs=box priority=normal' -- check || fail=1 +[ "$fail" = 0 ] && echo "build-kind: a call without a priority flag bounds suites and benches; a gate runs unbounded at nice 0" +exit $fail diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh index 5ca3f8e4..71e55cd3 100755 --- a/tools/ci/pre-push.sh +++ b/tools/ci/pre-push.sh @@ -75,6 +75,7 @@ tree_checks() { run "the remote checkout resets the mirror's tree before the branch checkout (the stale-overlay class)" bash -c 'bash tools/ci/mirror-reset-check.sh --self-test && bash tools/ci/mirror-reset-check.sh' run "the remote checkout's clean spares a lane's scratch (.igneum-scratch-spare, the fixed prefixes, never -x; the lost-scratch class)" bash -c 'bash tools/ci/scratch-spare-check.sh --self-test && bash tools/ci/scratch-spare-check.sh' run "long-running tools keep their body in one parsed block (the edited-while-running class)" bash -c 'bash tools/ci/whole-body-check.sh --self-test && bash tools/ci/whole-body-check.sh' + run "build-remote without a priority flag bounds suites and benches (nice 10, 32 cores); a gate runs unbounded" bash tools/ci/build-kind-default-check.sh run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh' run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh' run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test