igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

Build box scheduling: suites and benches at nice 10 on 32 cores with -j 32; a gate kind at nice 0 on the full set ahead of queued suites; kind, nice and cores on the slot label and the JSONL line; the default-class CI check

Browse source 
Main's order of 7 October 2026 after a load of 190 on 96 threads (a release join bench and the 0.3.19 app gate starving each
other, 'builds' of 16 minutes). tools/build-remote.sh resolves a class from the cargo subcommand and --priority: test and bench are
the bounded class (nice 10, the last 32 cores, -j 32) unless --priority gate (nice 0, the full set, the box's own jobs rule);
builds and checks are unchanged. remote-run.sh applies renice and taskset to the command's subshell, caps the jobs, lets a queued
gate (gate-pending-<pid>) take the next slot ahead of suites and benches, and prints nice and cores in the RESULT line and the
JSONL line (nice, cores, priority). The slot label carries '; kind=<k> nice=<n> cores=<c>' before '; agent=', so the dashboard's
job card shows why a job is slow. --plan prints the resolved class without the box; tools/ci/build-kind-default-check.sh (in the
gate) holds the five shapes. Smoke on the box: a suite at jobs=32 nice=10 cores=32, a gate at nice=0 cores=96.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-07 10:29:26 +00:00
parent 8fb2f32e13
commit 51304fa51b
5 changed files with 88 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
infra/build-server/lib.sh
View file

@ -281,8 +281,9 @@ bs_remote_run() {
BR_DIR="$dir" BR_LABEL="$label" BR_CMD="$cmd" BR_TOOL="${BS_TOOL:-build-remote}" BR_WT="$BS_WT" BR_CRATE="$BS_CRATE_REL" \
BR_BRANCH="$BS_BRANCH" BR_SHA="$BS_SHA" BR_AGENT="$agent" BR_KIND="${BR_KIND:-other}" BR_COMMAND="${BR_COMMAND:-}" \
BR_TARGET="${BR_TARGET:-}" BR_ARTEFACTS="${BR_ARTEFACTS:-}" BR_SDE="${BR_SDE:-}" BR_PAIRS_WITH="${BR_PAIRS_WITH:-}" \
BR_NICE="${BR_NICE:-0}" BR_CORES="${BR_CORES:-0}" BR_JOBS_CAP="${BR_JOBS_CAP:-0}" BR_PRIORITY="${BR_PRIORITY:-normal}" \
bash -c '
for v in BR_DIR BR_LABEL BR_CMD BR_TOOL BR_WT BR_CRATE BR_BRANCH BR_SHA BR_AGENT BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS BR_SDE BR_PAIRS_WITH; do
for v in BR_DIR BR_LABEL BR_CMD BR_TOOL BR_WT BR_CRATE BR_BRANCH BR_SHA BR_AGENT BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS BR_SDE BR_PAIRS_WITH BR_NICE BR_CORES BR_JOBS_CAP BR_PRIORITY; do
printf "export %s=%q\n" "$v" "${!v}"
done
cat "$0"' "$(dirname "${BASH_SOURCE[0]}")/remote-run.sh" | bs_ssh 'bash -s'

28
infra/build-server/remote-run.sh
View file

@ -238,6 +238,7 @@ d = {
"start": iso(e['BR_START']), "end": iso(e['BR_END']), "secs": num(e['BR_SECS']), "exit": num(e['BR_EXIT']),
"compiles": num(e['BR_COMPILES']), "jobs": num(e.get('BR_JOBS')), "measure": (e.get('BR_MEASURE') == '1') or None,
"source_date_epoch": num(e.get('BR_SDE')), "pairs_with": e.get('BR_PAIRS_WITH') or None,
"nice": num(e.get('BR_NICE')) or 0, "cores": (num(e.get('BR_CORES')) or 0) or os.cpu_count(), "priority": e.get('BR_PRIORITY') or "normal",
"class": e.get('BR_CLASS') or None, "run_log": e.get('BR_RUN_LOG') or None,
}
sc = {k: num(e[v]) for k, v in (("hits", "BR_HITS"), ("misses", "BR_MISSES"), ("hits_total", "BR_HITS_T"), ("misses_total", "BR_MISSES_T"))}
@ -321,6 +322,18 @@ else
flock -s -w 7200 "$mfd" || give_up "the measurement to end"
rm -f "$waitfile"; trap - EXIT
fi
# scheduling (main, 7 Oct 2026): a gate announces itself (gate-pending-<pid>) and takes the next free slot; a suite, bench or
# other non-gate run that has not taken a slot yet yields while any gate-pending marker younger than 15 min exists
gatefile=""
if [ "${BR_PRIORITY:-normal}" = gate ]; then gatefile="$SLOTS_DIR/gate-pending-$BR_PID"; holder_line 0 > "$gatefile"; trap 'rm -f "$gatefile"' EXIT
else
yt0=$(date +%s)
while pending=$(find "$SLOTS_DIR" -maxdepth 1 -name 'gate-pending-*' -mmin -15 2>/dev/null | head -1) && [ -n "$pending" ]; do
[ -f "$waitfile" ] || { echo "build-remote: a gate is queued ($(head -c 120 "$pending")), this ${BR_KIND:-run} yields" >&2; holder_line 0 > "$waitfile"; trap 'rm -f "$waitfile"' EXIT; }
[ $(( $(date +%s) - yt0 )) -lt 7200 ] || give_up "the queued gate"
sleep 5
done
fi
got=""
for k in $(seq 0 $((slots - 1))); do
exec {fd}>>"$SLOTS_DIR/build-$k"
@ -349,8 +362,11 @@ else
exec {tfd}>&-
done
if [ "$held" -gt 1 ]; then BR_JOBS=$JOBS_SHARED; else BR_JOBS=$JOBS_ALONE; fi
# the bounded classes cap the jobs (suite and bench: 32 unless the caller passed --priority gate)
if [ "${BR_JOBS_CAP:-0}" -gt 0 ] && [ "$BR_JOBS" -gt "$BR_JOBS_CAP" ]; then BR_JOBS=$BR_JOBS_CAP; fi
export CARGO_BUILD_JOBS="$BR_JOBS" BR_JOBS
echo "build-remote: holding build-$got on $BR_HOST (waited $waited s; $held of $slots slots held, CARGO_BUILD_JOBS=$BR_JOBS)" >&2
[ -n "$gatefile" ] && { rm -f "$gatefile"; trap - EXIT; }
echo "build-remote: holding build-$got on $BR_HOST (waited $waited s; $held of $slots slots held, CARGO_BUILD_JOBS=$BR_JOBS, kind ${BR_KIND:-other}, nice ${BR_NICE:-0}, cores $( [ "${BR_CORES:-0}" = 0 ] && nproc || echo "$BR_CORES"))" >&2
fi
release_slot() { if [ "$got" = measure ]; then : > "$SLOTS_DIR/measure"; else : > "$SLOTS_DIR/build-$got"; fi; }
@ -403,7 +419,11 @@ t1=$(date +%s)
# streams stay where they were for the Mac (stdout to stdout, stderr to stderr), each teed into the run log
RUN_LOG_DIR="$LOG_DIR/runs"; mkdir -p "$RUN_LOG_DIR"
BR_RUN_LOG="$RUN_LOG_DIR/$BR_HOST-$BR_T0-$BR_PID.log"; export BR_RUN_LOG
( eval "$BR_CMD" ) > >(tee -a "$BR_RUN_LOG") 2> >(tee -a "$BR_RUN_LOG" >&2)
# the class's nice and core set apply to the command's subshell and everything it starts (renice and taskset on the subshell's own
# pid, BASHPID; cores are the LAST N of the box's set, so gates and builds keep the first ones to themselves)
ncpu=$(nproc); cores_str="0-$((ncpu - 1))"
if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then cores_str="$((ncpu - BR_CORES))-$((ncpu - 1))"; fi
( [ "${BR_NICE:-0}" -gt 0 ] && renice -n "$BR_NICE" -p $BASHPID >/dev/null 2>&1; [ "$cores_str" != "0-$((ncpu - 1))" ] && taskset -cp "$cores_str" $BASHPID >/dev/null 2>&1; eval "$BR_CMD" ) > >(tee -a "$BR_RUN_LOG") 2> >(tee -a "$BR_RUN_LOG" >&2)
rc=$?
wait
t2=$(date +%s); secs=$(( t2 - t1 ))
@ -426,8 +446,8 @@ elif [[ "$BR_CMD" == cargo\ test* ]] && { [[ "$BR_CMD" == *" -- "* ]] || grep -q
echo "build-remote: REFUSED after the run: the test filter matched no test in any binary (every 'running 0 tests'); check the name" >&2
fi
export BR_CLASS
printf 'build-remote: RESULT rc=%s secs=%s compiles=%s sccache_hits=%s sccache_misses=%s sccache_hits_total=%s sccache_misses_total=%s jobs=%s load=%s class=%s\n' \
"$rc" "$secs" "$compiles" "$hits" "$misses" "${hits_after:-?}" "${misses_after:-?}" "${BR_JOBS:-measure}" "$(cut -d' ' -f1-3 /proc/loadavg)" "${BR_CLASS:-ok}"
printf 'build-remote: RESULT rc=%s secs=%s compiles=%s sccache_hits=%s sccache_misses=%s sccache_hits_total=%s sccache_misses_total=%s jobs=%s nice=%s cores=%s load=%s class=%s\n' \
"$rc" "$secs" "$compiles" "$hits" "$misses" "${hits_after:-?}" "${misses_after:-?}" "${BR_JOBS:-measure}" "${BR_NICE:-0}" "$( [ "${BR_CORES:-0}" = 0 ] && nproc || echo "$BR_CORES")" "$(cut -d' ' -f1-3 /proc/loadavg)" "${BR_CLASS:-ok}"
jsonlog "$rc" "$([ "$got" = measure ] && echo "" || echo "$got")" "$waited" "$t1" "$t2" "$secs" "$compiles" "$hits" "$misses" "${hits_after:-}" "${misses_after:-}"
[ "$rc" = 0 ] || redlog "$rc" "$secs" "$BR_CLASS"
release_slot

39
tools/build-remote.sh
View file

@ -24,6 +24,18 @@
# A plain build is native glibc 2.39: the box, the fleet's
# Ubuntu 24.04 hosts, never a seed or a rig (7 Oct 2026:
# a seed took 14 restarts on a 2.39 binary).
# tools/build-remote.sh --priority gate -- test ... a RELEASE GATE (the app gate, the canary cut, the pre-push
# self-tests): nice 0, the full core set, a slot ahead of
# queued suites and benches
# tools/build-remote.sh --plan [--priority gate] -- <cargo args> print the resolved class (kind, nice, cores, jobs) and stop;
# no box, no crate needed (the CI check uses it)
#
# Scheduling (main, 7 October 2026, after a load of 190 on 96 threads: a release join bench and the 0.3.19 app gate starving each
# other and "builds" of 16 minutes): every SUITE (cargo test) and BENCH (cargo bench) runs under nice 10 on the last 32 cores with
# -j 32 unless the caller passes --priority gate; builds keep the box's own jobs rule (90 alone, 45 beside another slot holder); a
# gate runs at nice 0 on the full set and takes a slot ahead of queued suites and benches (remote-run.sh gate-pending marker). The
# class travels in the slot label ("; kind=suite nice=10 cores=32") and the JSONL line, so the dashboard's job card shows why a
# job is slow. A call without a priority flag defaults to the bounded class for suites and benches: tools/ci/build-kind-default-check.sh.
# tools/build-remote.sh --self-test-repro [--full] from a fork worktree: igneum-miner built twice a minute
# apart without sccache into one target dir must give one
# sha256, and a per-run target path must not (prost's
@ -63,7 +75,7 @@ BS_TOOL=build-remote
# flight cannot reach the running copy (7 Oct 2026: build-remote.sh was edited mid-run and died on shifted bytes after a 4-min build)
# JOBS empty = the box decides: 90 alone, 45 beside another slot holder (remote-run.sh, main's ruling 6 Oct 2026)
JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0; SHIP=0; SHIP_CLASS="${SHIP_CLASS:-seed}"; GLIBC="${GLIBC:-}"
JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0; SHIP=0; SHIP_CLASS="${SHIP_CLASS:-seed}"; PRIORITY="${PRIORITY:-normal}"; PLAN=0; GLIBC="${GLIBC:-}"
while [ $# -gt 0 ]; do
case "$1" in
--jobs) JOBS="$2"; shift 2 ;;
@ -73,6 +85,9 @@ while [ $# -gt 0 ]; do
--no-fetch) FETCH=0; shift ;;
--self-test-repro) SELFTEST=1; shift ;;
--ship) SHIP=1; case "${2:-}" in hive|rig|seed|linux|native) SHIP_CLASS="$2"; shift 2 ;; *) shift ;; esac ;;
--priority) PRIORITY="$2"; shift 2 ;;
--gate) PRIORITY=gate; shift ;;
--plan) PLAN=1; shift ;;
--glibc) GLIBC="$2"; shift 2 ;;
--full) FULL=1; shift ;;
--) shift; CARGO_ARGS=("$@"); break ;;
@ -83,6 +98,25 @@ done
[ "${CARGO_ARGS[0]:-}" = cargo ] && CARGO_ARGS=("${CARGO_ARGS[@]:1}")
CARGO_ARGS_GIVEN=""; [ -n "${CARGO_ARGS[*]:-}" ] && CARGO_ARGS_GIVEN=1
# the scheduling class, from the cargo subcommand and the priority flag alone (no box, no crate): BR_NICE, BR_CORES (the last N of the
# box's 96; 0 = the full set), BR_JOBS_CAP (0 = the box's rule), BR_PRIORITY; the kind for the log is refined after bs_context
BR_NICE=0; BR_CORES=0; BR_JOBS_CAP=0; BR_PRIORITY="$PRIORITY"; SCHED_CLASS=build
case "$PRIORITY" in gate|normal) ;; *) bs_die "--priority takes gate or normal, not '$PRIORITY'" ;; esac
case "${CARGO_ARGS[0]:-build}" in
test) SCHED_CLASS=suite ;;
bench) SCHED_CLASS=bench ;;
check|clippy) SCHED_CLASS=check ;;
*) SCHED_CLASS=build ;;
esac
if [ "$PRIORITY" = gate ]; then BR_NICE=0; BR_CORES=0; BR_JOBS_CAP=0
elif [ "$SCHED_CLASS" = suite ] || [ "$SCHED_CLASS" = bench ]; then BR_NICE=10; BR_CORES=32; BR_JOBS_CAP=32; fi
export BR_NICE BR_CORES BR_JOBS_CAP BR_PRIORITY
if [ "$PLAN" = 1 ]; then
k="$SCHED_CLASS"; [ "$PRIORITY" = gate ] && k=gate
printf 'kind=%s nice=%s cores=%s jobs=%s priority=%s\n' "$k" "$BR_NICE" "$( [ "$BR_CORES" = 0 ] && echo 96 || echo "$BR_CORES")" "$( [ "$BR_JOBS_CAP" = 0 ] && echo box || echo "$BR_JOBS_CAP")" "$PRIORITY"
exit 0
fi
bs_host
bs_context
@ -175,6 +209,9 @@ fi
cmd="$(bs_repro_env)${pre}CARGO_TARGET_DIR='$TARGET_DIR' cargo $(printf '%q ' "${CARGO_ARGS[@]}")${JOBS:+-j $JOBS} 2>&1 | tee -a '$BS_REMOTE_WT/.build-remote.log'; rc=\${PIPESTATUS[0]}; [ \$rc = 0 ] && echo '$BS_SHA' > '.build-remote-sha-$TARGET_DIR'; ( exit \$rc )" # a subshell exit: the runner reads \$? and still prints its RESULT line
label="$BS_WT/$BS_CRATE_REL cargo ${CARGO_ARGS[*]}"
BR_KIND=$(bs_kind build-remote "$( [ "${CARGO_ARGS[0]}" = zigbuild ] && echo build || echo "${CARGO_ARGS[0]}")"); BR_COMMAND="cargo ${CARGO_ARGS[*]}"; BR_TARGET="${BR_TARGET_SHIP:-x86_64-unknown-linux-gnu}"
[ "$SCHED_CLASS" = bench ] && BR_KIND=bench
[ "$PRIORITY" = gate ] && BR_KIND=gate
label="$label; kind=$BR_KIND nice=$BR_NICE cores=$( [ "$BR_CORES" = 0 ] && echo 96 || echo "$BR_CORES")"
for ((i = 0; i < ${#CARGO_ARGS[@]}; i++)); do [ "${CARGO_ARGS[$i]}" = --target ] && BR_TARGET="${CARGO_ARGS[$((i + 1))]:-}"; done
BR_ARTEFACTS=""; [ "$FETCH" = 1 ] && BR_ARTEFACTS="$ARTEFACTS"
export BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS

23
tools/ci/build-kind-default-check.sh Executable file
View file

@ -0,0 +1,23 @@
#!/usr/bin/env bash
# The scheduling classes of tools/build-remote.sh (main, 7 October 2026, the load-190 night): a build-remote call WITHOUT a priority
# flag must put every suite (cargo test) and bench (cargo bench) into the bounded class, nice 10 on 32 cores with -j 32, while a
# build keeps the box's own jobs rule and --priority gate gives nice 0 on the full set. This check runs the tool's --plan mode (no box,
# no crate) for the four shapes and compares the resolved class; a change that lets a bare `cargo test` run unbounded again fails it.
#
# tools/ci/build-kind-default-check.sh # exit 1 with the shape that resolved wrongly
# tools/ci/build-kind-default-check.sh --self-test # the same four shapes (the check IS its self-test: every expectation is a known case)
set -euo pipefail
cd "$(dirname "$0")/../.."
expect() { # <expected line> <args...>
local want="$1"; shift
local got; got=$(bash tools/build-remote.sh --plan "$@" 2>/dev/null || true)
if [ "$got" = "$want" ]; then echo "build-kind: [$*] -> $got"; else echo "build-kind: [$*] resolved to '$got', expected '$want'" >&2; return 1; fi
}
fail=0
expect 'kind=suite nice=10 cores=32 jobs=32 priority=normal' -- test -p kaspa-consensus-core --lib || fail=1
expect 'kind=bench nice=10 cores=32 jobs=32 priority=normal' -- bench -p igneum-pow || fail=1
expect 'kind=build nice=0 cores=96 jobs=box priority=normal' -- build --release -p kaspad || fail=1
expect 'kind=gate nice=0 cores=96 jobs=box priority=gate' --priority gate -- test -p igneum-app || fail=1
expect 'kind=check nice=0 cores=96 jobs=box priority=normal' -- check || fail=1
[ "$fail" = 0 ] && echo "build-kind: a call without a priority flag bounds suites and benches; a gate runs unbounded at nice 0"
exit $fail

1
tools/ci/pre-push.sh
View file

@ -75,6 +75,7 @@ tree_checks() {
run "the remote checkout resets the mirror's tree before the branch checkout (the stale-overlay class)" bash -c 'bash tools/ci/mirror-reset-check.sh --self-test && bash tools/ci/mirror-reset-check.sh'
run "the remote checkout's clean spares a lane's scratch (.igneum-scratch-spare, the fixed prefixes, never -x; the lost-scratch class)" bash -c 'bash tools/ci/scratch-spare-check.sh --self-test && bash tools/ci/scratch-spare-check.sh'
run "long-running tools keep their body in one parsed block (the edited-while-running class)" bash -c 'bash tools/ci/whole-body-check.sh --self-test && bash tools/ci/whole-body-check.sh'
run "build-remote without a priority flag bounds suites and benches (nice 10, 32 cores); a gate runs unbounded" bash tools/ci/build-kind-default-check.sh
run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh'
run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh'
run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test