6f5787ed71
Main, 7 October 2026, 15:07 UK: one global exclusive measure flock across unrelated measurements stalled build-1 at load 120 with free slots (a stopped probe held it 5.5 h; an exclusive waiter queued every new shared taker) and build-2 behind a one-core VDF bench. lease.sh (installed at /srv/builds/_bin/lease by provision.sh and by hand on both boxes) takes one flock per core for a pinned measurement and the quiet file for a whole-box one; remote-run.sh takes quiet shared only for unbounded runs, excludes leased cores from its set, and its keeper refreshes the holder file and calls lease reap (a STOPPED holder of a lease, quiet or a slot for 5 minutes is killed with a line in _log/reaped.log). Keepers close the lock descriptors they inherit (an orphaned sleep held a slot and the worktree lock 20 s past the release; the slot self-test had rotted on that since the worktree lock landed). tools/ci/box-locks-check.sh runs lease.sh --self-test and remote-run.sh --self-test-slots on build-1. provision.sh also carries the 16 libraries headless Chromium needs (installed by hand on both boxes at 14:5x UK) and a headless self-test step. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
122 lines
7.6 KiB
Bash
Executable file
122 lines
7.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Shared helpers for the capacity layer on igneum-build-1 (infra/build-server/capacity). Sourced by run.sh and every
|
|
# job in jobs/. Nothing here takes a build slot (the slots are /srv/builds/_locks/build-<k>, owned by remote-run.sh)
|
|
# and nothing writes under /srv/builds/<worktree>. All work lives under /srv/capacity.
|
|
#
|
|
# Paths (overridable by environment for the smoke test and the dry run):
|
|
# CAP_ROOT /srv/capacity the layer's own tree
|
|
# CAP_SRC $CAP_ROOT/src a clone of /srv/igneum.git (master), the repo the jobs build and run from
|
|
# CAP_FORK $CAP_SRC/vendor/igneum-node a clone of /srv/igneum-node.git, the node fork
|
|
# CAP_OUT $CAP_ROOT/out/<job>/<date> results, one directory per job per UTC day
|
|
# CAP_STATE $CAP_ROOT/state per-job cursors (the continuous jobs advance their seed base here)
|
|
# CAP_LOG $CAP_ROOT/log per-job slice logs
|
|
# LOCKS /srv/builds/_locks read only: the build slots and the measure hold the layer yields to
|
|
# CAP_JSON /srv/workers/capacity.json the one-line-per-job summary the dashboard reads
|
|
set -uo pipefail
|
|
|
|
CAP_ROOT="${CAP_ROOT:-/srv/capacity}"
|
|
CAP_SRC="${CAP_SRC:-$CAP_ROOT/src}"
|
|
CAP_FORK="${CAP_FORK:-$CAP_SRC/vendor/igneum-node}"
|
|
CAP_OUT_ROOT="${CAP_OUT_ROOT:-$CAP_ROOT/out}"
|
|
CAP_STATE="${CAP_STATE:-$CAP_ROOT/state}"
|
|
CAP_LOG="${CAP_LOG:-$CAP_ROOT/log}"
|
|
LOCKS="${IGNEUM_BUILD_SLOTS_DIR:-/srv/builds/_locks}"
|
|
BUILDS_ROOT="${IGNEUM_BUILD_ROOT:-/srv/builds}"
|
|
export IGNEUM_CAPACITY_JSON="${IGNEUM_CAPACITY_JSON:-/srv/workers/capacity.json}"
|
|
REPO_MIRROR="${CAP_REPO_MIRROR:-/srv/igneum.git}"
|
|
NODE_MIRROR="${CAP_NODE_MIRROR:-/srv/igneum-node.git}"
|
|
NODE_BRANCH_DEFAULT="capacity-probe" # the sync-fuzz branch; falls back to the newest release-*-node on the mirror
|
|
REPO_BRANCH="${CAP_REPO_BRANCH:-master}" # the repo branch the layer builds and runs from; master in production, box-capacity until it merges (install.sh writes a drop-in)
|
|
HERE_LIB="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
SUMMARY="$HERE_LIB/summary.mjs"
|
|
NODE_BIN="${NODE_BIN:-/usr/local/bin/node}"
|
|
UTC_DATE() { date -u +%Y-%m-%d; }
|
|
cap_say() { printf '%s capacity/%s: %s\n' "$(date -u +%H:%M:%S)" "${CAP_JOB:-run}" "$*" >&2; }
|
|
# defaults so `set -u` never trips before the first cap_sync_checkout (a standalone job, or a smoke)
|
|
CAP_NODE_BRANCH="${CAP_NODE_BRANCH:-}"; CAP_REPO_SHA="${CAP_REPO_SHA:-}"; CAP_FORK_SHA="${CAP_FORK_SHA:-}"; CAP_REPO_BRANCH_USED="${CAP_REPO_BRANCH_USED:-}"
|
|
|
|
# ---- the build-slot and measure hold the layer yields to --------------------------------------------------------------
|
|
# a slot or the measure file is "held" when flock -n cannot take it (another process holds the fd). The same probe the
|
|
# collector uses (tools/workers/collect.mjs flockHeld). Returns 0 (true) when ANY build slot, the quiet hold or a core lease is taken.
|
|
cap_build_active() {
|
|
local slots k f
|
|
slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1
|
|
if [ -e "$LOCKS/quiet" ] && ! flock -n "$LOCKS/quiet" true 2>/dev/null; then return 0; fi # 7 Oct 2026: the quiet class replaced the measure file
|
|
for f in "$LOCKS"/core-*; do [ -e "$f" ] && ! flock -n "$f" true 2>/dev/null && return 0; done
|
|
for k in $(seq 0 $((slots - 1))); do
|
|
f="$LOCKS/build-$k"
|
|
[ -e "$f" ] || continue
|
|
if ! flock -n "$f" true 2>/dev/null; then return 0; fi
|
|
done
|
|
return 1
|
|
}
|
|
cap_hold_reason() {
|
|
local slots k
|
|
if [ -e "$LOCKS/quiet" ] && ! flock -n "$LOCKS/quiet" true 2>/dev/null; then echo "quiet hold"; return; fi
|
|
for f in "$LOCKS"/core-*; do [ -e "$f" ] && ! flock -n "$f" true 2>/dev/null && { echo "core lease"; return; }; done
|
|
slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1
|
|
for k in $(seq 0 $((slots - 1))); do
|
|
[ -e "$LOCKS/build-$k" ] || continue
|
|
if ! flock -n "$LOCKS/build-$k" true 2>/dev/null; then echo "build slot build-$k held"; return; fi
|
|
done
|
|
echo ""
|
|
}
|
|
|
|
# ---- the capacity checkout (never a /srv/builds worktree) -------------------------------------------------------------
|
|
# clone-or-fetch the repo at master and the fork at its branch into $CAP_SRC. Idempotent. The jobs build into target
|
|
# directories under this tree, never /srv/builds.
|
|
cap_sync_checkout() {
|
|
local node_branch="${1:-$NODE_BRANCH_DEFAULT}"
|
|
mkdir -p "$CAP_ROOT" "$CAP_STATE" "$CAP_LOG"
|
|
if [ ! -d "$CAP_SRC/.git" ]; then git clone -q "$REPO_MIRROR" "$CAP_SRC" || { cap_say "repo clone failed"; return 1; }; fi
|
|
git -C "$CAP_SRC" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' || cap_say "repo fetch failed (using the last checkout)"
|
|
git -C "$CAP_SRC" checkout -q -- . 2>/dev/null || true
|
|
git -C "$CAP_SRC" clean -qfd -e target -e 'target-*' -e vendor -e out 2>/dev/null || true
|
|
local rb="$REPO_BRANCH"
|
|
git -C "$CAP_SRC" rev-parse -q --verify "origin/$rb" >/dev/null 2>&1 || rb=master
|
|
git -C "$CAP_SRC" checkout -q -B capacity-run "origin/$rb" 2>/dev/null || git -C "$CAP_SRC" reset -q --hard "origin/$rb"
|
|
CAP_REPO_BRANCH_USED="$rb"; export CAP_REPO_BRANCH_USED
|
|
mkdir -p "$CAP_SRC/vendor"
|
|
if [ ! -d "$CAP_FORK/.git" ]; then git clone -q --no-checkout "$NODE_MIRROR" "$CAP_FORK" || { cap_say "fork clone failed"; return 1; }; fi
|
|
git -C "$CAP_FORK" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' || cap_say "fork fetch failed"
|
|
local b="$node_branch"
|
|
git -C "$CAP_FORK" rev-parse -q --verify "origin/$b" >/dev/null 2>&1 || b=$(git -C "$CAP_FORK" branch -r --list 'origin/release-*-node' | sed 's|.*/||' | sort -V | tail -1)
|
|
[ -n "$b" ] || b=master
|
|
git -C "$CAP_FORK" checkout -q -- . 2>/dev/null || true
|
|
git -C "$CAP_FORK" clean -qfd -e target -e 'target-*' 2>/dev/null || true
|
|
git -C "$CAP_FORK" checkout -q -B "$b" "origin/$b" 2>/dev/null || git -C "$CAP_FORK" reset -q --hard "origin/$b"
|
|
CAP_NODE_BRANCH="$b"
|
|
CAP_REPO_SHA=$(git -C "$CAP_SRC" rev-parse --short HEAD 2>/dev/null)
|
|
CAP_FORK_SHA=$(git -C "$CAP_FORK" rev-parse --short HEAD 2>/dev/null)
|
|
export CAP_NODE_BRANCH CAP_REPO_SHA CAP_FORK_SHA
|
|
cap_say "checkout: repo master $CAP_REPO_SHA, fork $CAP_NODE_BRANCH $CAP_FORK_SHA"
|
|
}
|
|
|
|
# cargo under the layer's discipline: no build slot, idle IO, SCHED_IDLE, nice 19, a bounded job count (the controller's
|
|
# SIGSTOP pauses it the instant a real build takes a slot; this keeps it gentle even while it runs).
|
|
cap_cargo() {
|
|
( cd "$1" && shift && CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS="${CAP_CARGO_JOBS:-16}" \
|
|
nice -n 19 ionice -c3 chrt -i 0 "$HOME/.cargo/bin/cargo" "$@" )
|
|
}
|
|
# cap_cargo with a timeout: <secs> <dir> <cargo args...>. `timeout` cannot run the cap_cargo shell function, so the
|
|
# timeout wraps the external cargo directly (same nice/ionice/chrt). Returns 124 on timeout.
|
|
cap_cargo_t() {
|
|
local secs="$1" dir="$2"; shift 2
|
|
( cd "$dir" && CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS="${CAP_CARGO_JOBS:-16}" \
|
|
timeout "$secs" nice -n 19 ionice -c3 chrt -i 0 "$HOME/.cargo/bin/cargo" "$@" )
|
|
}
|
|
|
|
cap_out_dir() { # <job>: make and echo today's out dir
|
|
local d="$CAP_OUT_ROOT/$1/$(UTC_DATE)"; mkdir -p "$d"; echo "$d"
|
|
}
|
|
|
|
# cap_summary <job> : read a JSON fragment on stdin and merge it into capacity.json under that job
|
|
cap_summary() { CAP_PRIORITY="${CAP_PRIORITY:-}" "$NODE_BIN" "$SUMMARY" job "$1"; }
|
|
cap_controller_summary() { "$NODE_BIN" "$SUMMARY" controller; }
|
|
|
|
# a cursor is a plain number per job (the continuous jobs' seed base); cap_cursor_get / cap_cursor_set
|
|
cap_cursor_get() { cat "$CAP_STATE/$1.cursor" 2>/dev/null || echo "${2:-0}"; }
|
|
cap_cursor_set() { mkdir -p "$CAP_STATE"; echo "$2" > "$CAP_STATE/$1.cursor"; }
|
|
|
|
# json_escape a string for a one-line summary (python, always present)
|
|
cap_json_str() { "$NODE_BIN" -e 'process.stdout.write(JSON.stringify(process.argv[1]||""))' "$1"; }
|