#!/usr/bin/env bash # Shared helpers for the capacity layer on igneum-build-1 (infra/build-server/capacity). Sourced by run.sh and every # job in jobs/. Nothing here takes a build slot (the slots are /srv/builds/_locks/build-, owned by remote-run.sh) # and nothing writes under /srv/builds/. All work lives under /srv/capacity. # # Paths (overridable by environment for the smoke test and the dry run): # CAP_ROOT /srv/capacity the layer's own tree # CAP_SRC $CAP_ROOT/src a clone of /srv/igneum.git (master), the repo the jobs build and run from # CAP_FORK $CAP_SRC/vendor/igneum-node a clone of /srv/igneum-node.git, the node fork # CAP_OUT $CAP_ROOT/out// results, one directory per job per UTC day # CAP_STATE $CAP_ROOT/state per-job cursors (the continuous jobs advance their seed base here) # CAP_LOG $CAP_ROOT/log per-job slice logs # LOCKS /srv/builds/_locks read only: the build slots and the measure hold the layer yields to # CAP_JSON /srv/workers/capacity.json the one-line-per-job summary the dashboard reads set -uo pipefail CAP_ROOT="${CAP_ROOT:-/srv/capacity}" CAP_SRC="${CAP_SRC:-$CAP_ROOT/src}" CAP_FORK="${CAP_FORK:-$CAP_SRC/vendor/igneum-node}" CAP_OUT_ROOT="${CAP_OUT_ROOT:-$CAP_ROOT/out}" CAP_STATE="${CAP_STATE:-$CAP_ROOT/state}" CAP_LOG="${CAP_LOG:-$CAP_ROOT/log}" LOCKS="${IGNEUM_BUILD_SLOTS_DIR:-/srv/builds/_locks}" BUILDS_ROOT="${IGNEUM_BUILD_ROOT:-/srv/builds}" export IGNEUM_CAPACITY_JSON="${IGNEUM_CAPACITY_JSON:-/srv/workers/capacity.json}" REPO_MIRROR="${CAP_REPO_MIRROR:-/srv/igneum.git}" NODE_MIRROR="${CAP_NODE_MIRROR:-/srv/igneum-node.git}" NODE_BRANCH_DEFAULT="capacity-probe" # the sync-fuzz branch; falls back to the newest release-*-node on the mirror REPO_BRANCH="${CAP_REPO_BRANCH:-master}" # the repo branch the layer builds and runs from; master in production, box-capacity until it merges (install.sh writes a drop-in) HERE_LIB="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SUMMARY="$HERE_LIB/summary.mjs" NODE_BIN="${NODE_BIN:-/usr/local/bin/node}" UTC_DATE() { date -u +%Y-%m-%d; } cap_say() { printf '%s capacity/%s: %s\n' "$(date -u +%H:%M:%S)" "${CAP_JOB:-run}" "$*" >&2; } # defaults so `set -u` never trips before the first cap_sync_checkout (a standalone job, or a smoke) CAP_NODE_BRANCH="${CAP_NODE_BRANCH:-}"; CAP_REPO_SHA="${CAP_REPO_SHA:-}"; CAP_FORK_SHA="${CAP_FORK_SHA:-}"; CAP_REPO_BRANCH_USED="${CAP_REPO_BRANCH_USED:-}" # ---- the build-slot and measure hold the layer yields to -------------------------------------------------------------- # a slot or the measure file is "held" when flock -n cannot take it (another process holds the fd). The same probe the # collector uses (tools/workers/collect.mjs flockHeld). Returns 0 (true) when ANY build slot, the quiet hold or a core lease is taken. cap_build_active() { local slots k f slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1 if [ -e "$LOCKS/quiet" ] && ! flock -n "$LOCKS/quiet" true 2>/dev/null; then return 0; fi # 7 Oct 2026: the quiet class replaced the measure file for f in "$LOCKS"/core-*; do [ -e "$f" ] && ! flock -n "$f" true 2>/dev/null && return 0; done for k in $(seq 0 $((slots - 1))); do f="$LOCKS/build-$k" [ -e "$f" ] || continue if ! flock -n "$f" true 2>/dev/null; then return 0; fi done return 1 } cap_hold_reason() { local slots k if [ -e "$LOCKS/quiet" ] && ! flock -n "$LOCKS/quiet" true 2>/dev/null; then echo "quiet hold"; return; fi for f in "$LOCKS"/core-*; do [ -e "$f" ] && ! flock -n "$f" true 2>/dev/null && { echo "core lease"; return; }; done slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1 for k in $(seq 0 $((slots - 1))); do [ -e "$LOCKS/build-$k" ] || continue if ! flock -n "$LOCKS/build-$k" true 2>/dev/null; then echo "build slot build-$k held"; return; fi done echo "" } # ---- the capacity checkout (never a /srv/builds worktree) ------------------------------------------------------------- # clone-or-fetch the repo at master and the fork at its branch into $CAP_SRC. Idempotent. The jobs build into target # directories under this tree, never /srv/builds. cap_sync_checkout() { local node_branch="${1:-$NODE_BRANCH_DEFAULT}" mkdir -p "$CAP_ROOT" "$CAP_STATE" "$CAP_LOG" if [ ! -d "$CAP_SRC/.git" ]; then git clone -q "$REPO_MIRROR" "$CAP_SRC" || { cap_say "repo clone failed"; return 1; }; fi git -C "$CAP_SRC" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' || cap_say "repo fetch failed (using the last checkout)" git -C "$CAP_SRC" checkout -q -- . 2>/dev/null || true git -C "$CAP_SRC" clean -qfd -e target -e 'target-*' -e vendor -e out 2>/dev/null || true local rb="$REPO_BRANCH" git -C "$CAP_SRC" rev-parse -q --verify "origin/$rb" >/dev/null 2>&1 || rb=master git -C "$CAP_SRC" checkout -q -B capacity-run "origin/$rb" 2>/dev/null || git -C "$CAP_SRC" reset -q --hard "origin/$rb" CAP_REPO_BRANCH_USED="$rb"; export CAP_REPO_BRANCH_USED mkdir -p "$CAP_SRC/vendor" if [ ! -d "$CAP_FORK/.git" ]; then git clone -q --no-checkout "$NODE_MIRROR" "$CAP_FORK" || { cap_say "fork clone failed"; return 1; }; fi git -C "$CAP_FORK" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' || cap_say "fork fetch failed" local b="$node_branch" git -C "$CAP_FORK" rev-parse -q --verify "origin/$b" >/dev/null 2>&1 || b=$(git -C "$CAP_FORK" branch -r --list 'origin/release-*-node' | sed 's|.*/||' | sort -V | tail -1) [ -n "$b" ] || b=master git -C "$CAP_FORK" checkout -q -- . 2>/dev/null || true git -C "$CAP_FORK" clean -qfd -e target -e 'target-*' 2>/dev/null || true git -C "$CAP_FORK" checkout -q -B "$b" "origin/$b" 2>/dev/null || git -C "$CAP_FORK" reset -q --hard "origin/$b" CAP_NODE_BRANCH="$b" CAP_REPO_SHA=$(git -C "$CAP_SRC" rev-parse --short HEAD 2>/dev/null) CAP_FORK_SHA=$(git -C "$CAP_FORK" rev-parse --short HEAD 2>/dev/null) export CAP_NODE_BRANCH CAP_REPO_SHA CAP_FORK_SHA cap_say "checkout: repo master $CAP_REPO_SHA, fork $CAP_NODE_BRANCH $CAP_FORK_SHA" } # cargo under the layer's discipline: no build slot, idle IO, SCHED_IDLE, nice 19, a bounded job count (the controller's # SIGSTOP pauses it the instant a real build takes a slot; this keeps it gentle even while it runs). cap_cargo() { ( cd "$1" && shift && CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS="${CAP_CARGO_JOBS:-16}" \ nice -n 19 ionice -c3 chrt -i 0 "$HOME/.cargo/bin/cargo" "$@" ) } # cap_cargo with a timeout: . `timeout` cannot run the cap_cargo shell function, so the # timeout wraps the external cargo directly (same nice/ionice/chrt). Returns 124 on timeout. cap_cargo_t() { local secs="$1" dir="$2"; shift 2 ( cd "$dir" && CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS="${CAP_CARGO_JOBS:-16}" \ timeout "$secs" nice -n 19 ionice -c3 chrt -i 0 "$HOME/.cargo/bin/cargo" "$@" ) } cap_out_dir() { # : make and echo today's out dir local d="$CAP_OUT_ROOT/$1/$(UTC_DATE)"; mkdir -p "$d"; echo "$d" } # cap_summary : read a JSON fragment on stdin and merge it into capacity.json under that job cap_summary() { CAP_PRIORITY="${CAP_PRIORITY:-}" "$NODE_BIN" "$SUMMARY" job "$1"; } cap_controller_summary() { "$NODE_BIN" "$SUMMARY" controller; } # a cursor is a plain number per job (the continuous jobs' seed base); cap_cursor_get / cap_cursor_set cap_cursor_get() { cat "$CAP_STATE/$1.cursor" 2>/dev/null || echo "${2:-0}"; } cap_cursor_set() { mkdir -p "$CAP_STATE"; echo "$2" > "$CAP_STATE/$1.cursor"; } # json_escape a string for a one-line summary (python, always present) cap_json_str() { "$NODE_BIN" -e 'process.stdout.write(JSON.stringify(process.argv[1]||""))' "$1"; }