diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b357020..755e799e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,9 @@ jobs: sims: name: simulators, quick modes needs: changes - if: ${{ needs.changes.outputs.code == 'true' }} + # master and release-* pushes, and pull requests into them, only (main, 7 October 2026: every code push cost two box jobs and the + # queue read 22); a feature-branch code push runs the igneum-pow tests alone. tools/ci/sims-branch-check.sh holds this rule. + if: ${{ needs.changes.outputs.code == 'true' && ((github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))) || (github.event_name == 'pull_request' && (github.base_ref == 'master' || startsWith(github.base_ref, 'release-')))) }} runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 24c5ca04..a7e907d5 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -377,3 +377,28 @@ spill-over. Now (lib.sh `bs_route_spill`, master from this commit): them keeps its number; since this commit a bounded run takes the band its slot owns (slot 0 the last 32 cores, slot 1 the 32 below, slot 2 the 32 below that), so three bounded runs never share a core. A gate still takes its slot ahead of queued suites. - `--box N` still pins. Self-test: tools/ci/route-spill-check.sh (thirteen cases through `BS_ROUTE_STATE_`, no ssh), in the gate. + +## 8. The measure file is retired: per-core leases and the quiet class (7 October 2026, 15:07 UK) + +Main's reading at 15:07 UK: on build-1 a sync-fuzz probe (the capacity lane's, SIGSTOPped since 09:45Z, no owner) held the global +measure flock for five and a half hours; beside it attack-f6's phase2b waited exclusive on the same file behind seven F2 solvers +holding it shared on cores 6-11,54-59, and every new shared taker (every build) queued behind the exclusive waiter: load 120, slots +free, nine waiting. On build-2 the era VDF bench held the file exclusive while pinned to one core and five jobs waited. One global +exclusive lock across unrelated measurements was the wrong design. Now: + +- `infra/build-server/lease.sh`, installed on every box at `/srv/builds/_bin/lease` (provision.sh; by hand on build-1 and build-2 + at 14:2x BST): `lease cores --label "..." [--owner ] -- ` takes a lease on THOSE CORES ONLY (one flock per + core, `_locks/core-`, a `wait-` file with the label while it waits), runs the command under nice 10 and taskset, and + releases. Nothing else is excluded. `lease quiet --label "..." --owner -- ` is the whole-box class: refused (exit + 73) while any slot or core lease is held, capped at 20 minutes, holder line with the owner in `_locks/quiet`. `lease status`, + `lease reap`. +- remote-run.sh: an unbounded run (nice 0, the full set) takes `quiet` shared and waits for it; a bounded run (suites, benches, + everything on box 2) never takes it. Every run keeps off leased cores (its set minus the `core-` flocks, said once). A run's + keeper refreshes its holder file's mtime every 20 s and calls `lease reap`: a holder of a lease, the quiet file or a slot whose + process has been STOPPED for 5 minutes is killed and its file cleared, one line each in `_log/reaped.log`. The keeper closes the + lock descriptors it inherits (an orphaned `sleep 20` held a slot and a worktree lock 20 s past the release). BR_MEASURE=1 is the + quiet class with the same refusals; it needs a named owner (IGNEUM_AGENT). +- The lanes' own `flock -s /srv/builds/_locks/measure -c "nice -n 10 taskset -c ..."` lines no longer hold anything a build + waits for; they become `/srv/builds/_bin/lease cores --label "..." -- `, and `flock -x .../measure` becomes + `lease quiet`. Self-tests: lease.sh --self-test and remote-run.sh --self-test-slots, run on build-1 by tools/ci/box-locks-check.sh + in the gate. diff --git a/infra/build-server/capacity/lib.sh b/infra/build-server/capacity/lib.sh index 7418d4b2..a7d36d3e 100755 --- a/infra/build-server/capacity/lib.sh +++ b/infra/build-server/capacity/lib.sh @@ -37,11 +37,12 @@ CAP_NODE_BRANCH="${CAP_NODE_BRANCH:-}"; CAP_REPO_SHA="${CAP_REPO_SHA:-}"; CAP_FO # ---- the build-slot and measure hold the layer yields to -------------------------------------------------------------- # a slot or the measure file is "held" when flock -n cannot take it (another process holds the fd). The same probe the -# collector uses (tools/workers/collect.mjs flockHeld). Returns 0 (true) when ANY build slot or the measure hold is taken. +# collector uses (tools/workers/collect.mjs flockHeld). Returns 0 (true) when ANY build slot, the quiet hold or a core lease is taken. cap_build_active() { local slots k f slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1 - if [ -e "$LOCKS/measure" ] && ! flock -n "$LOCKS/measure" true 2>/dev/null; then return 0; fi + if [ -e "$LOCKS/quiet" ] && ! flock -n "$LOCKS/quiet" true 2>/dev/null; then return 0; fi # 7 Oct 2026: the quiet class replaced the measure file + for f in "$LOCKS"/core-*; do [ -e "$f" ] && ! flock -n "$f" true 2>/dev/null && return 0; done for k in $(seq 0 $((slots - 1))); do f="$LOCKS/build-$k" [ -e "$f" ] || continue @@ -51,7 +52,8 @@ cap_build_active() { } cap_hold_reason() { local slots k - if [ -e "$LOCKS/measure" ] && ! flock -n "$LOCKS/measure" true 2>/dev/null; then echo "measure hold"; return; fi + if [ -e "$LOCKS/quiet" ] && ! flock -n "$LOCKS/quiet" true 2>/dev/null; then echo "quiet hold"; return; fi + for f in "$LOCKS"/core-*; do [ -e "$f" ] && ! flock -n "$f" true 2>/dev/null && { echo "core lease"; return; }; done slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1 for k in $(seq 0 $((slots - 1))); do [ -e "$LOCKS/build-$k" ] || continue diff --git a/infra/build-server/lease.sh b/infra/build-server/lease.sh new file mode 100755 index 00000000..09baf355 --- /dev/null +++ b/infra/build-server/lease.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Measurement leases on a build box (main, 7 October 2026, 15:07 UK: one global exclusive "measure" flock across unrelated +# measurements stalled build-1 at load 120 with free slots, an exclusive waiter queueing every new shared taker behind it, and +# a stopped probe held the box for five and a half hours). The measure file is retired. In its place: +# +# lease cores --label "" [--owner ] [--nice N] -- +# A measurement that pins cores takes a lease on THOSE CORES ONLY (one flock per core, _locks/core-, taken in ascending +# order, waited for up to 2 h with a wait- file carrying the label), runs the command under nice N (default 10) and +# taskset on the set, and releases. Builds and suites keep off leased cores (remote-run.sh reads the core files before it +# pins its own set). Nothing else is excluded: the box stays open. +# lease quiet --label "" --owner [--cap-s N] -- +# A WHOLE-BOX quiet measurement: its own class, refused (exit 73) while any build slot or any core lease is held, capped at +# 20 minutes (timeout; --cap-s at most 1200), holder line with the owner in _locks/quiet. Unbounded builds take quiet +# shared and wait for it; bounded suites (nice 10, a 32-core band) never take it. +# lease status every lease, the quiet holder and every waiter with its label +# lease reap a holder (lease, quiet or build slot) whose process has been STOPPED (state T) for 5 minutes or more is +# killed and its file cleared, one line each in _log/reaped.log; remote-run.sh's keeper calls this every +# 20 s while any run is on the box (LEASE_REAP_S overrides the 300 s for the self-test) +# lease --self-test the known cases against a scratch lock directory (in the gate: tools/ci/pre-push.sh) +# +# Installed on every box at /srv/builds/_bin/lease by provision.sh (and copied by hand on 7 October 2026); the lock directory is +# IGNEUM_BUILD_SLOTS_DIR (the profile sets /srv/builds/_locks), the log directory IGNEUM_BUILD_LOG_DIR. Files: core- (flock), +# lease- (holder line "pid N since HH:MM:SSZ cores :