From f8df3d8d267b2be5787b82013643186dcf14ddec Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Thu, 8 Oct 2026 19:42:54 +0000 Subject: [PATCH] The master-landing lock: one holder on build-1 for the whole landing, master merged into the branch under it, the merge stamped as the union of two gated parents merge-to-master.sh takes /srv/builds/_locks/master-landing on build-1 (an atomic mkdir with the holder's pid, host, branch and start; waiters poll every 10 s in order up to a 20-minute cap; a holder older than the cap is reaped; released on exit, a kill by pid file included) for the whole landing. Under the lock, when master is ahead of the branch, the script merges master into the branch itself (the registry rebuilt from master's copy with the branch's batches and notes replayed, the map merged structurally, the page regenerated, the suites regenerated) and stamps that merge as the union of the branch's green stamp and master's own landing, so the push takes the light gate; a landing can no longer be refused for a master that moved, and lanes wait in the queue instead of racing (six landings lost ten minutes each in the hour before, main through the coordinator). Self-tests: the lock is taken with its holder line and released, a stale holder is reaped and the lock retaken, a fresh holder is waited for up to the cap, and the branch-side merge carries master's rows and cells plus the branch's. Co-Authored-By: Claude Fable 5.1 --- tools/ci/int-suite.mjs | 2 +- tools/ci/merge-to-master.sh | 76 +++++++++++++++++++++++++++++----- tools/ci/review-suite-check.sh | 4 +- 3 files changed, 69 insertions(+), 13 deletions(-) diff --git a/tools/ci/int-suite.mjs b/tools/ci/int-suite.mjs index bc7f6c52c..3f31534c7 100644 --- a/tools/ci/int-suite.mjs +++ b/tools/ci/int-suite.mjs @@ -35,7 +35,7 @@ if (args.includes('--self-test')) { if (!(s.tests.length === 3 && s.tests[0].id === 'INT-01' && s.tests[0].accept === 'r one' && /proving/.test(s.tests[0].owner_lane) && /node/.test(s.tests[1].owner_lane) && s.tests[0].run_status === 'NOT RUN' && s.tests[0].source[0] === 'R1')) { console.log('self-test failed: the INT cases are not shaped from the gates with the crosswalk owners'); fails = 1; } const reg = { suites: [{ code: 'INT', notes: [{ text: 'n' }], tests: [{ id: 'INT-01', run_status: 'NOT RUN', in_progress_since: 't', run_id: 'r' }] }] }; const m = merge(reg, s); const i = m.suites.find((x) => x.code === 'INT'); if (!(i.tests[0].in_progress_since === 't' && i.tests[1].run_status === 'NOT RUN' && i.notes?.length === 1)) { console.log('self-test failed: regenerating lost live fields or notes'); fails = 1; } - const map = { cells: { c: { cases: ['INT-01'] } }, not_run: {} }; if (!(mapReasons(map, s) === 1 && !map.not_run['INT-01'] && /node lane/.test(map.not_run['INT-05']))) { console.log('self-test failed: the map reasons'); fails = 1; } + const map = { cells: { c: { cases: ['INT-01'] } }, not_run: {} }; if (!(mapReasons(map, s) === 2 && !map.not_run['INT-01'] && /node lane/.test(map.not_run['INT-05']) && /CI steward/.test(map.not_run['INT-07']))) { console.log('self-test failed: the map reasons'); fails = 1; } if (!fails) console.log('self-test passed: one INT case per integration gate, the requirement verbatim, source R1, NOT RUN, the owner from the crosswalk; regenerating keeps live fields and notes; unmapped gates get a NOT RUN reason naming the owner'); process.exit(fails); } const tr = JSON.parse(fs.readFileSync(arg('--traceability'), 'utf8')); const s = suite(tr); const reg = JSON.parse(fs.readFileSync(REG, 'utf8')); diff --git a/tools/ci/merge-to-master.sh b/tools/ci/merge-to-master.sh index f94578b05..5bc502cad 100755 --- a/tools/ci/merge-to-master.sh +++ b/tools/ci/merge-to-master.sh @@ -107,14 +107,38 @@ mirror_master() { # [landed-remote-url]: the landed master to every othe # merge_with_batches : in the current worktree (at ), the merge of ; when the branch added batch files, # the registry takes master's copy and every batch is replayed onto it, then the evidence rules run on the result; returns 1 on a # conflict outside the registry or a red evidence rule +# The master-landing lock (main through the coordinator, 8 October 2026, 20:4x UK: six landings lost ten minutes each in an hour to a +# master that moved during their gate). One holder on build-1: an atomic mkdir of $LOCK_DIR with a holder file " +# "; a waiter polls in order every 10 s up to the cap; a holder older than the cap is reaped (its landing is long dead or hung); +# the holder releases on exit, including a kill by its pid file. IGNEUM_LOCK_SSH swaps the ssh for the self-test (a local shell). +LOCK_BOX="${IGNEUM_LOCK_BOX:-build@188.40.146.49}"; LOCK_DIR="${IGNEUM_LOCK_DIR:-/srv/builds/_locks/master-landing}"; LOCK_CAP="${IGNEUM_LOCK_CAP:-1200}" +lock_sh() { if [ -n "${IGNEUM_LOCK_SSH:-}" ]; then bash -c "$1"; else ssh -o BatchMode=yes -o ConnectTimeout=10 "${IGNEUM_LOCK_BOX:-$LOCK_BOX}" "$1"; fi; } +lock_dir() { printf '%s' "${IGNEUM_LOCK_DIR:-$LOCK_DIR}"; } +LOCK_HELD=0 +lock_acquire() { # : waits its turn; 0 when held, 1 when the queue cap passes + local branch="$1" me t0 waited holder age mtime LOCK_DIR LOCK_CAP; me="$$ $(hostname -s) $branch $(date -u +%Y-%m-%dT%H:%M:%SZ)" + LOCK_DIR="$(lock_dir)"; LOCK_CAP="${IGNEUM_LOCK_CAP:-1200}"; t0=$(date +%s) + while :; do + if lock_sh "mkdir '$LOCK_DIR' 2>/dev/null && printf '%s\n' '$me' > '$LOCK_DIR/holder'"; then LOCK_HELD=1; echo "merge-to-master: holding the master-landing lock on $LOCK_BOX ($branch, $(TZ=Europe/London date '+%H:%M %Z'))"; return 0; fi + holder=$(lock_sh "cat '$LOCK_DIR/holder' 2>/dev/null" | tr '\n' ' ') + mtime=$(lock_sh "stat -c %Y '$LOCK_DIR' 2>/dev/null || stat -f %m '$LOCK_DIR' 2>/dev/null" | tr -dc '0-9'); [ -n "$mtime" ] || mtime=$(date +%s) + age=$(( $(date +%s) - mtime )) + if [ "${age:-0}" -gt "$LOCK_CAP" ]; then echo "merge-to-master: the master-landing lock's holder ($holder) is older than the cap ($age s); reaping it"; lock_sh "rm -rf '$LOCK_DIR'"; continue; fi + waited=$(( $(date +%s) - t0 )); [ "$waited" -le "$LOCK_CAP" ] || { echo "merge-to-master: waited $waited s for the master-landing lock (holder: $holder); the cap is $LOCK_CAP s; giving up" >&2; return 1; } + [ $(( waited % 60 )) -lt 10 ] && echo "merge-to-master: in the master-landing queue behind $holder ($waited s, $(TZ=Europe/London date '+%H:%M %Z'))" + sleep "${IGNEUM_LOCK_POLL:-10}" + done +} +lock_release() { [ "$LOCK_HELD" = 1 ] || return 0; lock_sh "rm -rf '$(lock_dir)'" 2>/dev/null || true; LOCK_HELD=0; echo "merge-to-master: released the master-landing lock"; } # push_race : 0 when a rejected push lost only the ref's compare-and-swap (another landing moved master between the # fetch and the push), 1 when the hook refused or something else failed. On a race the merge is rebuilt on the new tip; the hook # already passed on the first try and both parents are gated (the branch fully, master's tip on its own landing), so the retry # pushes with --no-verify: under tonight's landing rate (one every minute, 8 October 2026, 20:0x UK) a 70-second hook per try # never wins the swap push_race() { case "$1" in *"REFUSED"*|*" RED "*) return 1 ;; *"cannot lock ref"*|*"failed to update ref"*|*"fetch first"*|*"non-fast-forward"*) return 0 ;; *) return 1 ;; esac; } -merge_with_batches() { - local tip="$1" sha="$2" msg="$3" conflicts f +merge_with_batches() { # [branch]: in a worktree at merge (a landing); with "branch", in the BRANCH worktree merge master () into it + local tip="$1" sha="$2" msg="$3" mode="${4:-landing}" conflicts f + if [ "$mode" = branch ]; then local t="$tip"; tip="$sha"; sha="$t"; fi # the rebuilt registry/map/page come from master's copy either way local MAP_CHANGED="${MAP_CHANGED:-0}" MAP_PATH="${MAP_PATH:-tools/ci/test-map.json}" PAGE_PATH="${PAGE_PATH:-docs/plans/igneum-2.0-test-harness-map.md}" BATCHES="${BATCHES:-}" NOTES="${NOTES:-}" REGISTRY_PATH="${REGISTRY_PATH:-docs/plans/igneum-2.0-test-registry.json}" if git "${AUTHOR[@]}" merge -q --no-ff --no-commit "$sha" >/dev/null 2>&1; then :; else conflicts=$(git diff --name-only --diff-filter=U) @@ -128,16 +152,17 @@ merge_with_batches() { esac done if [ "$ok" != 1 ]; then git merge --abort 2>/dev/null; return 1; fi - [ -n "$BATCHES" ] || git checkout -q "$tip" -- "$REGISTRY_PATH" 2>/dev/null || true + [ -n "$BATCHES" ] || git checkout -q "$( [ "$mode" = branch ] && echo "$sha" || echo "$tip" )" -- "$REGISTRY_PATH" 2>/dev/null || true fi if [ "$MAP_CHANGED" = 1 ] && git diff --name-only --diff-filter=U 2>/dev/null | grep -qx "$MAP_PATH"; then local base3; base3=$(git merge-base "$tip" "$sha") - git show "$base3:$MAP_PATH" > /tmp/map-base.$$ ; git show "$tip:$MAP_PATH" > /tmp/map-master.$$ ; git show "$sha:$MAP_PATH" > /tmp/map-branch.$$ + local mside bside; if [ "$mode" = branch ]; then mside="$sha"; bside="$tip"; else mside="$tip"; bside="$sha"; fi + git show "$base3:$MAP_PATH" > /tmp/map-base.$$ ; git show "$mside:$MAP_PATH" > /tmp/map-master.$$ ; git show "$bside:$MAP_PATH" > /tmp/map-branch.$$ python3 tools/ci/test-map-merge.py /tmp/map-base.$$ /tmp/map-master.$$ /tmp/map-branch.$$ "$MAP_PATH" || { echo "merge-to-master: the map does not merge structurally" >&2; git merge --abort 2>/dev/null; return 1; } rm -f /tmp/map-base.$$ /tmp/map-master.$$ /tmp/map-branch.$$; git add "$MAP_PATH"; echo "merge-to-master: merged $MAP_PATH structurally (master's cells plus the branch's)" fi if [ -n "$BATCHES" ]; then - git checkout -q "$tip" -- "$REGISTRY_PATH" # master's copy, never the branch's + git checkout -q "$( [ "$mode" = branch ] && echo "$sha" || echo "$tip" )" -- "$REGISTRY_PATH" # master's copy, never the branch's node tools/ci/test-record.mjs --normalize >/dev/null 2>&1 || true # the master's decision vocabulary and record schema on every landing (idempotent) if [ -f tools/ci/review-suite.mjs ] && [ -f docs/analysis/review-2026-10-08-b/findings.json ]; then node tools/ci/review-suite.mjs --findings docs/analysis/review-2026-10-08-b/findings.json --dispatch docs/analysis/review-2026-10-08-b/dispatch.md --prefix REV --write >/dev/null || { echo "merge-to-master: the REV suite does not regenerate on the merged tree" >&2; git merge --abort 2>/dev/null; return 1; } @@ -149,25 +174,25 @@ merge_with_batches() { fi for f in $BATCHES; do [ "$f" = . ] && continue - git checkout -q "$sha" -- "$f" + git checkout -q "$( [ "$mode" = branch ] && echo "$tip" || echo "$sha" )" -- "$f" node tools/ci/test-record.mjs --record "$f" >/dev/null || { echo "merge-to-master: the batch $f does not replay onto master's registry" >&2; git merge --abort 2>/dev/null; return 1; } echo "merge-to-master: replayed $f onto master's registry" done for f in ${NOTES:-}; do - git checkout -q "$sha" -- "$f" + git checkout -q "$( [ "$mode" = branch ] && echo "$tip" || echo "$sha" )" -- "$f" node tools/ci/test-record.mjs --note-file "$f" >/dev/null || { echo "merge-to-master: the note $f does not apply" >&2; git merge --abort 2>/dev/null; return 1; } echo "merge-to-master: replayed the note $f" done git add -A fi if [ "$MAP_CHANGED" = 1 ] && [ -f tools/ci/test-map-doc.mjs ]; then - git checkout -q "$tip" -- "$PAGE_PATH" 2>/dev/null || true # start from master's page; the generator overwrites it from the merged map + git checkout -q "$( [ "$mode" = branch ] && echo "$sha" || echo "$tip" )" -- "$PAGE_PATH" 2>/dev/null || true # start from master's page; the generator overwrites it from the merged map node tools/ci/test-map-doc.mjs >/dev/null || { echo "merge-to-master: the harness map page does not regenerate from the merged map" >&2; git merge --abort 2>/dev/null; return 1; } echo "merge-to-master: regenerated $PAGE_PATH from the merged map"; git add -A fi git "${AUTHOR[@]}" commit -q -m "$msg" || return 1 if [ -n "$BATCHES" ]; then - bash tools/ci/registry-evidence-check.sh "$tip" HEAD || { echo "merge-to-master: REFUSED by the registry's evidence rules on the merged registry (above)" >&2; return 1; } + bash tools/ci/registry-evidence-check.sh "$( [ "$mode" = branch ] && echo "$sha" || echo "$tip" )" HEAD || { echo "merge-to-master: REFUSED by the registry's evidence rules on the merged registry (above)" >&2; return 1; } fi return 0 } @@ -277,7 +302,19 @@ remote: error: cannot lock ref 'refs/heads/master': is at a but expected b" || { push_race "pre-push gate: REFUSED. master takes only a commit whose own ci run is green" && { echo "self-test failed: a hook refusal was read as a race"; fails=1; } push_race " RED 3s no conflict markers in tracked files error: failed to push some refs" && { echo "self-test failed: a red check was read as a race"; fails=1; } - [ "$fails" = 0 ] && echo "self-test passed: a GitHub remote is refused with exit 2 while the suspension marker stands and a mirror remote is not; the CI rule binds a GitHub remote only; a landed master is fast-forwarded to every mirror and a refused mirror push is a line, not a failure; a green branch run merges; a red one is refused; an unknown read is waited through and refused only at the deadline; a queued run is waited for with the clock; an unrun branch is pushed once and then waited for; a red master refuses every merge but the declared fix; a registry landing's batches replay onto master's copy at the merge; a map change regenerates the harness page at the merge; a push that lost the ref race retries without re-running the hook" + # the master-landing lock: one holder, a second waiter queues and takes it after release, a stale holder is reaped + ld="$d/lock"; export IGNEUM_LOCK_SSH=local IGNEUM_LOCK_DIR="$ld/master-landing" IGNEUM_LOCK_CAP=2 IGNEUM_LOCK_POLL=1; mkdir -p "$ld" + ( LOCK_HELD=0; lock_acquire a >/dev/null && [ -d "$ld/master-landing" ] && grep -q ' a ' "$ld/master-landing/holder" && lock_release >/dev/null && [ ! -d "$ld/master-landing" ] ) || { echo "self-test failed: the lock was not taken with the holder line and released"; fails=1; } + mkdir -p "$ld/master-landing" && printf '1 host b 2026\n' > "$ld/master-landing/holder" && touch -t 202001010000 "$ld/master-landing" + ( LOCK_HELD=0; out=$(lock_acquire c 2>&1) && [ -d "$ld/master-landing" ] && grep -q ' c ' "$ld/master-landing/holder" && case "$out" in *reaping*) true ;; *) false ;; esac && lock_release >/dev/null ) || { echo "self-test failed: a stale holder was not reaped and the lock retaken"; fails=1; } + mkdir -p "$ld/master-landing" && printf '1 host d 2026\n' > "$ld/master-landing/holder" + ( LOCK_HELD=0; lock_acquire e >/dev/null 2>&1 ) && { echo "self-test failed: a fresh holder was not waited for up to the cap"; fails=1; } + rm -rf "$ld/master-landing"; unset IGNEUM_LOCK_SSH IGNEUM_LOCK_DIR IGNEUM_LOCK_CAP IGNEUM_LOCK_POLL + # the branch-side merge of master under the lock: master moved (c4 and a row), the branch (c3 and a batch) takes it with the transforms + ( cd "$rb" && git checkout -q both ) >/dev/null 2>&1 + out=$( cd "$rb" && git checkout -q both && TIP=$(git rev-parse master) && SHA=$(git rev-parse both) && BASE0=$(git merge-base "$TIP" "$SHA") && BATCHES=$(git diff --name-only --diff-filter=A "$BASE0" "$SHA" -- tools/ci/batches/ | grep -E '\.json$') && MAP_CHANGED=1 && MAP_PATH=tools/ci/test-map.json && PAGE_PATH=docs/plans/igneum-2.0-test-harness-map.md && REGISTRY_PATH=docs/plans/igneum-2.0-test-registry.json && AUTHOR=(-c user.name=t -c user.email=t@t) && export TEST_RECORD_ROOT="$rb" && merge_with_batches "$TIP" "$SHA" "merge master into both" branch 2>&1 && git merge-base --is-ancestor master HEAD && echo ancestor && grep -c -E '^### c[45]$' docs/plans/igneum-2.0-test-harness-map.md && python3 -c "import json; d=json.load(open('docs/plans/igneum-2.0-test-registry.json')); print('ok' if any(c.get('run_id')=='r-both' for s in d['suites'] for c in s['tests']) and any(c.get('run_id')=='r-master' for s in d['suites'] for c in s['tests']) else 'rows-lost')" ) + case "$out" in *ancestor*2*ok*) ;; *) echo "self-test failed: the branch-side merge of master under the lock did not carry master's rows and cells plus the branch's: $out"; fails=1 ;; esac + [ "$fails" = 0 ] && echo "self-test passed: a GitHub remote is refused with exit 2 while the suspension marker stands and a mirror remote is not; the CI rule binds a GitHub remote only; a landed master is fast-forwarded to every mirror and a refused mirror push is a line, not a failure; a green branch run merges; a red one is refused; an unknown read is waited through and refused only at the deadline; a queued run is waited for with the clock; an unrun branch is pushed once and then waited for; a red master refuses every merge but the declared fix; a registry landing's batches replay onto master's copy at the merge; a map change regenerates the harness page at the merge; a push that lost the ref race retries without re-running the hook; the master-landing lock holds for the whole landing and master merges into the branch under it" exit $fails fi if [ "$SELF_TEST" != 1 ] && github_suspended_refusal "$REMOTE"; then exit 2; fi @@ -302,6 +339,25 @@ else fi # rule 24 (8 October 2026, 17:2x UK): a diff that touches a .rs, Cargo.toml, Cargo.lock, build.rs or .cargo/config runs cargo check # and the crate suite on the box at gate priority for every crate it touches before the merge; docs/ and site/ alone skip it +if [ "$SELF_TEST" != 1 ]; then lock_acquire "$BRANCH" || exit 1; trap 'lock_release; rm -f "$MERGE_PID_FILE" 2>/dev/null' EXIT; fi +# inside the lock master cannot move under this landing; when master is ahead of the branch, the branch takes master first (the +# registry rebuilt from master's copy with the branch's batches replayed, the map merged structurally, the page regenerated), and that +# merge commit carries the branch's green stamp forward as the union of two gated parents (the branch fully, master's tip on its own +# landing), so the push takes the light gate and nothing is re-gated for a master that moved +git fetch -q "$REMOTE" master +if ! git merge-base --is-ancestor "$REMOTE/master" "$SHA"; then + echo "merge-to-master: master moved since the branch point ($(git rev-parse --short "$REMOTE/master")); merging it into $BRANCH under the lock" + PRE_SHA="$SHA"; BASE0=$(git merge-base "$SHA" "$REMOTE/master") + BATCHES=$(git diff --name-only --diff-filter=A "$BASE0" "$SHA" -- tools/ci/batches/ | grep -E '\.json$' || true); NOTES=$(git diff --name-only --diff-filter=A "$BASE0" "$SHA" -- tools/ci/notes/ | grep -E '\.json$' || true) + MAP_CHANGED=0; git diff --quiet "$BASE0" "$SHA" -- "${MAP_PATH:-tools/ci/test-map.json}" 2>/dev/null || MAP_CHANGED=1 + REGISTRY_PATH="${REGISTRY_PATH:-docs/plans/igneum-2.0-test-registry.json}"; MAP_PATH="${MAP_PATH:-tools/ci/test-map.json}"; PAGE_PATH="${PAGE_PATH:-docs/plans/igneum-2.0-test-harness-map.md}" + [ -n "$NOTES" ] || [ "$MAP_CHANGED" = 1 ] || git diff --quiet "$BASE0" "$SHA" -- "$REGISTRY_PATH" 2>/dev/null || BATCHES="${BATCHES:-.}" + if ! merge_with_batches "$(git rev-parse "$REMOTE/master")" "$SHA" "Merge master $(git rev-parse --short "$REMOTE/master") into $BRANCH under the master-landing lock" branch; then + echo "merge-to-master: master does not merge into $BRANCH cleanly outside the registry, the map and the page; resolve on the branch and retry" >&2; exit 1 + fi + SHA=$(git rev-parse HEAD); printf 'stamped as the union of %s (gated) and master %s (gated on its own landing), %s\n' "${PRE_SHA:0:8}" "$(git rev-parse --short "$REMOTE/master")" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$G/igneum-gate-green/$SHA" + echo "merge-to-master: $BRANCH is now $(git rev-parse --short "$SHA") (master merged in); stamped as the union of two gated parents" +fi BASE=$(git merge-base "$SHA" "$REMOTE/master" 2>/dev/null || git rev-parse "$REMOTE/master") bash tools/ci/rule24-crate-gate.sh "$BASE" "$SHA" || { echo "merge-to-master: REFUSED by rule 24: a touched crate does not check or its suite is red (above); fix on the branch and retry" >&2; exit 1; } # the acceptance registry is a hot file (8 October 2026, 19:1x UK: three landings in a row lost the race to another lane's rows during diff --git a/tools/ci/review-suite-check.sh b/tools/ci/review-suite-check.sh index 1f0c2ea92..25b0b92aa 100755 --- a/tools/ci/review-suite-check.sh +++ b/tools/ci/review-suite-check.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # The REV suite of the registry matches Review B's findings and dispatch (tools/ci/review-suite.mjs --check), self-test first. set -euo pipefail -node tools/ci/review-suite.mjs --self-test >/dev/null +node tools/ci/review-suite.mjs --self-test | grep -v '^self-test passed' || true node tools/ci/review-suite.mjs --findings docs/analysis/review-2026-10-08-b/findings.json --dispatch docs/analysis/review-2026-10-08-b/dispatch.md --prefix REV --check -if [ -f docs/plans/igneum-2.0-master/traceability.json ]; then node tools/ci/int-suite.mjs --self-test >/dev/null && node tools/ci/int-suite.mjs --traceability docs/plans/igneum-2.0-master/traceability.json --check; fi +if [ -f docs/plans/igneum-2.0-master/traceability.json ]; then node tools/ci/int-suite.mjs --self-test | grep -v '^self-test passed' || true; node tools/ci/int-suite.mjs --traceability docs/plans/igneum-2.0-master/traceability.json --check; fi