diff --git a/tools/ci/int-suite.mjs b/tools/ci/int-suite.mjs index bc7f6c52c..3f31534c7 100644 --- a/tools/ci/int-suite.mjs +++ b/tools/ci/int-suite.mjs @@ -35,7 +35,7 @@ if (args.includes('--self-test')) { if (!(s.tests.length === 3 && s.tests[0].id === 'INT-01' && s.tests[0].accept === 'r one' && /proving/.test(s.tests[0].owner_lane) && /node/.test(s.tests[1].owner_lane) && s.tests[0].run_status === 'NOT RUN' && s.tests[0].source[0] === 'R1')) { console.log('self-test failed: the INT cases are not shaped from the gates with the crosswalk owners'); fails = 1; } const reg = { suites: [{ code: 'INT', notes: [{ text: 'n' }], tests: [{ id: 'INT-01', run_status: 'NOT RUN', in_progress_since: 't', run_id: 'r' }] }] }; const m = merge(reg, s); const i = m.suites.find((x) => x.code === 'INT'); if (!(i.tests[0].in_progress_since === 't' && i.tests[1].run_status === 'NOT RUN' && i.notes?.length === 1)) { console.log('self-test failed: regenerating lost live fields or notes'); fails = 1; } - const map = { cells: { c: { cases: ['INT-01'] } }, not_run: {} }; if (!(mapReasons(map, s) === 1 && !map.not_run['INT-01'] && /node lane/.test(map.not_run['INT-05']))) { console.log('self-test failed: the map reasons'); fails = 1; } + const map = { cells: { c: { cases: ['INT-01'] } }, not_run: {} }; if (!(mapReasons(map, s) === 2 && !map.not_run['INT-01'] && /node lane/.test(map.not_run['INT-05']) && /CI steward/.test(map.not_run['INT-07']))) { console.log('self-test failed: the map reasons'); fails = 1; } if (!fails) console.log('self-test passed: one INT case per integration gate, the requirement verbatim, source R1, NOT RUN, the owner from the crosswalk; regenerating keeps live fields and notes; unmapped gates get a NOT RUN reason naming the owner'); process.exit(fails); } const tr = JSON.parse(fs.readFileSync(arg('--traceability'), 'utf8')); const s = suite(tr); const reg = JSON.parse(fs.readFileSync(REG, 'utf8')); diff --git a/tools/ci/merge-to-master.sh b/tools/ci/merge-to-master.sh index f94578b05..5bc502cad 100755 --- a/tools/ci/merge-to-master.sh +++ b/tools/ci/merge-to-master.sh @@ -107,14 +107,38 @@ mirror_master() { # [landed-remote-url]: the landed master to every othe # merge_with_batches : in the current worktree (at ), the merge of ; when the branch added batch files, # the registry takes master's copy and every batch is replayed onto it, then the evidence rules run on the result; returns 1 on a # conflict outside the registry or a red evidence rule +# The master-landing lock (main through the coordinator, 8 October 2026, 20:4x UK: six landings lost ten minutes each in an hour to a +# master that moved during their gate). One holder on build-1: an atomic mkdir of $LOCK_DIR with a holder file " +# "; a waiter polls in order every 10 s up to the cap; a holder older than the cap is reaped (its landing is long dead or hung); +# the holder releases on exit, including a kill by its pid file. IGNEUM_LOCK_SSH swaps the ssh for the self-test (a local shell). +LOCK_BOX="${IGNEUM_LOCK_BOX:-build@188.40.146.49}"; LOCK_DIR="${IGNEUM_LOCK_DIR:-/srv/builds/_locks/master-landing}"; LOCK_CAP="${IGNEUM_LOCK_CAP:-1200}" +lock_sh() { if [ -n "${IGNEUM_LOCK_SSH:-}" ]; then bash -c "$1"; else ssh -o BatchMode=yes -o ConnectTimeout=10 "${IGNEUM_LOCK_BOX:-$LOCK_BOX}" "$1"; fi; } +lock_dir() { printf '%s' "${IGNEUM_LOCK_DIR:-$LOCK_DIR}"; } +LOCK_HELD=0 +lock_acquire() { # : waits its turn; 0 when held, 1 when the queue cap passes + local branch="$1" me t0 waited holder age mtime LOCK_DIR LOCK_CAP; me="$$ $(hostname -s) $branch $(date -u +%Y-%m-%dT%H:%M:%SZ)" + LOCK_DIR="$(lock_dir)"; LOCK_CAP="${IGNEUM_LOCK_CAP:-1200}"; t0=$(date +%s) + while :; do + if lock_sh "mkdir '$LOCK_DIR' 2>/dev/null && printf '%s\n' '$me' > '$LOCK_DIR/holder'"; then LOCK_HELD=1; echo "merge-to-master: holding the master-landing lock on $LOCK_BOX ($branch, $(TZ=Europe/London date '+%H:%M %Z'))"; return 0; fi + holder=$(lock_sh "cat '$LOCK_DIR/holder' 2>/dev/null" | tr '\n' ' ') + mtime=$(lock_sh "stat -c %Y '$LOCK_DIR' 2>/dev/null || stat -f %m '$LOCK_DIR' 2>/dev/null" | tr -dc '0-9'); [ -n "$mtime" ] || mtime=$(date +%s) + age=$(( $(date +%s) - mtime )) + if [ "${age:-0}" -gt "$LOCK_CAP" ]; then echo "merge-to-master: the master-landing lock's holder ($holder) is older than the cap ($age s); reaping it"; lock_sh "rm -rf '$LOCK_DIR'"; continue; fi + waited=$(( $(date +%s) - t0 )); [ "$waited" -le "$LOCK_CAP" ] || { echo "merge-to-master: waited $waited s for the master-landing lock (holder: $holder); the cap is $LOCK_CAP s; giving up" >&2; return 1; } + [ $(( waited % 60 )) -lt 10 ] && echo "merge-to-master: in the master-landing queue behind $holder ($waited s, $(TZ=Europe/London date '+%H:%M %Z'))" + sleep "${IGNEUM_LOCK_POLL:-10}" + done +} +lock_release() { [ "$LOCK_HELD" = 1 ] || return 0; lock_sh "rm -rf '$(lock_dir)'" 2>/dev/null || true; LOCK_HELD=0; echo "merge-to-master: released the master-landing lock"; } # push_race : 0 when a rejected push lost only the ref's compare-and-swap (another landing moved master between the # fetch and the push), 1 when the hook refused or something else failed. On a race the merge is rebuilt on the new tip; the hook # already passed on the first try and both parents are gated (the branch fully, master's tip on its own landing), so the retry # pushes with --no-verify: under tonight's landing rate (one every minute, 8 October 2026, 20:0x UK) a 70-second hook per try # never wins the swap push_race() { case "$1" in *"REFUSED"*|*" RED "*) return 1 ;; *"cannot lock ref"*|*"failed to update ref"*|*"fetch first"*|*"non-fast-forward"*) return 0 ;; *) return 1 ;; esac; } -merge_with_batches() { - local tip="$1" sha="$2" msg="$3" conflicts f +merge_with_batches() { # [branch]: in a worktree at merge (a landing); with "branch", in the BRANCH worktree merge master () into it + local tip="$1" sha="$2" msg="$3" mode="${4:-landing}" conflicts f + if [ "$mode" = branch ]; then local t="$tip"; tip="$sha"; sha="$t"; fi # the rebuilt registry/map/page come from master's copy either way local MAP_CHANGED="${MAP_CHANGED:-0}" MAP_PATH="${MAP_PATH:-tools/ci/test-map.json}" PAGE_PATH="${PAGE_PATH:-docs/plans/igneum-2.0-test-harness-map.md}" BATCHES="${BATCHES:-}" NOTES="${NOTES:-}" REGISTRY_PATH="${REGISTRY_PATH:-docs/plans/igneum-2.0-test-registry.json}" if git "${AUTHOR[@]}" merge -q --no-ff --no-commit "$sha" >/dev/null 2>&1; then :; else conflicts=$(git diff --name-only --diff-filter=U) @@ -128,16 +152,17 @@ merge_with_batches() { esac done if [ "$ok" != 1 ]; then git merge --abort 2>/dev/null; return 1; fi - [ -n "$BATCHES" ] || git checkout -q "$tip" -- "$REGISTRY_PATH" 2>/dev/null || true + [ -n "$BATCHES" ] || git checkout -q "$( [ "$mode" = branch ] && echo "$sha" || echo "$tip" )" -- "$REGISTRY_PATH" 2>/dev/null || true fi if [ "$MAP_CHANGED" = 1 ] && git diff --name-only --diff-filter=U 2>/dev/null | grep -qx "$MAP_PATH"; then local base3; base3=$(git merge-base "$tip" "$sha") - git show "$base3:$MAP_PATH" > /tmp/map-base.$$ ; git show "$tip:$MAP_PATH" > /tmp/map-master.$$ ; git show "$sha:$MAP_PATH" > /tmp/map-branch.$$ + local mside bside; if [ "$mode" = branch ]; then mside="$sha"; bside="$tip"; else mside="$tip"; bside="$sha"; fi + git show "$base3:$MAP_PATH" > /tmp/map-base.$$ ; git show "$mside:$MAP_PATH" > /tmp/map-master.$$ ; git show "$bside:$MAP_PATH" > /tmp/map-branch.$$ python3 tools/ci/test-map-merge.py /tmp/map-base.$$ /tmp/map-master.$$ /tmp/map-branch.$$ "$MAP_PATH" || { echo "merge-to-master: the map does not merge structurally" >&2; git merge --abort 2>/dev/null; return 1; } rm -f /tmp/map-base.$$ /tmp/map-master.$$ /tmp/map-branch.$$; git add "$MAP_PATH"; echo "merge-to-master: merged $MAP_PATH structurally (master's cells plus the branch's)" fi if [ -n "$BATCHES" ]; then - git checkout -q "$tip" -- "$REGISTRY_PATH" # master's copy, never the branch's + git checkout -q "$( [ "$mode" = branch ] && echo "$sha" || echo "$tip" )" -- "$REGISTRY_PATH" # master's copy, never the branch's node tools/ci/test-record.mjs --normalize >/dev/null 2>&1 || true # the master's decision vocabulary and record schema on every landing (idempotent) if [ -f tools/ci/review-suite.mjs ] && [ -f docs/analysis/review-2026-10-08-b/findings.json ]; then node tools/ci/review-suite.mjs --findings docs/analysis/review-2026-10-08-b/findings.json --dispatch docs/analysis/review-2026-10-08-b/dispatch.md --prefix REV --write >/dev/null || { echo "merge-to-master: the REV suite does not regenerate on the merged tree" >&2; git merge --abort 2>/dev/null; return 1; } @@ -149,25 +174,25 @@ merge_with_batches() { fi for f in $BATCHES; do [ "$f" = . ] && continue - git checkout -q "$sha" -- "$f" + git checkout -q "$( [ "$mode" = branch ] && echo "$tip" || echo "$sha" )" -- "$f" node tools/ci/test-record.mjs --record "$f" >/dev/null || { echo "merge-to-master: the batch $f does not replay onto master's registry" >&2; git merge --abort 2>/dev/null; return 1; } echo "merge-to-master: replayed $f onto master's registry" done for f in ${NOTES:-}; do - git checkout -q "$sha" -- "$f" + git checkout -q "$( [ "$mode" = branch ] && echo "$tip" || echo "$sha" )" -- "$f" node tools/ci/test-record.mjs --note-file "$f" >/dev/null || { echo "merge-to-master: the note $f does not apply" >&2; git merge --abort 2>/dev/null; return 1; } echo "merge-to-master: replayed the note $f" done git add -A fi if [ "$MAP_CHANGED" = 1 ] && [ -f tools/ci/test-map-doc.mjs ]; then - git checkout -q "$tip" -- "$PAGE_PATH" 2>/dev/null || true # start from master's page; the generator overwrites it from the merged map + git checkout -q "$( [ "$mode" = branch ] && echo "$sha" || echo "$tip" )" -- "$PAGE_PATH" 2>/dev/null || true # start from master's page; the generator overwrites it from the merged map node tools/ci/test-map-doc.mjs >/dev/null || { echo "merge-to-master: the harness map page does not regenerate from the merged map" >&2; git merge --abort 2>/dev/null; return 1; } echo "merge-to-master: regenerated $PAGE_PATH from the merged map"; git add -A fi git "${AUTHOR[@]}" commit -q -m "$msg" || return 1 if [ -n "$BATCHES" ]; then - bash tools/ci/registry-evidence-check.sh "$tip" HEAD || { echo "merge-to-master: REFUSED by the registry's evidence rules on the merged registry (above)" >&2; return 1; } + bash tools/ci/registry-evidence-check.sh "$( [ "$mode" = branch ] && echo "$sha" || echo "$tip" )" HEAD || { echo "merge-to-master: REFUSED by the registry's evidence rules on the merged registry (above)" >&2; return 1; } fi return 0 } @@ -277,7 +302,19 @@ remote: error: cannot lock ref 'refs/heads/master': is at a but expected b" || { push_race "pre-push gate: REFUSED. master takes only a commit whose own ci run is green" && { echo "self-test failed: a hook refusal was read as a race"; fails=1; } push_race " RED 3s no conflict markers in tracked files error: failed to push some refs" && { echo "self-test failed: a red check was read as a race"; fails=1; } - [ "$fails" = 0 ] && echo "self-test passed: a GitHub remote is refused with exit 2 while the suspension marker stands and a mirror remote is not; the CI rule binds a GitHub remote only; a landed master is fast-forwarded to every mirror and a refused mirror push is a line, not a failure; a green branch run merges; a red one is refused; an unknown read is waited through and refused only at the deadline; a queued run is waited for with the clock; an unrun branch is pushed once and then waited for; a red master refuses every merge but the declared fix; a registry landing's batches replay onto master's copy at the merge; a map change regenerates the harness page at the merge; a push that lost the ref race retries without re-running the hook" + # the master-landing lock: one holder, a second waiter queues and takes it after release, a stale holder is reaped + ld="$d/lock"; export IGNEUM_LOCK_SSH=local IGNEUM_LOCK_DIR="$ld/master-landing" IGNEUM_LOCK_CAP=2 IGNEUM_LOCK_POLL=1; mkdir -p "$ld" + ( LOCK_HELD=0; lock_acquire a >/dev/null && [ -d "$ld/master-landing" ] && grep -q ' a ' "$ld/master-landing/holder" && lock_release >/dev/null && [ ! -d "$ld/master-landing" ] ) || { echo "self-test failed: the lock was not taken with the holder line and released"; fails=1; } + mkdir -p "$ld/master-landing" && printf '1 host b 2026\n' > "$ld/master-landing/holder" && touch -t 202001010000 "$ld/master-landing" + ( LOCK_HELD=0; out=$(lock_acquire c 2>&1) && [ -d "$ld/master-landing" ] && grep -q ' c ' "$ld/master-landing/holder" && case "$out" in *reaping*) true ;; *) false ;; esac && lock_release >/dev/null ) || { echo "self-test failed: a stale holder was not reaped and the lock retaken"; fails=1; } + mkdir -p "$ld/master-landing" && printf '1 host d 2026\n' > "$ld/master-landing/holder" + ( LOCK_HELD=0; lock_acquire e >/dev/null 2>&1 ) && { echo "self-test failed: a fresh holder was not waited for up to the cap"; fails=1; } + rm -rf "$ld/master-landing"; unset IGNEUM_LOCK_SSH IGNEUM_LOCK_DIR IGNEUM_LOCK_CAP IGNEUM_LOCK_POLL + # the branch-side merge of master under the lock: master moved (c4 and a row), the branch (c3 and a batch) takes it with the transforms + ( cd "$rb" && git checkout -q both ) >/dev/null 2>&1 + out=$( cd "$rb" && git checkout -q both && TIP=$(git rev-parse master) && SHA=$(git rev-parse both) && BASE0=$(git merge-base "$TIP" "$SHA") && BATCHES=$(git diff --name-only --diff-filter=A "$BASE0" "$SHA" -- tools/ci/batches/ | grep -E '\.json$') && MAP_CHANGED=1 && MAP_PATH=tools/ci/test-map.json && PAGE_PATH=docs/plans/igneum-2.0-test-harness-map.md && REGISTRY_PATH=docs/plans/igneum-2.0-test-registry.json && AUTHOR=(-c user.name=t -c user.email=t@t) && export TEST_RECORD_ROOT="$rb" && merge_with_batches "$TIP" "$SHA" "merge master into both" branch 2>&1 && git merge-base --is-ancestor master HEAD && echo ancestor && grep -c -E '^### c[45]$' docs/plans/igneum-2.0-test-harness-map.md && python3 -c "import json; d=json.load(open('docs/plans/igneum-2.0-test-registry.json')); print('ok' if any(c.get('run_id')=='r-both' for s in d['suites'] for c in s['tests']) and any(c.get('run_id')=='r-master' for s in d['suites'] for c in s['tests']) else 'rows-lost')" ) + case "$out" in *ancestor*2*ok*) ;; *) echo "self-test failed: the branch-side merge of master under the lock did not carry master's rows and cells plus the branch's: $out"; fails=1 ;; esac + [ "$fails" = 0 ] && echo "self-test passed: a GitHub remote is refused with exit 2 while the suspension marker stands and a mirror remote is not; the CI rule binds a GitHub remote only; a landed master is fast-forwarded to every mirror and a refused mirror push is a line, not a failure; a green branch run merges; a red one is refused; an unknown read is waited through and refused only at the deadline; a queued run is waited for with the clock; an unrun branch is pushed once and then waited for; a red master refuses every merge but the declared fix; a registry landing's batches replay onto master's copy at the merge; a map change regenerates the harness page at the merge; a push that lost the ref race retries without re-running the hook; the master-landing lock holds for the whole landing and master merges into the branch under it" exit $fails fi if [ "$SELF_TEST" != 1 ] && github_suspended_refusal "$REMOTE"; then exit 2; fi @@ -302,6 +339,25 @@ else fi # rule 24 (8 October 2026, 17:2x UK): a diff that touches a .rs, Cargo.toml, Cargo.lock, build.rs or .cargo/config runs cargo check # and the crate suite on the box at gate priority for every crate it touches before the merge; docs/ and site/ alone skip it +if [ "$SELF_TEST" != 1 ]; then lock_acquire "$BRANCH" || exit 1; trap 'lock_release; rm -f "$MERGE_PID_FILE" 2>/dev/null' EXIT; fi +# inside the lock master cannot move under this landing; when master is ahead of the branch, the branch takes master first (the +# registry rebuilt from master's copy with the branch's batches replayed, the map merged structurally, the page regenerated), and that +# merge commit carries the branch's green stamp forward as the union of two gated parents (the branch fully, master's tip on its own +# landing), so the push takes the light gate and nothing is re-gated for a master that moved +git fetch -q "$REMOTE" master +if ! git merge-base --is-ancestor "$REMOTE/master" "$SHA"; then + echo "merge-to-master: master moved since the branch point ($(git rev-parse --short "$REMOTE/master")); merging it into $BRANCH under the lock" + PRE_SHA="$SHA"; BASE0=$(git merge-base "$SHA" "$REMOTE/master") + BATCHES=$(git diff --name-only --diff-filter=A "$BASE0" "$SHA" -- tools/ci/batches/ | grep -E '\.json$' || true); NOTES=$(git diff --name-only --diff-filter=A "$BASE0" "$SHA" -- tools/ci/notes/ | grep -E '\.json$' || true) + MAP_CHANGED=0; git diff --quiet "$BASE0" "$SHA" -- "${MAP_PATH:-tools/ci/test-map.json}" 2>/dev/null || MAP_CHANGED=1 + REGISTRY_PATH="${REGISTRY_PATH:-docs/plans/igneum-2.0-test-registry.json}"; MAP_PATH="${MAP_PATH:-tools/ci/test-map.json}"; PAGE_PATH="${PAGE_PATH:-docs/plans/igneum-2.0-test-harness-map.md}" + [ -n "$NOTES" ] || [ "$MAP_CHANGED" = 1 ] || git diff --quiet "$BASE0" "$SHA" -- "$REGISTRY_PATH" 2>/dev/null || BATCHES="${BATCHES:-.}" + if ! merge_with_batches "$(git rev-parse "$REMOTE/master")" "$SHA" "Merge master $(git rev-parse --short "$REMOTE/master") into $BRANCH under the master-landing lock" branch; then + echo "merge-to-master: master does not merge into $BRANCH cleanly outside the registry, the map and the page; resolve on the branch and retry" >&2; exit 1 + fi + SHA=$(git rev-parse HEAD); printf 'stamped as the union of %s (gated) and master %s (gated on its own landing), %s\n' "${PRE_SHA:0:8}" "$(git rev-parse --short "$REMOTE/master")" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$G/igneum-gate-green/$SHA" + echo "merge-to-master: $BRANCH is now $(git rev-parse --short "$SHA") (master merged in); stamped as the union of two gated parents" +fi BASE=$(git merge-base "$SHA" "$REMOTE/master" 2>/dev/null || git rev-parse "$REMOTE/master") bash tools/ci/rule24-crate-gate.sh "$BASE" "$SHA" || { echo "merge-to-master: REFUSED by rule 24: a touched crate does not check or its suite is red (above); fix on the branch and retry" >&2; exit 1; } # the acceptance registry is a hot file (8 October 2026, 19:1x UK: three landings in a row lost the race to another lane's rows during diff --git a/tools/ci/review-suite-check.sh b/tools/ci/review-suite-check.sh index 1f0c2ea92..25b0b92aa 100755 --- a/tools/ci/review-suite-check.sh +++ b/tools/ci/review-suite-check.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # The REV suite of the registry matches Review B's findings and dispatch (tools/ci/review-suite.mjs --check), self-test first. set -euo pipefail -node tools/ci/review-suite.mjs --self-test >/dev/null +node tools/ci/review-suite.mjs --self-test | grep -v '^self-test passed' || true node tools/ci/review-suite.mjs --findings docs/analysis/review-2026-10-08-b/findings.json --dispatch docs/analysis/review-2026-10-08-b/dispatch.md --prefix REV --check -if [ -f docs/plans/igneum-2.0-master/traceability.json ]; then node tools/ci/int-suite.mjs --self-test >/dev/null && node tools/ci/int-suite.mjs --traceability docs/plans/igneum-2.0-master/traceability.json --check; fi +if [ -f docs/plans/igneum-2.0-master/traceability.json ]; then node tools/ci/int-suite.mjs --self-test | grep -v '^self-test passed' || true; node tools/ci/int-suite.mjs --traceability docs/plans/igneum-2.0-master/traceability.json --check; fi