From f9ca7252c3869ba8023c26d64f9addbe96caab90 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:12:07 +0000 Subject: [PATCH] A landing on any remote fans master out to every other mirror, as a no-verify copy merge-to-master.sh called mirror_master only after a GitHub landing (remote_is_github && ...), so every box landing under the suspension exception left the other mirrors where they were: build-3 and build-4 sat at 27801032 (7 October 15:27) while the box mirror carried b2ac5ea7, and lanes cut branches from the stale tip. The landing path now fans out for whichever remote took it, skipping that remote, and pushes with --no-verify: the sha is the landed remote's master read back, a copy of what the gate passed, not a landing (with the hook on, a stale mirror re-ran the full gate for six minutes per box). Self-tests: the landing line is pinned, and the mirror that took the landing is not pushed to again. Co-Authored-By: Claude Fable 5.1 --- tools/ci/README.md | 2 +- tools/ci/merge-to-master.sh | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/tools/ci/README.md b/tools/ci/README.md index 75e0f87ee..4316d6241 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -10,7 +10,7 @@ | the red watcher fires on cancelled and timed-out runs too (`ci-red.yml`, `red-watch.mjs`) | The watcher's `if` missing any of failure, cancelled, timed_out, or the conclusion not handed to the record step (the self-test reads the workflow file); the line names the kind: CI red, CI cancelled, CI timed out. | 7 October 2026 | | gh's active account is the stored Igneum entry (`gh-account-check.sh`, in Igneum's own gh directory `~/.config/gh-igneum` through `gh-env.sh`, never the founder's) | A push or a landing from this Mac while Igneum's gh directory names any other account as active, or none (the refusal names the one step: the founder or main stores the Igneum token there with `GH_CONFIG_DIR=~/.config/gh-igneum gh auth login --with-token`; no lane does); skipped with a line while `github-suspended` stands. RULE: no lane switches gh accounts on this Mac, ever; the second owner's login belongs to other projects and must never touch Igneum; the stored entry's name is in ~/.config/igneum/gh-user, never in the repository. | 7 October 2026, 21:41 UK: a lane switched gh to the other login during the suspension; nobody could say which | -| no landing on the public host while the marker stands (`pre-push.sh` `forgejo_master_frozen`, `merge-to-master.sh` `forgejo_master_refusal`); the hook binds every remote rule to the remote's URL | A push of master to git.igneum.network, or a `--remote` naming it, while `tools/ci/github-suspended` stands: its master is a rewritten copy replaced at cut-over, so the landing would be lost (a branch pushed there for safekeeping passes). Before the fix the hook matched the remote NAME, so `git push origin master` bound neither the GitHub refusal nor the CI rule; the self-test now drives the hook by name through a fixture repo. Also: `gate-manifest-check.sh` and four other pipefail checks no longer pipe a file-sized producer into `grep -q` (GNU sed took SIGPIPE on an early match and the check read it as a missing run line on the Linux runners and boxes); `mirror_master` fast-forwards every box with a build-server file | 8 Oct 2026 | +| no landing on the public host while the marker stands (`pre-push.sh` `forgejo_master_frozen`, `merge-to-master.sh` `forgejo_master_refusal`); the hook binds every remote rule to the remote's URL | A push of master to git.igneum.network, or a `--remote` naming it, while `tools/ci/github-suspended` stands: its master is a rewritten copy replaced at cut-over, so the landing would be lost (a branch pushed there for safekeeping passes). Before the fix the hook matched the remote NAME, so `git push origin master` bound neither the GitHub refusal nor the CI rule; the self-test now drives the hook by name through a fixture repo. Also: `gate-manifest-check.sh` and four other pipefail checks no longer pipe a file-sized producer into `grep -q` (GNU sed took SIGPIPE on an early match and the check read it as a missing run line on the Linux runners and boxes); `mirror_master` fast-forwards every box with a build-server file after a landing on ANY remote (before, only a GitHub landing fanned out, so a box landing left build-3 and build-4 at a tip 23 hours old), as a `--no-verify` copy of the master the gate already passed (a stale mirror had re-run the full gate for six minutes per box) | 8 Oct 2026 | | kill by exact command or pid file (owed as a check) | 6 October 2026, 21:09Z: a Mac-side `pkill -f ` matched nothing (the log name was a redirect, not part of the command line), the roll-everything script lived on and wiped a box it had been told to hold. Rule: a job is stopped by its pid file (`tools/fleet/fleet-bg.sh start|stop `) or by a pattern anchored on its exact command line (`^python3 -u /root/fleet/in/box-prover.py`), never by a word that may or may not appear in it. The check that flags a `pkill -f`/`pgrep -f` whose literal is a path or a name that never starts a command line is owed to the CI lane | diff --git a/tools/ci/merge-to-master.sh b/tools/ci/merge-to-master.sh index 204f4157d..4dd0f9c18 100755 --- a/tools/ci/merge-to-master.sh +++ b/tools/ci/merge-to-master.sh @@ -89,12 +89,15 @@ master_gate() { # a4bca198 while GitHub's carried cf7d6ccb, and three branches were cut from the stale tip; the mirrors only ever received what a # build happened to push). One push per mirror in IGNEUM_MIRRORS (default: both box files), fast-forward only, best effort: a mirror # that is down prints a line and never fails the landing. -mirror_master() { # - local sha="$1" m host key="${BS_KEY:-$HOME/.ssh/igneum_ed25519}" list="${IGNEUM_MIRRORS:-}" +mirror_master() { # [landed-remote-url]: the landed master to every other mirror + local sha="$1" landed="${2:-}" m host key="${BS_KEY:-$HOME/.ssh/igneum_ed25519}" list="${IGNEUM_MIRRORS:-}" # every box with a build-server file (8 October 2026, 13:5x UK: build-3 and build-4's mirrors sat at 7 October 15:27 with only the first two listed) if [ -z "$list" ]; then for f in "$HOME/.config/igneum/build-server" "$HOME"/.config/igneum/build-server-[0-9]*; do [ -s "$f" ] && list="$list $(head -1 "$f" | tr -d '[:space:]'):/srv/igneum.git"; done; fi for m in $list; do - if GIT_SSH_COMMAND="ssh -i $key -o BatchMode=yes -o ConnectTimeout=10" git push -q "$m" "$sha:refs/heads/master" 2>/dev/null; then echo "merge-to-master: mirror $m master -> ${sha:0:8}" + case "$landed" in *"${m#*@}"*) continue ;; esac # the mirror that took the landing itself + # --no-verify: this is a copy of a master the gate already passed on landing (the sha is the landed remote's master, read back), + # not a landing; with the hook on, a stale mirror re-ran the full gate for six minutes per box (8 October 2026, 14:0x UK) + if GIT_SSH_COMMAND="ssh -i $key -o BatchMode=yes -o ConnectTimeout=10" git push -q --no-verify "$m" "$sha:refs/heads/master" 2>/dev/null; then echo "merge-to-master: mirror $m master -> ${sha:0:8}" else echo "merge-to-master: mirror $m did not take master ${sha:0:8} (down, or not a fast-forward); the next landing tries again"; fi done } @@ -163,6 +166,8 @@ success 4 u push run git clone -q --bare "$d/src" "$d/mirror.git" && ( cd "$d/mirror.git" && git update-ref refs/heads/master "$(git rev-parse master~1)" ) tip=$(git -C "$d/src" rev-parse master) out=$( cd "$d/src" && IGNEUM_MIRRORS="$d/mirror.git" mirror_master "$tip" ) + grep -qE '^ mirror_master "\$\(git rev-parse "\$REMOTE/master"\)"' "$0" || { echo "self-test failed: the landing path does not fan master out to the mirrors for every remote"; fails=1; } + out2=$( cd "$d/src" && IGNEUM_MIRRORS="$d/mirror.git" mirror_master "$tip" "file://$d/mirror.git" ); case "$out2" in *"mirror"*) echo "self-test failed: the mirror that took the landing was pushed to again: $out2"; fails=1 ;; esac [ "$(git -C "$d/mirror.git" rev-parse master)" = "$tip" ] || { echo "self-test failed: the mirror was not fast-forwarded to the landed master: $out"; fails=1; } ( cd "$d/src" && git checkout -q -b other master~1 && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m fork ); fork=$(git -C "$d/src" rev-parse other) out=$( cd "$d/src" && IGNEUM_MIRRORS="$d/mirror.git" mirror_master "$fork" ) @@ -204,7 +209,9 @@ for i in $(seq 1 "$TRIES"); do if ( cd "$W" && git push -q "$REMOTE" HEAD:master ); then # on a GitHub remote the hook asks ci-state about ${SHA:0:8} once more git worktree remove --force "$W"; git fetch -q "$REMOTE" master echo "merge-to-master: pushed on try $i: $REMOTE/master $(git log -1 --format='%h %ci' "$REMOTE/master") $(TZ=Europe/London date '+%H:%M %Z')" - remote_is_github && mirror_master "$(git rev-parse "$REMOTE/master")"; exit 0 + # the landed master to every other mirror, whichever remote took the landing (8 October 2026, 14:0x UK: a box landing never fanned + # out, so build-3 and build-4 cut branches from a tip 23 hours old) + mirror_master "$(git rev-parse "$REMOTE/master")" "$(git remote get-url "$REMOTE" 2>/dev/null)"; exit 0 fi echo "merge-to-master: try $i: the push was rejected (master moved or the hook was red); again" else