diff --git a/tools/ci/README.md b/tools/ci/README.md index 75e0f87ee..4316d6241 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -10,7 +10,7 @@ | the red watcher fires on cancelled and timed-out runs too (`ci-red.yml`, `red-watch.mjs`) | The watcher's `if` missing any of failure, cancelled, timed_out, or the conclusion not handed to the record step (the self-test reads the workflow file); the line names the kind: CI red, CI cancelled, CI timed out. | 7 October 2026 | | gh's active account is the stored Igneum entry (`gh-account-check.sh`, in Igneum's own gh directory `~/.config/gh-igneum` through `gh-env.sh`, never the founder's) | A push or a landing from this Mac while Igneum's gh directory names any other account as active, or none (the refusal names the one step: the founder or main stores the Igneum token there with `GH_CONFIG_DIR=~/.config/gh-igneum gh auth login --with-token`; no lane does); skipped with a line while `github-suspended` stands. RULE: no lane switches gh accounts on this Mac, ever; the second owner's login belongs to other projects and must never touch Igneum; the stored entry's name is in ~/.config/igneum/gh-user, never in the repository. | 7 October 2026, 21:41 UK: a lane switched gh to the other login during the suspension; nobody could say which | -| no landing on the public host while the marker stands (`pre-push.sh` `forgejo_master_frozen`, `merge-to-master.sh` `forgejo_master_refusal`); the hook binds every remote rule to the remote's URL | A push of master to git.igneum.network, or a `--remote` naming it, while `tools/ci/github-suspended` stands: its master is a rewritten copy replaced at cut-over, so the landing would be lost (a branch pushed there for safekeeping passes). Before the fix the hook matched the remote NAME, so `git push origin master` bound neither the GitHub refusal nor the CI rule; the self-test now drives the hook by name through a fixture repo. Also: `gate-manifest-check.sh` and four other pipefail checks no longer pipe a file-sized producer into `grep -q` (GNU sed took SIGPIPE on an early match and the check read it as a missing run line on the Linux runners and boxes); `mirror_master` fast-forwards every box with a build-server file | 8 Oct 2026 | +| no landing on the public host while the marker stands (`pre-push.sh` `forgejo_master_frozen`, `merge-to-master.sh` `forgejo_master_refusal`); the hook binds every remote rule to the remote's URL | A push of master to git.igneum.network, or a `--remote` naming it, while `tools/ci/github-suspended` stands: its master is a rewritten copy replaced at cut-over, so the landing would be lost (a branch pushed there for safekeeping passes). Before the fix the hook matched the remote NAME, so `git push origin master` bound neither the GitHub refusal nor the CI rule; the self-test now drives the hook by name through a fixture repo. Also: `gate-manifest-check.sh` and four other pipefail checks no longer pipe a file-sized producer into `grep -q` (GNU sed took SIGPIPE on an early match and the check read it as a missing run line on the Linux runners and boxes); `mirror_master` fast-forwards every box with a build-server file after a landing on ANY remote (before, only a GitHub landing fanned out, so a box landing left build-3 and build-4 at a tip 23 hours old), as a `--no-verify` copy of the master the gate already passed (a stale mirror had re-run the full gate for six minutes per box) | 8 Oct 2026 | | kill by exact command or pid file (owed as a check) | 6 October 2026, 21:09Z: a Mac-side `pkill -f ` matched nothing (the log name was a redirect, not part of the command line), the roll-everything script lived on and wiped a box it had been told to hold. Rule: a job is stopped by its pid file (`tools/fleet/fleet-bg.sh start|stop `) or by a pattern anchored on its exact command line (`^python3 -u /root/fleet/in/box-prover.py`), never by a word that may or may not appear in it. The check that flags a `pkill -f`/`pgrep -f` whose literal is a path or a name that never starts a command line is owed to the CI lane | diff --git a/tools/ci/merge-to-master.sh b/tools/ci/merge-to-master.sh index 204f4157d..4dd0f9c18 100755 --- a/tools/ci/merge-to-master.sh +++ b/tools/ci/merge-to-master.sh @@ -89,12 +89,15 @@ master_gate() { # a4bca198 while GitHub's carried cf7d6ccb, and three branches were cut from the stale tip; the mirrors only ever received what a # build happened to push). One push per mirror in IGNEUM_MIRRORS (default: both box files), fast-forward only, best effort: a mirror # that is down prints a line and never fails the landing. -mirror_master() { # - local sha="$1" m host key="${BS_KEY:-$HOME/.ssh/igneum_ed25519}" list="${IGNEUM_MIRRORS:-}" +mirror_master() { # [landed-remote-url]: the landed master to every other mirror + local sha="$1" landed="${2:-}" m host key="${BS_KEY:-$HOME/.ssh/igneum_ed25519}" list="${IGNEUM_MIRRORS:-}" # every box with a build-server file (8 October 2026, 13:5x UK: build-3 and build-4's mirrors sat at 7 October 15:27 with only the first two listed) if [ -z "$list" ]; then for f in "$HOME/.config/igneum/build-server" "$HOME"/.config/igneum/build-server-[0-9]*; do [ -s "$f" ] && list="$list $(head -1 "$f" | tr -d '[:space:]'):/srv/igneum.git"; done; fi for m in $list; do - if GIT_SSH_COMMAND="ssh -i $key -o BatchMode=yes -o ConnectTimeout=10" git push -q "$m" "$sha:refs/heads/master" 2>/dev/null; then echo "merge-to-master: mirror $m master -> ${sha:0:8}" + case "$landed" in *"${m#*@}"*) continue ;; esac # the mirror that took the landing itself + # --no-verify: this is a copy of a master the gate already passed on landing (the sha is the landed remote's master, read back), + # not a landing; with the hook on, a stale mirror re-ran the full gate for six minutes per box (8 October 2026, 14:0x UK) + if GIT_SSH_COMMAND="ssh -i $key -o BatchMode=yes -o ConnectTimeout=10" git push -q --no-verify "$m" "$sha:refs/heads/master" 2>/dev/null; then echo "merge-to-master: mirror $m master -> ${sha:0:8}" else echo "merge-to-master: mirror $m did not take master ${sha:0:8} (down, or not a fast-forward); the next landing tries again"; fi done } @@ -163,6 +166,8 @@ success 4 u push run git clone -q --bare "$d/src" "$d/mirror.git" && ( cd "$d/mirror.git" && git update-ref refs/heads/master "$(git rev-parse master~1)" ) tip=$(git -C "$d/src" rev-parse master) out=$( cd "$d/src" && IGNEUM_MIRRORS="$d/mirror.git" mirror_master "$tip" ) + grep -qE '^ mirror_master "\$\(git rev-parse "\$REMOTE/master"\)"' "$0" || { echo "self-test failed: the landing path does not fan master out to the mirrors for every remote"; fails=1; } + out2=$( cd "$d/src" && IGNEUM_MIRRORS="$d/mirror.git" mirror_master "$tip" "file://$d/mirror.git" ); case "$out2" in *"mirror"*) echo "self-test failed: the mirror that took the landing was pushed to again: $out2"; fails=1 ;; esac [ "$(git -C "$d/mirror.git" rev-parse master)" = "$tip" ] || { echo "self-test failed: the mirror was not fast-forwarded to the landed master: $out"; fails=1; } ( cd "$d/src" && git checkout -q -b other master~1 && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m fork ); fork=$(git -C "$d/src" rev-parse other) out=$( cd "$d/src" && IGNEUM_MIRRORS="$d/mirror.git" mirror_master "$fork" ) @@ -204,7 +209,9 @@ for i in $(seq 1 "$TRIES"); do if ( cd "$W" && git push -q "$REMOTE" HEAD:master ); then # on a GitHub remote the hook asks ci-state about ${SHA:0:8} once more git worktree remove --force "$W"; git fetch -q "$REMOTE" master echo "merge-to-master: pushed on try $i: $REMOTE/master $(git log -1 --format='%h %ci' "$REMOTE/master") $(TZ=Europe/London date '+%H:%M %Z')" - remote_is_github && mirror_master "$(git rev-parse "$REMOTE/master")"; exit 0 + # the landed master to every other mirror, whichever remote took the landing (8 October 2026, 14:0x UK: a box landing never fanned + # out, so build-3 and build-4 cut branches from a tip 23 hours old) + mirror_master "$(git rev-parse "$REMOTE/master")" "$(git remote get-url "$REMOTE" 2>/dev/null)"; exit 0 fi echo "merge-to-master: try $i: the push was rejected (master moved or the hook was red); again" else