diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 37bd83123..572c2920a 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -89,6 +89,7 @@ Also logged for context: the Mac's Linux cross-build with zig (`infra/cross/buil | Case | What happened | Fix | |---|---|---| | Stale overlay blocks the next checkout (PC 1 worker, 6 Oct 2026) | build-remote.sh rsyncs uncommitted files over the box's checkout; on the next commit `git checkout -B` refused with "local changes would be overwritten" | remote-run.sh `checkout` mode: `git checkout -- .` and `git clean -fd` (target dirs, sha stamps and ignored files kept) before the branch checkout, then the overlay; `remote-run.sh --self-test` reproduces the dirty tree and shows the mode landing on the new commit clean | +| The same class met twice more by the UI lane on its mirror (worktrees whose tools predate 3e6a488 pipe the OLD remote-run.sh to the box per build) | nothing builds until the tree is cleared by hand | the fix is in master's remote-run.sh since 3e6a488; a worktree gets it by merging master; `tools/ci/mirror-reset-check.sh` (in the pre-push gate) reads checkout_tree() and fails when the reset and the clean do not both come before the branch checkout; its self-test shows the real script passing, a copy without the reset failing, and the same lines moved after the checkout failing. cargo-audit is owned by provision.sh step_cargo_tools (1eec354, the night battery's agent), confirmed `ok (cargo-audit 0.22.2)` on the box | | An all-identical overlay listed only directories | the first run's touch pipeline got an empty file list and failed | files only are counted and re-stamped (lib.sh bs_overlay_dir) | | bash 3.2 on the Mac treats an empty array as unbound under `set -u` | run-from-mac.sh died on `PASS[*]` | a string instead of an array | | `grep -q` plus `pipefail` turned a strings hit into a miss | the commit-string gate failed a stamped igneumd on its first use (SIGPIPE on `strings`) | `grep -c` | diff --git a/tools/ci/mirror-reset-check.sh b/tools/ci/mirror-reset-check.sh new file mode 100755 index 000000000..0a83bb966 --- /dev/null +++ b/tools/ci/mirror-reset-check.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# The stale-overlay class (6 October 2026, the PC 1 worker at 18:4x UTC, the UI lane twice later the same night): tools/build-remote.sh +# rsyncs a worktree's uncommitted files over the box's checkout; once those files are committed and HEAD moves, the next run's +# `git checkout -B ` on the box refuses with "Your local changes would be overwritten" and nothing builds until +# someone clears the tree by hand. Rule: the remote checkout RESETS the working tree first (`git checkout -- .` and a `git clean` +# of the untracked overlay files, target dirs and stamps kept), then checks the branch out; the overlay re-applies the Mac's +# working-tree state afterwards anyway. The code lives in infra/build-server/remote-run.sh checkout_tree() and is piped to the box +# from each Mac worktree per build, so a worktree whose tools predate the fix keeps the old behaviour until it has master. +# This check reads checkout_tree() and fails when the reset and the clean do not both come BEFORE the branch checkout. +# +# tools/ci/mirror-reset-check.sh # exit 1 with the reason +# tools/ci/mirror-reset-check.sh --self-test # the real script passes; a copy with the reset lines removed fails +set -euo pipefail +cd "$(dirname "$0")/../.." +check() { # : exit 0 when checkout_tree resets and cleans before the branch checkout + local f="$1" body reset clean co + body=$(sed -n '/^checkout_tree()/,/^}/p' "$f") + [ -n "$body" ] || { echo "mirror-reset: $f has no checkout_tree() function"; return 1; } + reset=$(printf '%s\n' "$body" | grep -nE '^\s*git checkout -q -- \.' | head -1 | cut -d: -f1) + clean=$(printf '%s\n' "$body" | grep -nE '^\s*git clean -q?f' | head -1 | cut -d: -f1) + co=$(printf '%s\n' "$body" | grep -nE '^\s*git checkout -q -B ' | head -1 | cut -d: -f1) + [ -n "$co" ] || { echo "mirror-reset: $f: no branch checkout (git checkout -q -B) in checkout_tree"; return 1; } + [ -n "$reset" ] && [ "$reset" -lt "$co" ] || { echo "mirror-reset: $f: no 'git checkout -- .' before the branch checkout (line $co of checkout_tree)"; return 1; } + [ -n "$clean" ] && [ "$clean" -lt "$co" ] || { echo "mirror-reset: $f: no 'git clean' of the overlay before the branch checkout (line $co of checkout_tree)"; return 1; } + printf '%s\n' "$body" | grep -qE "git clean .*-e target" || { echo "mirror-reset: $f: the clean must keep target dirs (-e target)"; return 1; } + echo "mirror-reset: $f resets (line $reset) and cleans (line $clean) before the branch checkout (line $co)" +} +if [ "${1:-}" = --self-test ]; then + t=$(mktemp -d); trap 'rm -rf "$t"' EXIT + check infra/build-server/remote-run.sh >/dev/null || { echo "mirror-reset self-test: the real script FAILED the check"; exit 1; } + grep -vE '^\s*git (checkout -q -- \.|clean -q?f)' infra/build-server/remote-run.sh > "$t/no-reset.sh" + if check "$t/no-reset.sh" >/dev/null 2>&1; then echo "mirror-reset self-test: a script without the reset PASSED (the check is blind)"; exit 1; fi + # the order matters: the same lines after the branch checkout must fail too + python3 - infra/build-server/remote-run.sh "$t/wrong-order.sh" <<'PY' +import re, sys +s = open(sys.argv[1]).read() +body = re.search(r'^checkout_tree\(\).*?^}', s, flags=re.S | re.M).group(0) +lines = body.split("\n"); moved = [l for l in lines if re.match(r'\s*git (checkout -q -- \.|clean -q?f)', l)] +rest = [l for l in lines if l not in moved] +i = next(k for k, l in enumerate(rest) if re.match(r'\s*git checkout -q -B ', l)) +rest[i + 1:i + 1] = moved +open(sys.argv[2], "w").write(s.replace(body, "\n".join(rest))) +PY + if check "$t/wrong-order.sh" >/dev/null 2>&1; then echo "mirror-reset self-test: a reset AFTER the branch checkout PASSED (the check ignores order)"; exit 1; fi + echo "mirror-reset self-test: the real script passes; no reset fails; reset after the checkout fails"; exit 0 +fi +check infra/build-server/remote-run.sh diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh index 5c6b335a7..32e4e4b7f 100755 --- a/tools/ci/pre-push.sh +++ b/tools/ci/pre-push.sh @@ -71,6 +71,7 @@ tree_checks() { run "root prover playbooks kill the GPU server and unlink its socket" bash tools/ci/prover-socket-check.sh run "commit-string gate self-test" bash tools/ci/commit-string-check.sh --self-test run "build server remote checkout self-test" bash infra/build-server/remote-run.sh --self-test + run "the remote checkout resets the mirror's tree before the branch checkout (the stale-overlay class)" bash -c 'bash tools/ci/mirror-reset-check.sh --self-test && bash tools/ci/mirror-reset-check.sh' run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh' run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh' run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test