From 9df9688b59d3be127f513c44e732c566deba1777 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:16:04 +0000 Subject: [PATCH] Build server: the remote checkout discards the previous overlay first (the stale-overlay class, PC 1 worker, 6 October 2026) remote-run.sh gains a checkout mode used by lib.sh: git checkout -- . and git clean -fd (target dirs, the sha stamps and ignored files kept), fetch, branch at the commit, then a clean-tree check; the overlay follows. Before this, the rsync of uncommitted files stayed in the box's tree and the next commit's git checkout -B refused with 'local changes would be overwritten'. remote-run.sh --self-test reproduces the dirty tree (edited tracked file, untracked file, target dir, sha stamp), shows the plain checkout refusing and the mode landing clean on the new commit; it runs in ci.yml and passed on the Mac and the box; a live dirty-then-clean pair on the fork worktree passed too. Plan: section 5, gotchas of the first day. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 2 ++ docs/plans/build-server.md | 12 ++++++- infra/build-server/lib.sh | 11 +++--- infra/build-server/remote-run.sh | 57 +++++++++++++++++++++++++++++++- 4 files changed, 76 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c104ba6bd..90af33b10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,8 @@ jobs: run: bash tools/ci/prover-socket-check.sh - name: commit-string gate self-test (the empty-commit class of 6 October 2026; the gate itself runs in build-remote.sh, cross-remote.sh and cross-build.sh on every node binary) run: bash tools/ci/commit-string-check.sh --self-test + - name: build server remote checkout self-test (the stale-overlay class of 6 October 2026) + run: bash infra/build-server/remote-run.sh --self-test - name: no secret file names and no 64-hex secrets in the tree (self-test first, then the tree) run: bash tools/ci/no-secrets-check.sh --self-test && bash tools/ci/no-secrets-check.sh - name: faucet unit tests (validation, the daily limits, the signed transaction; keccak, RLP and secp256k1 vectors) diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 9e8b78dd1..1b1cc697d 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -83,7 +83,17 @@ Also logged for context: the Mac's Linux cross-build with zig (`infra/cross/buil | R6 | The box is never a node host for the live devnet and never holds a secret (no `~/.config/igneum` there). The Devnet 2 seed on it runs under its own unit with `--devnet --devnet-suffix=` on 26611 (ufw already open) when that work starts. | | R7 | CLAUDE.md line "nothing is built on a server" and "Windows builds go to the GitHub runner, Linux binaries come from infra/cross/build-linux.sh" are rewritten when R1 and R2 are adopted; until then the box is the measured option, not the rule. | -## 5. What the box does not do yet +## 5. Gotchas met on the first day + +| Case | What happened | Fix | +|---|---|---| +| Stale overlay blocks the next checkout (PC 1 worker, 6 Oct 2026) | build-remote.sh rsyncs uncommitted files over the box's checkout; on the next commit `git checkout -B` refused with "local changes would be overwritten" | remote-run.sh `checkout` mode: `git checkout -- .` and `git clean -fd` (target dirs, sha stamps and ignored files kept) before the branch checkout, then the overlay; `remote-run.sh --self-test` reproduces the dirty tree and shows the mode landing on the new commit clean | +| An all-identical overlay listed only directories | the first run's touch pipeline got an empty file list and failed | files only are counted and re-stamped (lib.sh bs_overlay_dir) | +| bash 3.2 on the Mac treats an empty array as unbound under `set -u` | run-from-mac.sh died on `PASS[*]` | a string instead of an array | +| `grep -q` plus `pipefail` turned a strings hit into a miss | the commit-string gate failed a stamped igneumd on its first use (SIGPIPE on `strings`) | `grep -c` | +| Let's Encrypt saw NXDOMAIN for build.igneum.network | the deSEC record was minutes old; Ubuntu's Caddy then fell back to ZeroSSL and failed with HTTP 422 for ever | issuer pinned to Let's Encrypt; the retry got the certificate | + +## 6. What the box does not do yet | Gap | Why it matters | Next step | |---|---|---| diff --git a/infra/build-server/lib.sh b/infra/build-server/lib.sh index b0dcfbd9e..b16fb76c0 100755 --- a/infra/build-server/lib.sh +++ b/infra/build-server/lib.sh @@ -111,10 +111,13 @@ bs_push_and_checkout() { [ "$BS_TOP_REL" = . ] && remote_top="$BS_REMOTE_WT" bs_log "push $BS_TOP HEAD $BS_SHA ($BS_BRANCH) -> $url" GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$BS_TOP" push -q --force "$url" "HEAD:refs/heads/$BS_BRANCH" || bs_die "push to the mirror failed" - bs_ssh "set -e; mkdir -p '$BS_REMOTE_WT' - if [ ! -d '$remote_top/.git' ]; then rm -rf '$remote_top'; git clone -q --no-checkout '$BS_MIRROR' '$remote_top'; fi - cd '$remote_top'; git fetch -q origin '+refs/heads/*:refs/remotes/origin/*'; git checkout -q -B '$BS_BRANCH' '$BS_SHA'; git reset -q --hard '$BS_SHA' - git status --porcelain | head -3" || bs_die "remote checkout at $remote_top failed" + # the checkout runs as remote-run.sh's `checkout` mode: discard the previous overlay (tracked edits and untracked files, + # target dirs kept), then the branch at the commit. 6 October 2026, PC 1 worker's first use: the overlay of an earlier + # commit's uncommitted files stayed in the box's tree and `git checkout -B` refused with "local changes would be overwritten". + BR_MODE=checkout BR_CO_DIR="$remote_top" BR_CO_MIRROR="$BS_MIRROR" BR_CO_BRANCH="$BS_BRANCH" BR_CO_SHA="$BS_SHA" BR_CO_WT="$BS_REMOTE_WT" \ + bash -c ' + for v in BR_MODE BR_CO_DIR BR_CO_MIRROR BR_CO_BRANCH BR_CO_SHA BR_CO_WT; do printf "export %s=%q\n" "$v" "${!v}"; done + cat "$0"' "$(dirname "${BASH_SOURCE[0]}")/remote-run.sh" | bs_ssh 'bash -s' || bs_die "remote checkout at $remote_top failed" BS_REMOTE_TOP="$remote_top" } diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh index 2ef2bdbd8..a4ddc8b52 100755 --- a/infra/build-server/remote-run.sh +++ b/infra/build-server/remote-run.sh @@ -17,8 +17,63 @@ # 2026): on success, on failure and on the slot give-up. UTC ISO 8601 Z times, numbers unquoted, unknown fields omitted, # artefacts with bytes and sha256 only when the run succeeded (a failed build would list the previous build's files), # the line kept under 4 KB. +# Modes (BR_MODE, default run): `checkout` resets the box's tree and checks the branch out at the commit (lib.sh +# bs_push_and_checkout: BR_CO_DIR, BR_CO_MIRROR, BR_CO_BRANCH, BR_CO_SHA, BR_CO_WT); `--self-test` (first argument) builds a +# scratch mirror and clone, dirties the clone the way a build's overlay does, moves the mirror one commit on, and shows the +# checkout mode lands on the new commit with a clean tree (the 6 October 2026 case: a stale overlay made `git checkout -B` +# refuse with "local changes would be overwritten"). set -uo pipefail -. /etc/profile.d/igneum-build.sh +[ -f /etc/profile.d/igneum-build.sh ] && . /etc/profile.d/igneum-build.sh + +# discard the previous overlay (tracked edits and untracked files; target dirs, the sha stamps and anything ignored are kept), +# fetch, then the branch at the commit. Runs in BR_CO_DIR, clones it from BR_CO_MIRROR when it has no .git. +checkout_tree() { + local dir="$1" mirror="$2" branch="$3" sha="$4" wt="${5:-}" + set -e + [ -n "$wt" ] && mkdir -p "$wt" + if [ ! -d "$dir/.git" ]; then rm -rf "$dir"; git clone -q --no-checkout "$mirror" "$dir"; fi + cd "$dir" + git checkout -q -- . 2>/dev/null || true + git clean -qfd -e target -e 'target-*' -e '.build-remote-sha-*' -e '.cross-remote-sha-*' -e sccache + git fetch -q origin '+refs/heads/*:refs/remotes/origin/*' + git checkout -q -B "$branch" "$sha" + git reset -q --hard "$sha" + # what may remain untracked: target dirs, the sha stamps of build-remote.sh and cross-remote.sh (kept so a build after the + # checkout knows whether to clean kaspa-build-info), sccache; the first live run after this mode was added failed on a + # stamp it had itself kept (6 October 2026, 18:14 UTC: the self-test had no stamp file; it has one now) + local left; left=$(git status --porcelain | grep -vE '^\?\? (target|target-|sccache|\.build-remote-sha-|\.cross-remote-sha-)' | head -3 || true) + [ -z "$left" ] || { echo "checkout: tree not clean after reset at $dir: $left" >&2; return 1; } + set +e +} + +if [ "${1:-}" = --self-test ]; then + t=$(mktemp -d); trap 'rm -rf "$t"' EXIT + git init -q --bare -b master "$t/mirror.git" + git init -q -b master "$t/src"; git -C "$t/src" -c user.name=t -c user.email=t@t commit -q --allow-empty -m one + echo a > "$t/src/a.txt"; git -C "$t/src" add a.txt; git -C "$t/src" -c user.name=t -c user.email=t@t commit -q -m two + git -C "$t/src" push -q "$t/mirror.git" master + sha1=$(git -C "$t/src" rev-parse HEAD) + checkout_tree "$t/box" "$t/mirror.git" master "$sha1" || { echo "self-test: first checkout failed"; exit 1; } + # the overlay of a build: a tracked file edited, an untracked file added, a target dir that must survive + echo edited > "$t/box/a.txt"; echo new > "$t/box/b.txt"; mkdir -p "$t/box/target/release"; echo bin > "$t/box/target/release/x"; echo "$sha1" > "$t/box/.build-remote-sha-target" + # the Mac moves on: a new commit that changes a.txt + echo a2 > "$t/src/a.txt"; git -C "$t/src" -c user.name=t -c user.email=t@t commit -qam three; git -C "$t/src" push -q "$t/mirror.git" master + sha2=$(git -C "$t/src" rev-parse HEAD) + if (cd "$t/box" && git fetch -q origin && git checkout -q -B master "$sha2" 2>/dev/null); then echo "self-test: the plain checkout did NOT refuse on the dirty tree (the case no longer reproduces; the reset is still right)"; else echo "self-test: the plain checkout refuses on the dirty tree, as on 6 October"; fi + checkout_tree "$t/box" "$t/mirror.git" master "$sha2" || { echo "self-test: checkout mode FAILED on the dirty tree"; exit 1; } + [ "$(git -C "$t/box" rev-parse HEAD)" = "$sha2" ] || { echo "self-test: wrong commit"; exit 1; } + [ "$(cat "$t/box/a.txt")" = a2 ] || { echo "self-test: tracked edit survived"; exit 1; } + [ ! -e "$t/box/b.txt" ] || { echo "self-test: untracked overlay file survived"; exit 1; } + [ -f "$t/box/target/release/x" ] || { echo "self-test: target dir was cleaned"; exit 1; } + [ -f "$t/box/.build-remote-sha-target" ] || { echo "self-test: the sha stamp was cleaned"; exit 1; } + echo "self-test: checkout mode lands on the new commit with a clean tree, target dir and sha stamp kept"; exit 0 +fi + +if [ "${BR_MODE:-run}" = checkout ]; then + : "${BR_CO_DIR:?}" "${BR_CO_MIRROR:?}" "${BR_CO_BRANCH:?}" "${BR_CO_SHA:?}" + checkout_tree "$BR_CO_DIR" "$BR_CO_MIRROR" "$BR_CO_BRANCH" "$BR_CO_SHA" "${BR_CO_WT:-}"; exit $? +fi + : "${BR_DIR:?}" "${BR_CMD:?}" "${BR_LABEL:?}" "${BR_TOOL:?}" "${BR_KIND:?}" BR_HOST=$(hostname); BR_PID=$$; BR_T0=$(date +%s) export BR_HOST BR_PID BR_T0