#!/usr/bin/env bash # Shared by infra/build-server/run-from-mac.sh, tools/build-remote.sh and tools/cross-remote.sh. Source it, do not run it. # Everything that talks to igneum-build-1 from the Mac goes through here: the host line, the ssh options (the ops key # ~/.ssh/igneum_ed25519, a shared control socket so one build is one ssh session), the mirror push, the remote checkout # and the source overlay (rsync by checksum, changed files re-stamped: the copied-sources rule of 5 October 2026). # # Layout on the box (provision.sh): /srv/builds/ mirrors the Mac's igneum worktree ROOT (the directory that holds # igneum-pow/, app/, proving/ and vendor/), so the fork's relative path dependency `../../../../igneum-pow` # (vendor/igneum-node/consensus/pow/Cargo.toml) resolves on the box exactly as on the Mac: # Mac /Users/joshm/Projects/igneum-wt-ship0311/vendor/igneum-node-0311 -> box /srv/builds/igneum-wt-ship0311/vendor/igneum-node-0311 # Mac /Users/joshm/Projects/igneum-wt-ship0311/igneum-pow -> box /srv/builds/igneum-wt-ship0311/igneum-pow # Mac /Users/joshm/Projects/igneum/app/igneum-app -> box /srv/builds/igneum/app/igneum-app # The fork's kaspa-build-info reads `git rev-parse HEAD` at build time and the release plans check the commit in the binary's # strings, so the fork tree on the box is a real clone of the bare mirror /srv/igneum-node.git checked out at the Mac's HEAD, # with the Mac's uncommitted changes rsynced on top. The igneum repo's crates get the same from /srv/igneum.git. # shellcheck disable=SC2034 # shared with the scripts that source lib.sh BS_KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}" BS_HOST_FILE="${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" # one line: build@ (box 1; box N is build-server-N) # Several boxes (main, 7 October 2026: igneum-build-2 and igneum-build-3 on order). One host file per box: ~/.config/igneum/build-server # is box 1, build-server-2 is box 2, build-server-3 is box 3 (run-from-mac.sh --box N writes it). The route by class, unless the # caller passes --box: gates, builds, checks, cross-builds, the workers, the hands and the observer stay on box 1; suites, benches and # the attack rows go to box 2; proving and aggregation CPU work, the second prover's shadow runner and the pool's fast-time NETWORK go # to box 3. A class whose box has no host file yet falls back to box 1, and the log line says so. No box mines, ever (README.md). # Since 7 October 2026 15:xx UK the class is a preference with spill-over (bs_route_spill below): a full or overloaded box hands # the job to the other one. bs_box_file() { case "${1:-1}" in 1) echo "$BS_HOST_FILE" ;; *) echo "${BS_HOST_FILE}-$1" ;; esac; } bs_route() { # -> the PREFERRED box number, falling back to 1 local want=1 case "$1" in suite|bench|attack) want=2 ;; prove|shadow|fasttime) want=3 ;; esac if [ "$want" != 1 ] && [ ! -s "$(bs_box_file "$want")" ]; then bs_log "class $1 prefers box $want, which has no host file yet ($(bs_box_file "$want")): box 1"; want=1; fi echo "$want" } # Spill-over (the founder, 7 October 2026, 15:02 UK: build-1 at load 139 with a queue of 1 h 40 min while build-2 read 4.5 with both # slots free). The class is a PREFERENCE, not a pin: a job goes to its class's box unless that box has no free slot or its 1-minute # load is above BS_SPILL_LOAD (80 since 19:4x BST, was 64), in which case it goes to the other box when THAT one has a free slot under the same load # line; when neither qualifies it queues on its own box. The alternate of box 1 is box 2, of box 2 box 1, of box 3 box 1; a box # without a host file is never chosen. The decision is one line on the Mac (bs_log) and travels to the box in BR_ROUTE_* for the # JSONL row ("route": preferred, box, spilled, reason), so the dashboard shows it per job. A box is read with one ssh # (bs_box_state: free slots of the slot count, load1); BS_ROUTE_STATE_ in the environment replaces the ssh for the self-test # (tools/ci/route-spill-check.sh), "down" standing for an unreachable box. BS_SPILL_LOAD="${BS_SPILL_LOAD:-80}" # the founder, 7 Oct 2026 19:4x BST: both boxes to near max; was 64 bs_box_state() { # -> "free= slots= load1=" | "absent" | "down" local b="$1" v f h v=$(eval "printf '%s' \"\${BS_ROUTE_STATE_$b:-}\""); if [ -n "$v" ]; then printf '%s' "$v"; return 0; fi f=$(bs_box_file "$b"); [ -s "$f" ] || { printf 'absent'; return 0; } h="$(head -1 "$f" | tr -d '[:space:]')" # the probe runs BEFORE bs_host builds BS_SSH_OPTS (the pool lane, 7 Oct 2026 15:1x UK: bash 3.2 under set -u refuses an # unset array), so it carries its own options: the ops key, batch mode, a short connect timeout, no control socket ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 "$h" 'd=/srv/builds/_locks; n=$(cat $d/slots 2>/dev/null || echo 1); free=0; k=0; while [ $k -lt $n ]; do exec 9>>$d/build-$k; if flock -n 9; then free=$((free+1)); fi; exec 9>&-; k=$((k+1)); done; printf "free=%s slots=%s load1=%s" $free $n "$(cut -d" " -f1 /proc/loadavg)"' 2>/dev/null || printf 'down' } bs_state_ok() { # -> 0 when the box can take a job now (a free slot, load1 at or under the line) local st="$1" free load case "$st" in free=*) ;; *) return 1 ;; esac free=${st#free=}; free=${free%% *}; load=${st##*load1=} [ "$free" -gt 0 ] 2>/dev/null || return 1 awk -v l="$load" -v m="$BS_SPILL_LOAD" 'BEGIN { exit !(l + 0 <= m + 0) }' } bs_route_spill() { # [priority] -> the box number; sets BS_ROUTE_PREF, BS_ROUTE_BOX, BS_ROUTE_SPILLED, BS_ROUTE_REASON local class="$1" pref alt ps as pref=$(bs_route "$class" 2>/dev/null) case "$pref" in 1) alt=2 ;; 2) alt=1 ;; *) alt=1 ;; esac BS_ROUTE_PREF=$pref; BS_ROUTE_BOX=$pref; BS_ROUTE_SPILLED=0 ps=$(bs_box_state "$pref") if bs_state_ok "$ps"; then BS_ROUTE_REASON="box $pref ($ps) takes it" else as=$(bs_box_state "$alt") if bs_state_ok "$as"; then BS_ROUTE_BOX=$alt; BS_ROUTE_SPILLED=1; BS_ROUTE_REASON="box $pref ($ps) is full or over load $BS_SPILL_LOAD: spilled to box $alt ($as)" else BS_ROUTE_REASON="box $pref ($ps) and box $alt ($as) are both full or over load $BS_SPILL_LOAD: queued on box $pref"; fi fi bs_log "route: class $class prefers box $pref; $BS_ROUTE_REASON" BR_ROUTE_PREF=$BS_ROUTE_PREF BR_ROUTE_BOX=$BS_ROUTE_BOX BR_ROUTE_SPILLED=$BS_ROUTE_SPILLED BR_ROUTE_REASON=$BS_ROUTE_REASON export BR_ROUTE_PREF BR_ROUTE_BOX BR_ROUTE_SPILLED BR_ROUTE_REASON echo "$BS_ROUTE_BOX" } BS_ROOT_REMOTE=/srv/builds BS_MIRROR_REPO=/srv/igneum.git BS_MIRROR_NODE=/srv/igneum-node.git bs_log() { printf '%s %s: %s\n' "$(date -u +%H:%M:%S)" "${BS_TOOL:-build-server}" "$*" >&2; } bs_die() { bs_log "ERROR: $*"; exit 1; } bs_host() { # [box number, default BS_BOX or 1] BS_BOX="${1:-${BS_BOX:-1}}" BS_HOST="${BUILD_HOST:-}" if [ -z "$BS_HOST" ]; then local f; f=$(bs_box_file "$BS_BOX") [ -s "$f" ] || bs_die "no build server for box $BS_BOX: write build@ to $f (infra/build-server/run-from-mac.sh --box $BS_BOX does) or set BUILD_HOST" BS_HOST="$(head -1 "$f" | tr -d '[:space:]')" fi case "$BS_HOST" in *@*) ;; *) bs_die "BUILD_HOST must be user@host, got '$BS_HOST'" ;; esac [ -r "$BS_KEY" ] || bs_die "no ssh key at $BS_KEY" mkdir -p "$HOME/.ssh/cm" BS_SSH_OPTS=(-i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o ControlMaster=auto -o ControlPath="$HOME/.ssh/cm/igneum-build-%r@%h:%p" -o ControlPersist=900) BS_SSH_CMD="ssh"; local o; for o in "${BS_SSH_OPTS[@]}"; do BS_SSH_CMD="$BS_SSH_CMD $(printf '%q' "$o")"; done } bs_ssh() { ssh "${BS_SSH_OPTS[@]}" "$BS_HOST" "$@"; } bs_rsync() { rsync -e "$BS_SSH_CMD" "$@"; } # the two sides' rustc must agree (the box is pinned by provision.sh RUST_TOOLCHAIN; the Mac runs rustup's stable): # a different compiler gives different bytes and, across a minor version, different lints and errors # the pin (main, 7 October 2026): rust-toolchain.toml at the worktree root (and the fork's own copy) names the channel; the Mac's # rustc AS RESOLVED IN THE CRATE DIR (rustup reads the file), the box's rustc and the pin must agree, else the build is refused # a tree without the file (an older release branch) gives an empty pin, never a failed pipeline: under pipefail the sed status would # become the assignment's status and set -e would end the caller silently (7 Oct 2026, 03:36 UTC: the b3c228fa builds under the # 0.3.16 app tree 5d118f58 died right after the pairing line) bs_pin() { local f="${1:-$BS_WT_ROOT}/rust-toolchain.toml"; [ -f "$f" ] || { echo ""; return 0; }; { sed -n 's/^channel *= *"\([^"]*\)".*/\1/p' "$f" 2>/dev/null || true; } | head -1; } bs_toolchain_check() { local mac box pin pin=$(bs_pin "$BS_WT_ROOT"); [ -n "$pin" ] || pin=$(bs_pin "$BS_TOP") mac=$(cd "${BS_CRATE:-.}" && "${CARGO_HOME:-$HOME/.cargo}/bin/rustc" --version 2>/dev/null | awk '{ print $2 }') box=$(bs_ssh '. /etc/profile.d/igneum-build.sh; rustc --version' 2>/dev/null | awk '{ print $2 }') [ -n "$box" ] || bs_die "cannot read rustc on $BS_HOST (is it provisioned? infra/build-server/run-from-mac.sh)" if [ -n "$pin" ] && { [ "$mac" != "$pin" ] || [ "$box" != "$pin" ]; } || [ "$mac" != "$box" ]; then if [ "${IGNEUM_TOOLCHAIN_MISMATCH:-}" = ok ]; then bs_log "WARNING: pin ${pin:-none}, rustc $mac on the Mac, $box on the box (IGNEUM_TOOLCHAIN_MISMATCH=ok)" else bs_die "toolchain mismatch: rust-toolchain.toml pins ${pin:-nothing}, the Mac resolves rustc $mac in $BS_CRATE, the box runs $box; align the pin (one commit), 'rustup toolchain install $pin' on the Mac, 'RUST_TOOLCHAIN=$pin infra/build-server/run-from-mac.sh ' for the box, or set IGNEUM_TOOLCHAIN_MISMATCH=ok"; fi else bs_log "rustc $box on both sides${pin:+ (pinned $pin by rust-toolchain.toml)}"; fi } # Where am I? Sets BS_KIND (node = a worktree of the fork under vendor/; repo = a crate of the igneum repo), BS_TOP (the git # top level of the crate's repo), BS_WT_ROOT (the igneum worktree root), BS_WT (its name = the directory on the box), # BS_CRATE (the crate dir, = $PWD), BS_CRATE_REL (relative to BS_WT_ROOT), BS_MIRROR, BS_BRANCH, BS_SHA, BS_REMOTE_WT, # BS_REMOTE_CRATE, and BS_LOCAL_DIRS (every directory of a path dependency, relative to BS_WT_ROOT, from cargo metadata). bs_context() { local d BS_CRATE="$PWD" [ -f "$BS_CRATE/Cargo.toml" ] || bs_die "no Cargo.toml in $BS_CRATE: run from the crate directory (the fork worktree, igneum-pow, app/igneum-app, proving/igneum-prove)" BS_TOP=$(git -C "$BS_CRATE" rev-parse --show-toplevel 2>/dev/null) || bs_die "$BS_CRATE is not inside a git worktree" case "$BS_TOP" in */vendor/*) BS_KIND=node; BS_MIRROR=$BS_MIRROR_NODE BS_WT_ROOT=$(cd "$BS_TOP/../.." && pwd) [ -f "$BS_WT_ROOT/igneum-pow/Cargo.toml" ] || bs_die "$BS_TOP looks like a fork worktree but $BS_WT_ROOT/igneum-pow is missing" ;; *) BS_KIND=repo; BS_MIRROR=$BS_MIRROR_REPO; BS_WT_ROOT="$BS_TOP" ;; esac BS_WT=$(basename "$BS_WT_ROOT") BS_CRATE_REL=$(python3 -c 'import os, sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))' "$BS_CRATE" "$BS_WT_ROOT") BS_TOP_REL=$(python3 -c 'import os, sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))' "$BS_TOP" "$BS_WT_ROOT") case "$BS_CRATE_REL" in ..*) bs_die "$BS_CRATE is outside the worktree root $BS_WT_ROOT" ;; esac BS_BRANCH=$(git -C "$BS_TOP" branch --show-current 2>/dev/null || true) BS_SHA=$(git -C "$BS_TOP" rev-parse HEAD) [ -n "$BS_BRANCH" ] || BS_BRANCH="detached-$(git -C "$BS_TOP" rev-parse --short HEAD)" BS_REMOTE_WT="$BS_ROOT_REMOTE/$BS_WT" BS_REMOTE_CRATE="$BS_REMOTE_WT/$BS_CRATE_REL" # every local (path) package of the crate's dependency graph, as directories relative to the worktree root; the ones inside # BS_TOP are covered by the git checkout plus the overlay of BS_TOP itself (node kind) or synced one by one (repo kind) # A path dependency that lives inside another git repository under vendor/ (6 October 2026, the shipper's proving build: # proving/igneum-prove -> vendor/igneum-node-exec/igneum/evm-types, a MEMBER of the fork's workspace that inherits # `thiserror` from the fork's root manifest) is not a directory to copy: it needs its whole repository on the box, checked # out at the Mac's commit, as BS_TOP gets. Such repositories are listed in BS_VENDOR_REPOS (relative to the worktree root), # synced whole by bs_sync_sources, and dropped from BS_LOCAL_DIRS. BS_VENDOR_REPOS="" BS_LOCAL_DIRS=$(cd "$BS_CRATE" && "${CARGO_HOME:-$HOME/.cargo}/bin/cargo" metadata --format-version 1 2>/dev/null | python3 -c ' import json, os, subprocess, sys d = json.load(sys.stdin); root = sys.argv[1]; top = sys.argv[2]; kind = sys.argv[3] def toplevel(m): try: return subprocess.run(["git", "-C", m, "rev-parse", "--show-toplevel"], capture_output=True, text=True, check=True).stdout.strip() except subprocess.CalledProcessError: return None dirs, repos = set(), set() for p in d["packages"]: if p["source"] is not None: continue m = os.path.dirname(p["manifest_path"]) # a repository under vendor/ first: on disk it also lies under the igneum top level (vendor/ is ignored, not a # submodule), so the path test alone would take it for a directory of BS_TOP (the first run of this detector did) t = toplevel(m) if t and t != top and t.startswith(root + "/vendor/"): repos.add(t); continue if m == top or m.startswith(top + "/"): dirs.add(top if kind == "node" else m); continue dirs.add(m) def rel(m): r = os.path.relpath(m, root) if r.startswith(".."): sys.exit("path dependency %s is outside the worktree root %s" % (m, root)) return r print("\n".join(rel(m) for m in sorted(dirs))) print("VENDOR_REPOS " + " ".join(rel(t) for t in sorted(repos)))' "$BS_WT_ROOT" "$BS_TOP" "$BS_KIND") || bs_die "cargo metadata failed in $BS_CRATE" BS_VENDOR_REPOS=$(printf '%s\n' "$BS_LOCAL_DIRS" | sed -n 's/^VENDOR_REPOS //p') BS_LOCAL_DIRS=$(printf '%s\n' "$BS_LOCAL_DIRS" | grep -v '^VENDOR_REPOS ' || true) # Files a crate reaches by include_bytes!/include_str! with a relative path that LEAVES its repository (the shipper, 7 Oct 2026: # the fork's igneum-exec embeds proving/igneum-prove/elf/igneum-prove-program.vk five levels up, and the box build failed with # "couldn't read ...: No such file or directory" because such a file is no path dependency and the overlay never carried it). # Every .rs under the trees that travel is scanned; a path resolving outside BS_TOP but inside the worktree root adds its # directory to the overlay. proving/igneum-prove/elf is added for a fork build whatever the scan finds (the fixed fallback). local extra extra=$(python3 - "$BS_WT_ROOT" "$BS_TOP" $BS_LOCAL_DIRS $BS_VENDOR_REPOS <<'PY2' import os, re, sys root, top, rels = sys.argv[1], sys.argv[2], sys.argv[3:] rx = re.compile(r'include_(?:bytes|str)!\(\s*"((?:\.\./)+[^"]+)"') out = set() for rel in rels: base = os.path.join(root, rel) for dp, dn, fn in os.walk(base): dn[:] = [d for d in dn if d not in ("target", ".git") and not d.startswith("target-")] for f in fn: if not f.endswith(".rs"): continue p = os.path.join(dp, f) try: text = open(p, encoding="utf-8", errors="ignore").read() except OSError: continue for m in rx.finditer(text): a = os.path.normpath(os.path.join(dp, m.group(1))) if (a == top or a.startswith(top + "/")): continue if not a.startswith(root + "/"): continue out.add(os.path.relpath(os.path.dirname(a), root)) print("\n".join(sorted(out))) PY2 ) || extra="" [ "$BS_KIND" = node ] && [ -d "$BS_WT_ROOT/proving/igneum-prove/elf" ] && extra=$(printf '%s\n%s\n' "$extra" "proving/igneum-prove/elf" | grep . | sort -u) for d in $extra; do case " $BS_LOCAL_DIRS " in *" $d "*) ;; *) BS_LOCAL_DIRS="$BS_LOCAL_DIRS $d"; bs_log "included by include_bytes!/include_str! outside the crate's repository: $d" ;; esac done [ -n "$BS_LOCAL_DIRS$BS_VENDOR_REPOS" ] || bs_die "cargo metadata listed no local packages in $BS_CRATE" } # which bare mirror on the box holds a repository under vendor/: a worktree of the fork (its common git dir is # vendor/igneum-node/.git) or the fork itself -> /srv/igneum-node.git; any other repository of its own -> /srv/.git, # created on the box on first use (run-from-mac.sh wires and pushes the ones the Cargo.toml files reach) bs_mirror_for() { local dir="$1" common common=$(cd "$dir" && git rev-parse --git-common-dir 2>/dev/null) || { echo ""; return; } common=$(cd "$dir" && cd "$common" && pwd -P) case "$common" in */vendor/igneum-node/.git) echo "$BS_MIRROR_NODE" ;; *) echo "/srv/$(basename "$(dirname "$common")").git" ;; esac } # push the crate repo's HEAD to its bare mirror on the box (fast after the first time), then check the remote tree out at that # commit ON A BRANCH of that name: kaspa-build-info (build-info/build.rs try_git_head) embeds the commit only when .git is a # directory AND HEAD is a symbolic ref to a loose branch file; a detached HEAD or a worktree's .git file gives an empty hash # (which is why the Mac's worktree builds print "igneumd 2.1.0" with no commit, 6 Oct 2026). The mirror doubles as the CI # runner's source later. # bs_push_and_checkout the crate's own repository (BS_TOP) at BS_SHA on BS_BRANCH # bs_push_and_checkout another repository under vendor/ (a path dependency's), at its own HEAD bs_push_and_checkout() { local top="${1:-$BS_TOP}" rel="${2:-$BS_TOP_REL}" mirror branch sha url remote_top if [ -z "${1:-}" ]; then mirror="$BS_MIRROR"; branch="$BS_BRANCH"; sha="$BS_SHA"; else mirror=$(bs_mirror_for "$top"); [ -n "$mirror" ] || bs_die "$top is not a git repository" sha=$(git -C "$top" rev-parse HEAD); branch=$(git -C "$top" branch --show-current 2>/dev/null || true); [ -n "$branch" ] || branch="detached-$(git -C "$top" rev-parse --short HEAD)" [ "$mirror" = "$BS_MIRROR_NODE" ] || [ "$mirror" = "$BS_MIRROR_REPO" ] || bs_ssh "[ -d '$mirror' ] || git init -q --bare -b master '$mirror'" || bs_die "cannot create the mirror $mirror on the box" fi url="$BS_HOST:$mirror"; remote_top="$BS_REMOTE_WT/$rel" [ "$rel" = . ] && remote_top="$BS_REMOTE_WT" bs_log "push $top HEAD $sha ($branch) -> $url" GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$top" push -q --force "$url" "HEAD:refs/heads/$branch" || bs_die "push to the mirror failed" # the checkout runs as remote-run.sh's `checkout` mode: discard the previous overlay (tracked edits and untracked files, # target dirs kept), then the branch at the commit. 6 October 2026, PC 1 worker's first use: the overlay of an earlier # commit's uncommitted files stayed in the box's tree and `git checkout -B` refused with "local changes would be overwritten". BR_MODE=checkout BR_CO_DIR="$remote_top" BR_CO_MIRROR="$mirror" BR_CO_BRANCH="$branch" BR_CO_SHA="$sha" BR_CO_WT="$BS_REMOTE_WT" \ bash -c ' for v in BR_MODE BR_CO_DIR BR_CO_MIRROR BR_CO_BRANCH BR_CO_SHA BR_CO_WT; do printf "export %s=%q\n" "$v" "${!v}"; done cat "$0"' "$(dirname "${BASH_SOURCE[0]}")/remote-run.sh" | bs_ssh 'bash -s' || bs_die "remote checkout at $remote_top failed" BS_REMOTE_TOP="$remote_top" } # rsync one directory of the worktree to the same place on the box. By checksum and WITHOUT preserving times, so a file whose # content changed is written with the box's clock and nothing older than the last build slips past cargo's mtime check (the # stale-build class, 4 and 5 October 2026); the files rsync wrote are listed and re-stamped with touch as well, so the rule is # visible here and tools/ci/copied-sources-check.sh sees it. target dirs and .git never travel; --delete keeps the box equal to # the Mac inside the directory (excluded paths are protected). bs_overlay_dir() { local rel="$1" src="$BS_WT_ROOT/$1" dst="$BS_REMOTE_WT/$1" list nfiles ndirs [ -d "$src" ] || bs_die "no $src" list=$(mktemp) bs_ssh "mkdir -p '$dst'" bs_rsync -rlpgoD --checksum --delete --out-format='%n' \ --exclude '/target' --exclude '/target-*' --exclude '/target/' --exclude 'target-*/' --exclude '.git' --exclude '.DS_Store' --exclude 'node_modules' \ "$src/" "$BS_HOST:$dst/" > "$list" || { rm -f "$list"; bs_die "rsync of $rel failed"; } # files only: directories are listed whenever an attribute differs (a fresh clone's ownership), and a tree whose files # were all identical lists ONLY directories (first run of 6 October 2026: an empty file list failed the pipeline) nfiles=$(grep -vc '/$' "$list" || true); ndirs=$(grep -c '/$' "$list" || true) if [ "${nfiles:-0}" -gt 0 ]; then if ! grep -v '/$' "$list" | tr '\n' '\0' | bs_ssh "cd '$dst' && xargs -0 -r touch --no-create"; then rm -f "$list"; bs_die "re-stamp of $rel failed"; fi fi bs_log "overlay $rel -> $dst: ${nfiles:-0} file(s) written and re-stamped, ${ndirs:-0} dir(s)" rm -f "$list" } # one run per worktree at a time on the box (the shipper, 6 October 2026, 18:48:56Z: a second run's checkout replaced the first's # sources mid-cargo and both died). The lock is a directory under /srv/builds/_locks made atomically with mkdir and holding the # Mac's pid, time and label; it spans sync, build and fetch (bs_wt_unlock on EXIT). A waiter polls every 10 s for up to 2 h and # takes over a lock older than 3 h (a Mac that died mid-run). bs_wt_lock() { local d="$BS_ROOT_REMOTE/_locks/wt-$BS_WT" t0 holder t0=$(date +%s) while :; do if bs_ssh "mkdir '$d' 2>/dev/null && printf 'pid %s since %sZ: %s\n' '$$' \"\$(date -u +%H:%M:%S)\" '${BS_TOOL:-build} $BS_CRATE_REL' > '$d/holder'"; then BS_WT_LOCKED="$d"; trap 'bs_wt_unlock' EXIT; return 0; fi holder=$(bs_ssh "cat '$d/holder' 2>/dev/null; find '$d' -maxdepth 0 -mmin +180 -print 2>/dev/null | grep -q . && echo STALE" 2>/dev/null || true) case "$holder" in *STALE*) bs_log "worktree lock $d is older than 3 h; taking it over"; bs_ssh "rm -rf '$d'"; continue ;; esac [ $(( $(date +%s) - t0 )) -lt 7200 ] || bs_die "gave up after 2 h waiting for the worktree lock $d (held: $holder)" [ $(( ($(date +%s) - t0) % 60 )) -lt 10 ] && bs_log "waiting for another run on worktree $BS_WT: ${holder:-?}" sleep 10 done } bs_wt_unlock() { [ -n "${BS_WT_LOCKED:-}" ] && bs_ssh "rm -rf '$BS_WT_LOCKED'" 2>/dev/null; BS_WT_LOCKED=""; } bs_sync_sources() { bs_wt_lock local d bs_push_and_checkout for d in $BS_LOCAL_DIRS; do bs_overlay_dir "$d"; done for d in $BS_VENDOR_REPOS; do bs_push_and_checkout "$BS_WT_ROOT/$d" "$d"; bs_overlay_dir "$d"; done } # Reproducible builds (main, 6 October 2026, from the 0.3.14 repro docs/evidence/reproduced/0.3.14.md): two classes made two # builds of one tree differ and sccache hid both. (1) prost's generated protowire.rs embeds OUT_DIR, so the TARGET PATH must be # the same between builds: every tool here builds into one fixed directory per target (`target`, or the name --target-dir gives, # never a per-run name). (2) libmimalloc-sys compiles mimalloc's C with __DATE__/__TIME__, so SOURCE_DATE_EPOCH is set to the # commit's author time and TZ to UTC; the same two exports go into the Mac's cross-build.sh and the PC job (jobbuild.rs). # bs_repro_env prints the export line a remote command starts with; BR_SDE carries the value into remote-run.sh's JSONL line. bs_sde() { git -C "${1:-$BS_TOP}" log -1 --format=%at HEAD; } bs_repro_env() { local sde; sde=$(bs_sde "${1:-$BS_TOP}"); BR_SDE="$sde"; export BR_SDE; printf 'export SOURCE_DATE_EPOCH=%s TZ=UTC; ' "$sde"; } bs_sha256() { shasum -a 256 "$1" | awk '{ print $1 }'; } bs_size() { stat -f %z "$1" 2>/dev/null || stat -c %s "$1"; } bs_fmt_secs() { local s=$1; printf '%d min %02d s' $((s / 60)) $((s % 60)); } # kind of a run for the box's JSONL log (main's rule of 6 October 2026): bs_kind() { local tool="$1" word="$2" case "$word" in test) echo suite; return ;; check|clippy) echo check; return ;; esac if [ "$tool" = cross-remote ]; then case "$BS_KIND:$BS_CRATE_REL" in node:*) echo node-windows ;; repo:app/igneum-app) echo app-windows ;; *) echo other ;; esac; return fi case "$BS_KIND:$BS_CRATE_REL" in node:*) echo node-linux ;; repo:app/igneum-app) echo app ;; repo:proving/igneum-prove) echo prove ;; *) echo other ;; esac } # the remote runner (infra/build-server/remote-run.sh, piped to `bash -s` on the box behind the BR_* exports): takes one of the # box's build slots (never the Mac's), runs the command in the crate dir with sccache, prints the RESULT line and appends one # JSON line to /srv/builds/_log/builds.jsonl for the worker dashboard. # bs_remote_run