igneum/infra/build-server/lib.sh
igneum-labs 67ae1e4c6d Reproducible builds: SOURCE_DATE_EPOCH from the commit's author time, TZ=UTC and one fixed target path in every build path; self-test
Main's rule of 6 October 2026 from the 0.3.14 repro (docs/evidence/reproduced/0.3.14.md): prost's protowire.rs embeds OUT_DIR,
libmimalloc-sys embeds __DATE__/__TIME__, sccache hid both. lib.sh bs_repro_env exports SOURCE_DATE_EPOCH=<author time> TZ=UTC in
front of every remote command (build-remote.sh, cross-remote.sh, workers-remote.sh); remote-run.sh exports BR_SDE too and logs it
as source_date_epoch; proto-cuda/windows-node/cross-build.sh exports the same; the PC job carries node.commit_time in the manifest
(push-build-inputs.sh) and exports it before every cargo build of a stage (jobbuild.rs, unit test asserts it; 4 of 4 pass on the
box). Target dirs stay one fixed path per target. tools/build-remote.sh --self-test-repro [--full] from a fork worktree, run on
the box: igneum-miner twice a minute apart without sccache (RUSTC_WRAPPER=/usr/bin/env, an empty value is unset to cargo) MATCH
91e130f5..., a per-run target path differs (OUT_DIR shown); --full: kaspad with libmimalloc-sys recompiled a minute later MATCH
70219bc2..., without the epoch differs (__DATE__ shown).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 20:09:23 +00:00

248 lines
17 KiB
Bash
Executable file

#!/usr/bin/env bash
# Shared by infra/build-server/run-from-mac.sh, tools/build-remote.sh and tools/cross-remote.sh. Source it, do not run it.
# Everything that talks to igneum-build-1 from the Mac goes through here: the host line, the ssh options (the ops key
# ~/.ssh/igneum_ed25519, a shared control socket so one build is one ssh session), the mirror push, the remote checkout
# and the source overlay (rsync by checksum, changed files re-stamped: the copied-sources rule of 5 October 2026).
#
# Layout on the box (provision.sh): /srv/builds/<worktree> mirrors the Mac's igneum worktree ROOT (the directory that holds
# igneum-pow/, app/, proving/ and vendor/), so the fork's relative path dependency `../../../../igneum-pow`
# (vendor/igneum-node/consensus/pow/Cargo.toml) resolves on the box exactly as on the Mac:
# Mac /Users/joshm/Projects/igneum-wt-ship0311/vendor/igneum-node-0311 -> box /srv/builds/igneum-wt-ship0311/vendor/igneum-node-0311
# Mac /Users/joshm/Projects/igneum-wt-ship0311/igneum-pow -> box /srv/builds/igneum-wt-ship0311/igneum-pow
# Mac /Users/joshm/Projects/igneum/app/igneum-app -> box /srv/builds/igneum/app/igneum-app
# The fork's kaspa-build-info reads `git rev-parse HEAD` at build time and the release plans check the commit in the binary's
# strings, so the fork tree on the box is a real clone of the bare mirror /srv/igneum-node.git checked out at the Mac's HEAD,
# with the Mac's uncommitted changes rsynced on top. The igneum repo's crates get the same from /srv/igneum.git.
# shellcheck disable=SC2034 # shared with the scripts that source lib.sh
BS_KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
BS_HOST_FILE="${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" # one line: build@<ip>
BS_ROOT_REMOTE=/srv/builds
BS_MIRROR_REPO=/srv/igneum.git
BS_MIRROR_NODE=/srv/igneum-node.git
bs_log() { printf '%s %s: %s\n' "$(date -u +%H:%M:%S)" "${BS_TOOL:-build-server}" "$*" >&2; }
bs_die() { bs_log "ERROR: $*"; exit 1; }
bs_host() {
BS_HOST="${BUILD_HOST:-}"
if [ -z "$BS_HOST" ]; then
[ -s "$BS_HOST_FILE" ] || bs_die "no build server: write build@<ip> to $BS_HOST_FILE (infra/build-server/run-from-mac.sh does) or set BUILD_HOST"
BS_HOST="$(head -1 "$BS_HOST_FILE" | tr -d '[:space:]')"
fi
case "$BS_HOST" in *@*) ;; *) bs_die "BUILD_HOST must be user@host, got '$BS_HOST'" ;; esac
[ -r "$BS_KEY" ] || bs_die "no ssh key at $BS_KEY"
mkdir -p "$HOME/.ssh/cm"
BS_SSH_OPTS=(-i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=6
-o ControlMaster=auto -o ControlPath="$HOME/.ssh/cm/igneum-build-%r@%h:%p" -o ControlPersist=900)
BS_SSH_CMD="ssh"; local o; for o in "${BS_SSH_OPTS[@]}"; do BS_SSH_CMD="$BS_SSH_CMD $(printf '%q' "$o")"; done
}
bs_ssh() { ssh "${BS_SSH_OPTS[@]}" "$BS_HOST" "$@"; }
bs_rsync() { rsync -e "$BS_SSH_CMD" "$@"; }
# the two sides' rustc must agree (the box is pinned by provision.sh RUST_TOOLCHAIN; the Mac runs rustup's stable):
# a different compiler gives different bytes and, across a minor version, different lints and errors
bs_toolchain_check() {
local mac box
mac=$("${CARGO_HOME:-$HOME/.cargo}/bin/rustc" --version 2>/dev/null | awk '{ print $2 }')
box=$(bs_ssh '. /etc/profile.d/igneum-build.sh; rustc --version' 2>/dev/null | awk '{ print $2 }')
[ -n "$box" ] || bs_die "cannot read rustc on $BS_HOST (is it provisioned? infra/build-server/run-from-mac.sh)"
if [ "$mac" != "$box" ]; then
if [ "${IGNEUM_TOOLCHAIN_MISMATCH:-}" = ok ]; then bs_log "WARNING: rustc $mac on the Mac, $box on the box (IGNEUM_TOOLCHAIN_MISMATCH=ok)"
else bs_die "rustc $mac on the Mac, $box on the box; re-provision with RUST_TOOLCHAIN=$mac or set IGNEUM_TOOLCHAIN_MISMATCH=ok"; fi
else bs_log "rustc $box on both sides"; fi
}
# Where am I? Sets BS_KIND (node = a worktree of the fork under vendor/; repo = a crate of the igneum repo), BS_TOP (the git
# top level of the crate's repo), BS_WT_ROOT (the igneum worktree root), BS_WT (its name = the directory on the box),
# BS_CRATE (the crate dir, = $PWD), BS_CRATE_REL (relative to BS_WT_ROOT), BS_MIRROR, BS_BRANCH, BS_SHA, BS_REMOTE_WT,
# BS_REMOTE_CRATE, and BS_LOCAL_DIRS (every directory of a path dependency, relative to BS_WT_ROOT, from cargo metadata).
bs_context() {
BS_CRATE="$PWD"
[ -f "$BS_CRATE/Cargo.toml" ] || bs_die "no Cargo.toml in $BS_CRATE: run from the crate directory (the fork worktree, igneum-pow, app/igneum-app, proving/igneum-prove)"
BS_TOP=$(git -C "$BS_CRATE" rev-parse --show-toplevel 2>/dev/null) || bs_die "$BS_CRATE is not inside a git worktree"
case "$BS_TOP" in
*/vendor/*)
BS_KIND=node; BS_MIRROR=$BS_MIRROR_NODE
BS_WT_ROOT=$(cd "$BS_TOP/../.." && pwd)
[ -f "$BS_WT_ROOT/igneum-pow/Cargo.toml" ] || bs_die "$BS_TOP looks like a fork worktree but $BS_WT_ROOT/igneum-pow is missing"
;;
*)
BS_KIND=repo; BS_MIRROR=$BS_MIRROR_REPO; BS_WT_ROOT="$BS_TOP"
;;
esac
BS_WT=$(basename "$BS_WT_ROOT")
BS_CRATE_REL=$(python3 -c 'import os, sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))' "$BS_CRATE" "$BS_WT_ROOT")
BS_TOP_REL=$(python3 -c 'import os, sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))' "$BS_TOP" "$BS_WT_ROOT")
case "$BS_CRATE_REL" in ..*) bs_die "$BS_CRATE is outside the worktree root $BS_WT_ROOT" ;; esac
BS_BRANCH=$(git -C "$BS_TOP" branch --show-current 2>/dev/null || true)
BS_SHA=$(git -C "$BS_TOP" rev-parse HEAD)
[ -n "$BS_BRANCH" ] || BS_BRANCH="detached-$(git -C "$BS_TOP" rev-parse --short HEAD)"
BS_REMOTE_WT="$BS_ROOT_REMOTE/$BS_WT"
BS_REMOTE_CRATE="$BS_REMOTE_WT/$BS_CRATE_REL"
# every local (path) package of the crate's dependency graph, as directories relative to the worktree root; the ones inside
# BS_TOP are covered by the git checkout plus the overlay of BS_TOP itself (node kind) or synced one by one (repo kind)
# A path dependency that lives inside another git repository under vendor/ (6 October 2026, the shipper's proving build:
# proving/igneum-prove -> vendor/igneum-node-exec/igneum/evm-types, a MEMBER of the fork's workspace that inherits
# `thiserror` from the fork's root manifest) is not a directory to copy: it needs its whole repository on the box, checked
# out at the Mac's commit, as BS_TOP gets. Such repositories are listed in BS_VENDOR_REPOS (relative to the worktree root),
# synced whole by bs_sync_sources, and dropped from BS_LOCAL_DIRS.
BS_VENDOR_REPOS=""
BS_LOCAL_DIRS=$(cd "$BS_CRATE" && "${CARGO_HOME:-$HOME/.cargo}/bin/cargo" metadata --format-version 1 2>/dev/null | python3 -c '
import json, os, subprocess, sys
d = json.load(sys.stdin); root = sys.argv[1]; top = sys.argv[2]; kind = sys.argv[3]
def toplevel(m):
try: return subprocess.run(["git", "-C", m, "rev-parse", "--show-toplevel"], capture_output=True, text=True, check=True).stdout.strip()
except subprocess.CalledProcessError: return None
dirs, repos = set(), set()
for p in d["packages"]:
if p["source"] is not None: continue
m = os.path.dirname(p["manifest_path"])
# a repository under vendor/ first: on disk it also lies under the igneum top level (vendor/ is ignored, not a
# submodule), so the path test alone would take it for a directory of BS_TOP (the first run of this detector did)
t = toplevel(m)
if t and t != top and t.startswith(root + "/vendor/"):
repos.add(t); continue
if m == top or m.startswith(top + "/"):
dirs.add(top if kind == "node" else m); continue
dirs.add(m)
def rel(m):
r = os.path.relpath(m, root)
if r.startswith(".."): sys.exit("path dependency %s is outside the worktree root %s" % (m, root))
return r
print("\n".join(rel(m) for m in sorted(dirs)))
print("VENDOR_REPOS " + " ".join(rel(t) for t in sorted(repos)))' "$BS_WT_ROOT" "$BS_TOP" "$BS_KIND") || bs_die "cargo metadata failed in $BS_CRATE"
BS_VENDOR_REPOS=$(printf '%s\n' "$BS_LOCAL_DIRS" | sed -n 's/^VENDOR_REPOS //p')
BS_LOCAL_DIRS=$(printf '%s\n' "$BS_LOCAL_DIRS" | grep -v '^VENDOR_REPOS ' || true)
[ -n "$BS_LOCAL_DIRS$BS_VENDOR_REPOS" ] || bs_die "cargo metadata listed no local packages in $BS_CRATE"
}
# which bare mirror on the box holds a repository under vendor/: a worktree of the fork (its common git dir is
# vendor/igneum-node/.git) or the fork itself -> /srv/igneum-node.git; any other repository of its own -> /srv/<name>.git,
# created on the box on first use (run-from-mac.sh wires and pushes the ones the Cargo.toml files reach)
bs_mirror_for() {
local dir="$1" common
common=$(cd "$dir" && git rev-parse --git-common-dir 2>/dev/null) || { echo ""; return; }
common=$(cd "$dir" && cd "$common" && pwd -P)
case "$common" in */vendor/igneum-node/.git) echo "$BS_MIRROR_NODE" ;; *) echo "/srv/$(basename "$(dirname "$common")").git" ;; esac
}
# push the crate repo's HEAD to its bare mirror on the box (fast after the first time), then check the remote tree out at that
# commit ON A BRANCH of that name: kaspa-build-info (build-info/build.rs try_git_head) embeds the commit only when .git is a
# directory AND HEAD is a symbolic ref to a loose branch file; a detached HEAD or a worktree's .git file gives an empty hash
# (which is why the Mac's worktree builds print "igneumd 2.1.0" with no commit, 6 Oct 2026). The mirror doubles as the CI
# runner's source later.
# bs_push_and_checkout the crate's own repository (BS_TOP) at BS_SHA on BS_BRANCH
# bs_push_and_checkout <local repo> <rel> another repository under vendor/ (a path dependency's), at its own HEAD
bs_push_and_checkout() {
local top="${1:-$BS_TOP}" rel="${2:-$BS_TOP_REL}" mirror branch sha url remote_top
if [ -z "${1:-}" ]; then mirror="$BS_MIRROR"; branch="$BS_BRANCH"; sha="$BS_SHA"; else
mirror=$(bs_mirror_for "$top"); [ -n "$mirror" ] || bs_die "$top is not a git repository"
sha=$(git -C "$top" rev-parse HEAD); branch=$(git -C "$top" branch --show-current 2>/dev/null || true); [ -n "$branch" ] || branch="detached-$(git -C "$top" rev-parse --short HEAD)"
[ "$mirror" = "$BS_MIRROR_NODE" ] || [ "$mirror" = "$BS_MIRROR_REPO" ] || bs_ssh "[ -d '$mirror' ] || git init -q --bare -b master '$mirror'" || bs_die "cannot create the mirror $mirror on the box"
fi
url="$BS_HOST:$mirror"; remote_top="$BS_REMOTE_WT/$rel"
[ "$rel" = . ] && remote_top="$BS_REMOTE_WT"
bs_log "push $top HEAD $sha ($branch) -> $url"
GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$top" push -q --force "$url" "HEAD:refs/heads/$branch" || bs_die "push to the mirror failed"
# the checkout runs as remote-run.sh's `checkout` mode: discard the previous overlay (tracked edits and untracked files,
# target dirs kept), then the branch at the commit. 6 October 2026, PC 1 worker's first use: the overlay of an earlier
# commit's uncommitted files stayed in the box's tree and `git checkout -B` refused with "local changes would be overwritten".
BR_MODE=checkout BR_CO_DIR="$remote_top" BR_CO_MIRROR="$mirror" BR_CO_BRANCH="$branch" BR_CO_SHA="$sha" BR_CO_WT="$BS_REMOTE_WT" \
bash -c '
for v in BR_MODE BR_CO_DIR BR_CO_MIRROR BR_CO_BRANCH BR_CO_SHA BR_CO_WT; do printf "export %s=%q\n" "$v" "${!v}"; done
cat "$0"' "$(dirname "${BASH_SOURCE[0]}")/remote-run.sh" | bs_ssh 'bash -s' || bs_die "remote checkout at $remote_top failed"
BS_REMOTE_TOP="$remote_top"
}
# rsync one directory of the worktree to the same place on the box. By checksum and WITHOUT preserving times, so a file whose
# content changed is written with the box's clock and nothing older than the last build slips past cargo's mtime check (the
# stale-build class, 4 and 5 October 2026); the files rsync wrote are listed and re-stamped with touch as well, so the rule is
# visible here and tools/ci/copied-sources-check.sh sees it. target dirs and .git never travel; --delete keeps the box equal to
# the Mac inside the directory (excluded paths are protected).
bs_overlay_dir() {
local rel="$1" src="$BS_WT_ROOT/$1" dst="$BS_REMOTE_WT/$1" list nfiles ndirs
[ -d "$src" ] || bs_die "no $src"
list=$(mktemp)
bs_ssh "mkdir -p '$dst'"
bs_rsync -rlpgoD --checksum --delete --out-format='%n' \
--exclude '/target' --exclude '/target-*' --exclude '/target/' --exclude 'target-*/' --exclude '.git' --exclude '.DS_Store' --exclude 'node_modules' \
"$src/" "$BS_HOST:$dst/" > "$list" || { rm -f "$list"; bs_die "rsync of $rel failed"; }
# files only: directories are listed whenever an attribute differs (a fresh clone's ownership), and a tree whose files
# were all identical lists ONLY directories (first run of 6 October 2026: an empty file list failed the pipeline)
nfiles=$(grep -vc '/$' "$list" || true); ndirs=$(grep -c '/$' "$list" || true)
if [ "${nfiles:-0}" -gt 0 ]; then
if ! grep -v '/$' "$list" | tr '\n' '\0' | bs_ssh "cd '$dst' && xargs -0 -r touch --no-create"; then rm -f "$list"; bs_die "re-stamp of $rel failed"; fi
fi
bs_log "overlay $rel -> $dst: ${nfiles:-0} file(s) written and re-stamped, ${ndirs:-0} dir(s)"
rm -f "$list"
}
# one run per worktree at a time on the box (the shipper, 6 October 2026, 18:48:56Z: a second run's checkout replaced the first's
# sources mid-cargo and both died). The lock is a directory under /srv/builds/_locks made atomically with mkdir and holding the
# Mac's pid, time and label; it spans sync, build and fetch (bs_wt_unlock on EXIT). A waiter polls every 10 s for up to 2 h and
# takes over a lock older than 3 h (a Mac that died mid-run).
bs_wt_lock() {
local d="$BS_ROOT_REMOTE/_locks/wt-$BS_WT" t0 holder
t0=$(date +%s)
while :; do
if bs_ssh "mkdir '$d' 2>/dev/null && printf 'pid %s since %sZ: %s\n' '$$' \"\$(date -u +%H:%M:%S)\" '${BS_TOOL:-build} $BS_CRATE_REL' > '$d/holder'"; then BS_WT_LOCKED="$d"; trap 'bs_wt_unlock' EXIT; return 0; fi
holder=$(bs_ssh "cat '$d/holder' 2>/dev/null; find '$d' -maxdepth 0 -mmin +180 -print 2>/dev/null | grep -q . && echo STALE" 2>/dev/null || true)
case "$holder" in *STALE*) bs_log "worktree lock $d is older than 3 h; taking it over"; bs_ssh "rm -rf '$d'"; continue ;; esac
[ $(( $(date +%s) - t0 )) -lt 7200 ] || bs_die "gave up after 2 h waiting for the worktree lock $d (held: $holder)"
[ $(( ($(date +%s) - t0) % 60 )) -lt 10 ] && bs_log "waiting for another run on worktree $BS_WT: ${holder:-?}"
sleep 10
done
}
bs_wt_unlock() { [ -n "${BS_WT_LOCKED:-}" ] && bs_ssh "rm -rf '$BS_WT_LOCKED'" 2>/dev/null; BS_WT_LOCKED=""; }
bs_sync_sources() {
bs_wt_lock
local d
bs_push_and_checkout
for d in $BS_LOCAL_DIRS; do bs_overlay_dir "$d"; done
for d in $BS_VENDOR_REPOS; do bs_push_and_checkout "$BS_WT_ROOT/$d" "$d"; bs_overlay_dir "$d"; done
}
# Reproducible builds (main, 6 October 2026, from the 0.3.14 repro docs/evidence/reproduced/0.3.14.md): two classes made two
# builds of one tree differ and sccache hid both. (1) prost's generated protowire.rs embeds OUT_DIR, so the TARGET PATH must be
# the same between builds: every tool here builds into one fixed directory per target (`target`, or the name --target-dir gives,
# never a per-run name). (2) libmimalloc-sys compiles mimalloc's C with __DATE__/__TIME__, so SOURCE_DATE_EPOCH is set to the
# commit's author time and TZ to UTC; the same two exports go into the Mac's cross-build.sh and the PC job (jobbuild.rs).
# bs_repro_env prints the export line a remote command starts with; BR_SDE carries the value into remote-run.sh's JSONL line.
bs_sde() { git -C "${1:-$BS_TOP}" log -1 --format=%at HEAD; }
bs_repro_env() { local sde; sde=$(bs_sde "${1:-$BS_TOP}"); BR_SDE="$sde"; export BR_SDE; printf 'export SOURCE_DATE_EPOCH=%s TZ=UTC; ' "$sde"; }
bs_sha256() { shasum -a 256 "$1" | awk '{ print $1 }'; }
bs_size() { stat -f %z "$1" 2>/dev/null || stat -c %s "$1"; }
bs_fmt_secs() { local s=$1; printf '%d min %02d s' $((s / 60)) $((s % 60)); }
# kind of a run for the box's JSONL log (main's rule of 6 October 2026): <tool> <first cargo word>
bs_kind() {
local tool="$1" word="$2"
case "$word" in test) echo suite; return ;; check|clippy) echo check; return ;; esac
if [ "$tool" = cross-remote ]; then
case "$BS_KIND:$BS_CRATE_REL" in node:*) echo node-windows ;; repo:app/igneum-app) echo app-windows ;; *) echo other ;; esac; return
fi
case "$BS_KIND:$BS_CRATE_REL" in node:*) echo node-linux ;; repo:app/igneum-app) echo app ;; repo:proving/igneum-prove) echo prove ;; *) echo other ;; esac
}
# the remote runner (infra/build-server/remote-run.sh, piped to `bash -s` on the box behind the BR_* exports): takes one of the
# box's build slots (never the Mac's), runs the command in the crate dir with sccache, prints the RESULT line and appends one
# JSON line to /srv/builds/_log/builds.jsonl for the worker dashboard.
# bs_remote_run <remote crate dir> <label> <shell command string>
# with BR_KIND, BR_COMMAND, BR_TARGET and BR_ARTEFACTS set by the caller; the agent name is IGNEUM_AGENT (default the worktree)
# and is appended to the label as "; agent=<name>".
bs_remote_run() {
local dir="$1" label="$2" cmd="$3" agent="${IGNEUM_AGENT:-$BS_WT}" v
label="$label; agent=$agent"
BR_DIR="$dir" BR_LABEL="$label" BR_CMD="$cmd" BR_TOOL="${BS_TOOL:-build-remote}" BR_WT="$BS_WT" BR_CRATE="$BS_CRATE_REL" \
BR_BRANCH="$BS_BRANCH" BR_SHA="$BS_SHA" BR_AGENT="$agent" BR_KIND="${BR_KIND:-other}" BR_COMMAND="${BR_COMMAND:-}" \
BR_TARGET="${BR_TARGET:-}" BR_ARTEFACTS="${BR_ARTEFACTS:-}" BR_SDE="${BR_SDE:-}" \
bash -c '
for v in BR_DIR BR_LABEL BR_CMD BR_TOOL BR_WT BR_CRATE BR_BRANCH BR_SHA BR_AGENT BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS BR_SDE; do
printf "export %s=%q\n" "$v" "${!v}"
done
cat "$0"' "$(dirname "${BASH_SOURCE[0]}")/remote-run.sh" | bs_ssh 'bash -s'
}