Build server: tools/build-remote.sh, tools/cross-remote.sh, infra/build-server/{lib,run-from-mac}.sh
build-remote.sh runs a cargo command on igneum-build-1 from any crate directory of any worktree: HEAD through the bare mirror (a real .git for kaspa-build-info), uncommitted changes by rsync --checksum with the written files re-stamped, a remote slot (/srv/builds/_locks, never the Mac's), sccache, -j 90, artefacts back into target-remote/ with size and sha256. cross-remote.sh is the Windows cross-build with the PC job's Ubuntu mingw-posix recipe plus the Mac's static flags, DLL list and sha256 per exe, --compare against the Mac's exes. run-from-mac.sh ships provision.sh, writes ~/.config/igneum/build-server, adds the build remotes and pushes every branch of both repos. shellcheck and the CI checks clean. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
parent
cd99adcd88
commit
7b61483f4b
5 changed files with 451 additions and 0 deletions
184
infra/build-server/lib.sh
Executable file
184
infra/build-server/lib.sh
Executable file
|
|
@ -0,0 +1,184 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Shared by infra/build-server/run-from-mac.sh, tools/build-remote.sh and tools/cross-remote.sh. Source it, do not run it.
|
||||||
|
# Everything that talks to igneum-build-1 from the Mac goes through here: the host line, the ssh options (the ops key
|
||||||
|
# ~/.ssh/igneum_ed25519, a shared control socket so one build is one ssh session), the mirror push, the remote checkout
|
||||||
|
# and the source overlay (rsync by checksum, changed files re-stamped: the copied-sources rule of 5 October 2026).
|
||||||
|
#
|
||||||
|
# Layout on the box (provision.sh): /srv/builds/<worktree> mirrors the Mac's igneum worktree ROOT (the directory that holds
|
||||||
|
# igneum-pow/, app/, proving/ and vendor/), so the fork's relative path dependency `../../../../igneum-pow`
|
||||||
|
# (vendor/igneum-node/consensus/pow/Cargo.toml) resolves on the box exactly as on the Mac:
|
||||||
|
# Mac /Users/joshm/Projects/igneum-wt-ship0311/vendor/igneum-node-0311 -> box /srv/builds/igneum-wt-ship0311/vendor/igneum-node-0311
|
||||||
|
# Mac /Users/joshm/Projects/igneum-wt-ship0311/igneum-pow -> box /srv/builds/igneum-wt-ship0311/igneum-pow
|
||||||
|
# Mac /Users/joshm/Projects/igneum/app/igneum-app -> box /srv/builds/igneum/app/igneum-app
|
||||||
|
# The fork's kaspa-build-info reads `git rev-parse HEAD` at build time and the release plans check the commit in the binary's
|
||||||
|
# strings, so the fork tree on the box is a real clone of the bare mirror /srv/igneum-node.git checked out at the Mac's HEAD,
|
||||||
|
# with the Mac's uncommitted changes rsynced on top. The igneum repo's crates get the same from /srv/igneum.git.
|
||||||
|
|
||||||
|
# shellcheck disable=SC2034 # shared with the scripts that source lib.sh
|
||||||
|
BS_KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
|
||||||
|
BS_HOST_FILE="${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" # one line: build@<ip>
|
||||||
|
BS_ROOT_REMOTE=/srv/builds
|
||||||
|
BS_MIRROR_REPO=/srv/igneum.git
|
||||||
|
BS_MIRROR_NODE=/srv/igneum-node.git
|
||||||
|
|
||||||
|
bs_log() { printf '%s %s: %s\n' "$(date -u +%H:%M:%S)" "${BS_TOOL:-build-server}" "$*" >&2; }
|
||||||
|
bs_die() { bs_log "ERROR: $*"; exit 1; }
|
||||||
|
|
||||||
|
bs_host() {
|
||||||
|
BS_HOST="${BUILD_HOST:-}"
|
||||||
|
if [ -z "$BS_HOST" ]; then
|
||||||
|
[ -s "$BS_HOST_FILE" ] || bs_die "no build server: write build@<ip> to $BS_HOST_FILE (infra/build-server/run-from-mac.sh does) or set BUILD_HOST"
|
||||||
|
BS_HOST="$(head -1 "$BS_HOST_FILE" | tr -d '[:space:]')"
|
||||||
|
fi
|
||||||
|
case "$BS_HOST" in *@*) ;; *) bs_die "BUILD_HOST must be user@host, got '$BS_HOST'" ;; esac
|
||||||
|
[ -r "$BS_KEY" ] || bs_die "no ssh key at $BS_KEY"
|
||||||
|
mkdir -p "$HOME/.ssh/cm"
|
||||||
|
BS_SSH_OPTS=(-i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=6
|
||||||
|
-o ControlMaster=auto -o ControlPath="$HOME/.ssh/cm/igneum-build-%r@%h:%p" -o ControlPersist=900)
|
||||||
|
BS_SSH_CMD="ssh"; local o; for o in "${BS_SSH_OPTS[@]}"; do BS_SSH_CMD="$BS_SSH_CMD $(printf '%q' "$o")"; done
|
||||||
|
}
|
||||||
|
|
||||||
|
bs_ssh() { ssh "${BS_SSH_OPTS[@]}" "$BS_HOST" "$@"; }
|
||||||
|
bs_rsync() { rsync -e "$BS_SSH_CMD" "$@"; }
|
||||||
|
|
||||||
|
# the two sides' rustc must agree (the box is pinned by provision.sh RUST_TOOLCHAIN; the Mac runs rustup's stable):
|
||||||
|
# a different compiler gives different bytes and, across a minor version, different lints and errors
|
||||||
|
bs_toolchain_check() {
|
||||||
|
local mac box
|
||||||
|
mac=$("${CARGO_HOME:-$HOME/.cargo}/bin/rustc" --version 2>/dev/null | awk '{ print $2 }')
|
||||||
|
box=$(bs_ssh '. /etc/profile.d/igneum-build.sh; rustc --version' 2>/dev/null | awk '{ print $2 }')
|
||||||
|
[ -n "$box" ] || bs_die "cannot read rustc on $BS_HOST (is it provisioned? infra/build-server/run-from-mac.sh)"
|
||||||
|
if [ "$mac" != "$box" ]; then
|
||||||
|
if [ "${IGNEUM_TOOLCHAIN_MISMATCH:-}" = ok ]; then bs_log "WARNING: rustc $mac on the Mac, $box on the box (IGNEUM_TOOLCHAIN_MISMATCH=ok)"
|
||||||
|
else bs_die "rustc $mac on the Mac, $box on the box; re-provision with RUST_TOOLCHAIN=$mac or set IGNEUM_TOOLCHAIN_MISMATCH=ok"; fi
|
||||||
|
else bs_log "rustc $box on both sides"; fi
|
||||||
|
}
|
||||||
|
|
||||||
|
# Where am I? Sets BS_KIND (node = a worktree of the fork under vendor/; repo = a crate of the igneum repo), BS_TOP (the git
|
||||||
|
# top level of the crate's repo), BS_WT_ROOT (the igneum worktree root), BS_WT (its name = the directory on the box),
|
||||||
|
# BS_CRATE (the crate dir, = $PWD), BS_CRATE_REL (relative to BS_WT_ROOT), BS_MIRROR, BS_BRANCH, BS_SHA, BS_REMOTE_WT,
|
||||||
|
# BS_REMOTE_CRATE, and BS_LOCAL_DIRS (every directory of a path dependency, relative to BS_WT_ROOT, from cargo metadata).
|
||||||
|
bs_context() {
|
||||||
|
BS_CRATE="$PWD"
|
||||||
|
[ -f "$BS_CRATE/Cargo.toml" ] || bs_die "no Cargo.toml in $BS_CRATE: run from the crate directory (the fork worktree, igneum-pow, app/igneum-app, proving/igneum-prove)"
|
||||||
|
BS_TOP=$(git -C "$BS_CRATE" rev-parse --show-toplevel 2>/dev/null) || bs_die "$BS_CRATE is not inside a git worktree"
|
||||||
|
case "$BS_TOP" in
|
||||||
|
*/vendor/*)
|
||||||
|
BS_KIND=node; BS_MIRROR=$BS_MIRROR_NODE
|
||||||
|
BS_WT_ROOT=$(cd "$BS_TOP/../.." && pwd)
|
||||||
|
[ -f "$BS_WT_ROOT/igneum-pow/Cargo.toml" ] || bs_die "$BS_TOP looks like a fork worktree but $BS_WT_ROOT/igneum-pow is missing"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
BS_KIND=repo; BS_MIRROR=$BS_MIRROR_REPO; BS_WT_ROOT="$BS_TOP"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
BS_WT=$(basename "$BS_WT_ROOT")
|
||||||
|
BS_CRATE_REL=$(python3 -c 'import os, sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))' "$BS_CRATE" "$BS_WT_ROOT")
|
||||||
|
BS_TOP_REL=$(python3 -c 'import os, sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))' "$BS_TOP" "$BS_WT_ROOT")
|
||||||
|
case "$BS_CRATE_REL" in ..*) bs_die "$BS_CRATE is outside the worktree root $BS_WT_ROOT" ;; esac
|
||||||
|
BS_BRANCH=$(git -C "$BS_TOP" branch --show-current 2>/dev/null || true)
|
||||||
|
BS_SHA=$(git -C "$BS_TOP" rev-parse HEAD)
|
||||||
|
[ -n "$BS_BRANCH" ] || BS_BRANCH="detached-$(git -C "$BS_TOP" rev-parse --short HEAD)"
|
||||||
|
BS_REMOTE_WT="$BS_ROOT_REMOTE/$BS_WT"
|
||||||
|
BS_REMOTE_CRATE="$BS_REMOTE_WT/$BS_CRATE_REL"
|
||||||
|
# every local (path) package of the crate's dependency graph, as directories relative to the worktree root; the ones inside
|
||||||
|
# BS_TOP are covered by the git checkout plus the overlay of BS_TOP itself (node kind) or synced one by one (repo kind)
|
||||||
|
BS_LOCAL_DIRS=$(cd "$BS_CRATE" && "${CARGO_HOME:-$HOME/.cargo}/bin/cargo" metadata --format-version 1 2>/dev/null | python3 -c '
|
||||||
|
import json, os, sys
|
||||||
|
d = json.load(sys.stdin); root = sys.argv[1]; top = sys.argv[2]; kind = sys.argv[3]
|
||||||
|
dirs = set()
|
||||||
|
for p in d["packages"]:
|
||||||
|
if p["source"] is not None: continue
|
||||||
|
m = os.path.dirname(p["manifest_path"])
|
||||||
|
if kind == "node" and (m == top or m.startswith(top + "/")): dirs.add(top); continue
|
||||||
|
dirs.add(m)
|
||||||
|
out = []
|
||||||
|
for m in sorted(dirs):
|
||||||
|
r = os.path.relpath(m, root)
|
||||||
|
if r.startswith(".."): sys.exit("path dependency %s is outside the worktree root %s" % (m, root))
|
||||||
|
out.append(r)
|
||||||
|
print("\n".join(out))' "$BS_WT_ROOT" "$BS_TOP" "$BS_KIND") || bs_die "cargo metadata failed in $BS_CRATE"
|
||||||
|
[ -n "$BS_LOCAL_DIRS" ] || bs_die "cargo metadata listed no local packages in $BS_CRATE"
|
||||||
|
}
|
||||||
|
|
||||||
|
# push the crate repo's HEAD to its bare mirror on the box (fast after the first time), then check the remote tree out at that
|
||||||
|
# commit: a real .git for kaspa-build-info, and the mirror doubles as the CI runner's source later
|
||||||
|
bs_push_and_checkout() {
|
||||||
|
local url="$BS_HOST:$BS_MIRROR" remote_top="$BS_REMOTE_WT/$BS_TOP_REL"
|
||||||
|
[ "$BS_TOP_REL" = . ] && remote_top="$BS_REMOTE_WT"
|
||||||
|
bs_log "push $BS_TOP HEAD $BS_SHA ($BS_BRANCH) -> $url"
|
||||||
|
GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$BS_TOP" push -q --force "$url" "HEAD:refs/heads/$BS_BRANCH" || bs_die "push to the mirror failed"
|
||||||
|
bs_ssh "set -e; mkdir -p '$BS_REMOTE_WT'
|
||||||
|
if [ ! -d '$remote_top/.git' ]; then rm -rf '$remote_top'; git clone -q --no-checkout '$BS_MIRROR' '$remote_top'; fi
|
||||||
|
cd '$remote_top'; git fetch -q origin '+refs/heads/*:refs/remotes/origin/*'; git checkout -q --detach '$BS_SHA'; git reset -q --hard '$BS_SHA'
|
||||||
|
git status --porcelain | head -3" || bs_die "remote checkout at $remote_top failed"
|
||||||
|
BS_REMOTE_TOP="$remote_top"
|
||||||
|
}
|
||||||
|
|
||||||
|
# rsync one directory of the worktree to the same place on the box. By checksum and WITHOUT preserving times, so a file whose
|
||||||
|
# content changed is written with the box's clock and nothing older than the last build slips past cargo's mtime check (the
|
||||||
|
# stale-build class, 4 and 5 October 2026); the files rsync wrote are listed and re-stamped with touch as well, so the rule is
|
||||||
|
# visible here and tools/ci/copied-sources-check.sh sees it. target dirs and .git never travel; --delete keeps the box equal to
|
||||||
|
# the Mac inside the directory (excluded paths are protected).
|
||||||
|
bs_overlay_dir() {
|
||||||
|
local rel="$1" src="$BS_WT_ROOT/$1" dst="$BS_REMOTE_WT/$1" list n
|
||||||
|
[ -d "$src" ] || bs_die "no $src"
|
||||||
|
list=$(mktemp)
|
||||||
|
bs_ssh "mkdir -p '$dst'"
|
||||||
|
bs_rsync -rlpgoD --checksum --delete --out-format='%n' \
|
||||||
|
--exclude '/target' --exclude '/target-*' --exclude '/target/' --exclude 'target-*/' --exclude '.git' --exclude '.DS_Store' --exclude 'node_modules' \
|
||||||
|
"$src/" "$BS_HOST:$dst/" > "$list" || { rm -f "$list"; bs_die "rsync of $rel failed"; }
|
||||||
|
n=$(grep -c . "$list" || true)
|
||||||
|
if [ "$n" -gt 0 ]; then
|
||||||
|
grep -v '/$' "$list" | tr '\n' '\0' | bs_ssh "cd '$dst' && xargs -0 -r touch --no-create" || bs_die "re-stamp of $rel failed"
|
||||||
|
fi
|
||||||
|
bs_log "overlay $rel -> $dst: $n entr$( [ "$n" = 1 ] && echo y || echo ies) written and re-stamped"
|
||||||
|
rm -f "$list"
|
||||||
|
}
|
||||||
|
|
||||||
|
bs_sync_sources() {
|
||||||
|
local d
|
||||||
|
bs_push_and_checkout
|
||||||
|
for d in $BS_LOCAL_DIRS; do bs_overlay_dir "$d"; done
|
||||||
|
}
|
||||||
|
|
||||||
|
bs_sha256() { shasum -a 256 "$1" | awk '{ print $1 }'; }
|
||||||
|
bs_size() { stat -f %z "$1" 2>/dev/null || stat -c %s "$1"; }
|
||||||
|
bs_fmt_secs() { local s=$1; printf '%d min %02d s' $((s / 60)) $((s % 60)); }
|
||||||
|
|
||||||
|
# the remote runner: takes a build slot (flock on /srv/builds/_locks/build-<k>, the box's own slot files, never the Mac's
|
||||||
|
# ~/.config/igneum/build-slots), runs the command in the crate dir with the box's profile, reports time and sccache stats.
|
||||||
|
# $1 remote crate dir, $2 holder label, $3... the command (already a shell string: `cargo build ...`)
|
||||||
|
bs_remote_run() {
|
||||||
|
local dir="$1" label="$2" cmd="$3"
|
||||||
|
bs_ssh "bash -s" <<EOF
|
||||||
|
set -uo pipefail
|
||||||
|
. /etc/profile.d/igneum-build.sh
|
||||||
|
slots=\$(cat "\$IGNEUM_BUILD_SLOTS_DIR/slots" 2>/dev/null || echo 1); [ "\$slots" -ge 1 ] 2>/dev/null || slots=1
|
||||||
|
got=""; t0=\$(date +%s)
|
||||||
|
for k in \$(seq 0 \$((slots - 1))); do
|
||||||
|
exec {fd}>"\$IGNEUM_BUILD_SLOTS_DIR/build-\$k"
|
||||||
|
if flock -n "\$fd"; then got=\$k; break; fi
|
||||||
|
exec {fd}>&-
|
||||||
|
done
|
||||||
|
if [ -z "\$got" ]; then
|
||||||
|
echo "build-remote: all \$slots slot(s) busy, waiting (up to 2 h) for build-0: \$(head -c 160 "\$IGNEUM_BUILD_SLOTS_DIR/build-0" 2>/dev/null)" >&2
|
||||||
|
exec {fd}>"\$IGNEUM_BUILD_SLOTS_DIR/build-0"; flock -w 7200 "\$fd" || { echo "build-remote: gave up waiting for a slot after 2 h" >&2; exit 75; }; got=0
|
||||||
|
fi
|
||||||
|
printf 'pid %s since %sZ waited %s s: %s\n' "\$\$" "\$(date -u +%H:%M:%S)" "\$(( \$(date +%s) - t0 ))" "$label" > "\$IGNEUM_BUILD_SLOTS_DIR/build-\$got"
|
||||||
|
echo "build-remote: holding build-\$got on \$(hostname) (waited \$(( \$(date +%s) - t0 )) s)" >&2
|
||||||
|
cd '$dir' || exit 2
|
||||||
|
sccache --start-server >/dev/null 2>&1 || true
|
||||||
|
before=\$(sccache --show-stats 2>/dev/null | awk '/^Compile requests executed/ { print \$4 }')
|
||||||
|
t1=\$(date +%s)
|
||||||
|
$cmd
|
||||||
|
rc=\$?
|
||||||
|
secs=\$(( \$(date +%s) - t1 ))
|
||||||
|
after=\$(sccache --show-stats 2>/dev/null | awk '/^Compile requests executed/ { print \$4 }')
|
||||||
|
hits=\$(sccache --show-stats 2>/dev/null | awk '/^Cache hits / { print \$3 }')
|
||||||
|
misses=\$(sccache --show-stats 2>/dev/null | awk '/^Cache misses / { print \$3 }')
|
||||||
|
printf 'build-remote: RESULT rc=%s secs=%s compiles=%s sccache_hits_total=%s sccache_misses_total=%s load=%s\n' "\$rc" "\$secs" "\$(( \${after:-0} - \${before:-0} ))" "\${hits:-?}" "\${misses:-?}" "\$(cut -d' ' -f1-3 /proc/loadavg)"
|
||||||
|
: > "\$IGNEUM_BUILD_SLOTS_DIR/build-\$got"
|
||||||
|
exit \$rc
|
||||||
|
EOF
|
||||||
|
}
|
||||||
0
infra/build-server/provision.sh
Normal file → Executable file
0
infra/build-server/provision.sh
Normal file → Executable file
62
infra/build-server/run-from-mac.sh
Executable file
62
infra/build-server/run-from-mac.sh
Executable file
|
|
@ -0,0 +1,62 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Provision igneum-build-1 from this Mac and wire the Mac to it. Idempotent; run it again after any change to provision.sh.
|
||||||
|
#
|
||||||
|
# infra/build-server/run-from-mac.sh <ip> install (if in rescue) or provision, then wire the Mac
|
||||||
|
# infra/build-server/run-from-mac.sh <ip> --wire-only skip provision.sh: only the host file, the remotes and the mirror push
|
||||||
|
# RUST_TOOLCHAIN=1.99.0 SLOTS=2 infra/build-server/run-from-mac.sh <ip> settings pass through to provision.sh
|
||||||
|
#
|
||||||
|
# What it does: 1. ssh root@<ip> with provision.sh on stdin, WORKTREES filled from `git worktree list` of the igneum repo
|
||||||
|
# (one /srv/builds/<name> per agent worktree); 2. writes build@<ip> to ~/.config/igneum/build-server (what tools/build-remote.sh
|
||||||
|
# and tools/cross-remote.sh read); 3. adds the `build` remote to the igneum repo and to the fork vendor/igneum-node and pushes
|
||||||
|
# every branch to the bare mirrors on the box (the fork exists only on this Mac; the mirror is its first copy elsewhere);
|
||||||
|
# 4. prints the ssh line. If the box is still in the rescue system, provision.sh installs Ubuntu and reboots; run this again
|
||||||
|
# when ssh answers (the host key changes: the old entry is removed here).
|
||||||
|
set -euo pipefail
|
||||||
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
REPO="$(cd "$HERE/../.." && pwd)"
|
||||||
|
MAIN_REPO="${IGNEUM_MAIN_REPO:-/Users/joshm/Projects/igneum}" # the shared checkout: the fork lives under its vendor/
|
||||||
|
# shellcheck disable=SC2034 # shared with the scripts that source lib.sh
|
||||||
|
BS_TOOL=run-from-mac
|
||||||
|
# shellcheck source=lib.sh
|
||||||
|
. "$HERE/lib.sh"
|
||||||
|
|
||||||
|
IP="${1:-}"; shift || true
|
||||||
|
[ -n "$IP" ] || bs_die "usage: run-from-mac.sh <ip> [--wire-only]"
|
||||||
|
WIRE_ONLY=0; [ "${1:-}" = --wire-only ] && WIRE_ONLY=1
|
||||||
|
PASS=()
|
||||||
|
for v in MODE RUST_TOOLCHAIN SCCACHE_GB SCCACHE_VERSION NODE_MAJOR SLOTS P2P_PORTS BOX_HOSTNAME SSH_PUBKEY; do
|
||||||
|
[ -n "${!v:-}" ] && PASS+=("$v=$(printf '%q' "${!v}")")
|
||||||
|
done
|
||||||
|
|
||||||
|
ROOT_SSH=(ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 "root@$IP")
|
||||||
|
|
||||||
|
if [ "$WIRE_ONLY" = 0 ]; then
|
||||||
|
WORKTREES=$(git -C "$MAIN_REPO" worktree list --porcelain | awk '/^worktree /{ print $2 }' | xargs -n1 basename | tr '\n' ' ')
|
||||||
|
bs_log "provisioning root@$IP with $(printf '%s\n' "$WORKTREES" | wc -w | tr -d ' ') worktree names${PASS[*]:+ and ${PASS[*]}}"
|
||||||
|
if ! "${ROOT_SSH[@]}" "WORKTREES='$WORKTREES' ${PASS[*]} bash -s" < "$HERE/provision.sh"; then
|
||||||
|
bs_log "provision.sh did not finish (an install-mode run ends with a reboot and a lost connection: wait for ssh, then run this again)"
|
||||||
|
ssh-keygen -R "$IP" >/dev/null 2>&1 || true
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if "${ROOT_SSH[@]}" 'hostname' 2>/dev/null | grep -q '^rescue'; then bs_die "still in the rescue system after provision.sh"; fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p "$(dirname "$BS_HOST_FILE")"
|
||||||
|
printf 'build@%s\n' "$IP" > "$BS_HOST_FILE"
|
||||||
|
bs_log "wrote $BS_HOST_FILE"
|
||||||
|
bs_host
|
||||||
|
bs_ssh 'hostname; nproc' >/dev/null || bs_die "build@$IP does not answer with $BS_KEY"
|
||||||
|
|
||||||
|
wire_remote() { # repo-dir mirror label
|
||||||
|
local dir="$1" mirror="$2" label="$3" url="$BS_HOST:$2" cur
|
||||||
|
cur=$(git -C "$dir" remote get-url build 2>/dev/null || true)
|
||||||
|
if [ -z "$cur" ]; then git -C "$dir" remote add build "$url"; bs_log "$label: remote build = $url"
|
||||||
|
elif [ "$cur" != "$url" ]; then git -C "$dir" remote set-url build "$url"; bs_log "$label: remote build -> $url"
|
||||||
|
else bs_log "$label: remote build ok"; fi
|
||||||
|
GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$dir" push -q --force build --all && bs_log "$label: every branch pushed to $mirror ($(git -C "$dir" branch --list | wc -l | tr -d ' ') branches)" || bs_die "$label: push to $mirror failed"
|
||||||
|
}
|
||||||
|
wire_remote "$MAIN_REPO" "$BS_MIRROR_REPO" "igneum"
|
||||||
|
wire_remote "$MAIN_REPO/vendor/igneum-node" "$BS_MIRROR_NODE" "igneum-node (the fork)"
|
||||||
|
|
||||||
|
bs_log "ssh line: ssh -i $BS_KEY build@$IP"
|
||||||
|
bs_log "next: cd <crate> && $REPO/tools/build-remote.sh (cross: tools/cross-remote.sh)"
|
||||||
98
tools/build-remote.sh
Executable file
98
tools/build-remote.sh
Executable file
|
|
@ -0,0 +1,98 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build on igneum-build-1 instead of this Mac. Run from any crate directory of any worktree on the Mac (a fork worktree under
|
||||||
|
# vendor/, igneum-pow, app/igneum-app, proving/igneum-prove): the sources go to /srv/builds/<worktree>/<same relative path> on
|
||||||
|
# the box (HEAD through the bare mirror, uncommitted changes by rsync, changed files re-stamped with touch in lib.sh's
|
||||||
|
# bs_overlay_dir: the copied-sources rule), the cargo command runs there
|
||||||
|
# with sccache and -j 90 under one of the box's build slots, and the artefacts come back into target-remote/ here.
|
||||||
|
#
|
||||||
|
# tools/build-remote.sh the default command for this crate (below)
|
||||||
|
# tools/build-remote.sh -- build --release -p kaspad --features kaspad/igneum-pow
|
||||||
|
# tools/build-remote.sh -- test --release -p kaspa-consensus-core --lib
|
||||||
|
# tools/build-remote.sh --artefacts "target/release/igneumd" --out /tmp/x -- build --release -p kaspad --features kaspad/igneum-pow
|
||||||
|
# tools/build-remote.sh --jobs 48 -- check
|
||||||
|
# tools/build-remote.sh --target-dir target-exp -- build --release another persistent target dir on the box
|
||||||
|
# tools/build-remote.sh --no-fetch -- clippy --all-targets nothing comes back (tests, check, clippy)
|
||||||
|
#
|
||||||
|
# Defaults by crate: a fork worktree builds `-p kaspad -p igneum-miner --features kaspad/igneum-pow` in release and fetches
|
||||||
|
# target/release/{igneumd,igneum-miner} (what packaging/README-ship.md and infra/cross expect); app/igneum-app builds release
|
||||||
|
# and fetches igneum-app, igneum-ota-sign, igneum-prove-verify; proving/igneum-prove fetches igneum-prove-host and
|
||||||
|
# igneum-prove-export; any other crate builds release and fetches nothing unless --artefacts names files.
|
||||||
|
#
|
||||||
|
# Artefacts land in <crate>/target-remote/<path without the leading target/> (target-remote/release/igneumd), NEVER in
|
||||||
|
# target/: the box builds x86_64 Linux ELF binaries (glibc 2.39, Ubuntu 24.04), which do not run on this Mac. Each one is
|
||||||
|
# reported with size and sha256. For Windows exes use tools/cross-remote.sh.
|
||||||
|
#
|
||||||
|
# Slots: the box has its own slot files (/srv/builds/_locks/build-<k>, count in /srv/builds/_locks/slots, default 1); this
|
||||||
|
# script takes one of THOSE, never the Mac's ~/.config/igneum/build-slots or tools/lock/with-lock.sh, so a remote build does
|
||||||
|
# not hold a Mac slot. A build waits up to 2 h for a remote slot, as with-lock.sh does.
|
||||||
|
#
|
||||||
|
# Needs: ~/.config/igneum/build-server (build@<ip>, written by infra/build-server/run-from-mac.sh), ~/.ssh/igneum_ed25519,
|
||||||
|
# the same rustc version on both sides (refused otherwise; IGNEUM_TOOLCHAIN_MISMATCH=ok overrides).
|
||||||
|
set -euo pipefail
|
||||||
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck disable=SC2034 # shared with the scripts that source lib.sh
|
||||||
|
BS_TOOL=build-remote
|
||||||
|
# shellcheck source=../infra/build-server/lib.sh
|
||||||
|
. "$HERE/../infra/build-server/lib.sh"
|
||||||
|
|
||||||
|
JOBS="${JOBS:-90}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=()
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--jobs) JOBS="$2"; shift 2 ;;
|
||||||
|
--out) OUT="$2"; shift 2 ;;
|
||||||
|
--artefacts) ARTEFACTS="$2"; ARTEFACTS_SET=1; shift 2 ;;
|
||||||
|
--target-dir) TARGET_DIR="$2"; shift 2 ;;
|
||||||
|
--no-fetch) FETCH=0; shift ;;
|
||||||
|
--) shift; CARGO_ARGS=("$@"); break ;;
|
||||||
|
-h|--help) sed -n '2,32p' "$0"; exit 0 ;;
|
||||||
|
*) CARGO_ARGS=("$@"); break ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
[ "${CARGO_ARGS[0]:-}" = cargo ] && CARGO_ARGS=("${CARGO_ARGS[@]:1}")
|
||||||
|
|
||||||
|
bs_host
|
||||||
|
bs_context
|
||||||
|
|
||||||
|
# defaults per crate
|
||||||
|
case "$BS_KIND:$BS_CRATE_REL" in
|
||||||
|
node:*)
|
||||||
|
[ "${#CARGO_ARGS[@]}" -gt 0 ] || CARGO_ARGS=(build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow)
|
||||||
|
[ -n "$ARTEFACTS" ] || ARTEFACTS="$TARGET_DIR/release/igneumd $TARGET_DIR/release/igneum-miner" ;;
|
||||||
|
repo:app/igneum-app)
|
||||||
|
[ "${#CARGO_ARGS[@]}" -gt 0 ] || CARGO_ARGS=(build --release)
|
||||||
|
[ -n "$ARTEFACTS" ] || ARTEFACTS="$TARGET_DIR/release/igneum-app $TARGET_DIR/release/igneum-ota-sign $TARGET_DIR/release/igneum-prove-verify" ;;
|
||||||
|
repo:proving/igneum-prove)
|
||||||
|
[ "${#CARGO_ARGS[@]}" -gt 0 ] || CARGO_ARGS=(build --release)
|
||||||
|
[ -n "$ARTEFACTS" ] || ARTEFACTS="$TARGET_DIR/release/igneum-prove-host $TARGET_DIR/release/igneum-prove-export" ;;
|
||||||
|
*)
|
||||||
|
[ "${#CARGO_ARGS[@]}" -gt 0 ] || CARGO_ARGS=(build --release) ;;
|
||||||
|
esac
|
||||||
|
case "${CARGO_ARGS[0]}" in build) ;; *) [ -n "${ARTEFACTS_SET:-}" ] || { FETCH=0; ARTEFACTS=""; } ;; esac # test, check, clippy: nothing to fetch
|
||||||
|
[ -n "$OUT" ] || OUT="$BS_CRATE/target-remote"
|
||||||
|
|
||||||
|
bs_log "$BS_KIND crate $BS_WT/$BS_CRATE_REL at $BS_SHA ($BS_BRANCH) -> $BS_HOST:$BS_REMOTE_CRATE; cargo ${CARGO_ARGS[*]} -j $JOBS; target dir $TARGET_DIR"
|
||||||
|
bs_toolchain_check
|
||||||
|
t_sync0=$(date +%s)
|
||||||
|
bs_sync_sources
|
||||||
|
bs_log "sources in place after $(( $(date +%s) - t_sync0 )) s"
|
||||||
|
|
||||||
|
cmd="CARGO_TARGET_DIR='$TARGET_DIR' cargo $(printf '%q ' "${CARGO_ARGS[@]}")-j $JOBS 2>&1 | tee -a '$BS_REMOTE_WT/.build-remote.log'; exit \${PIPESTATUS[0]}"
|
||||||
|
label="$BS_WT/$BS_CRATE_REL cargo ${CARGO_ARGS[*]}"
|
||||||
|
t0=$(date +%s)
|
||||||
|
set +e
|
||||||
|
bs_remote_run "$BS_REMOTE_CRATE" "$label" "$cmd" 2>&1 | tee "/tmp/build-remote-$$.log"
|
||||||
|
rc=${PIPESTATUS[0]}
|
||||||
|
set -e
|
||||||
|
secs=$(( $(date +%s) - t0 ))
|
||||||
|
result=$(grep -m1 '^build-remote: RESULT' "/tmp/build-remote-$$.log" || true); rm -f "/tmp/build-remote-$$.log"
|
||||||
|
if [ "$rc" != 0 ]; then bs_die "remote cargo failed (rc $rc) after $(bs_fmt_secs "$secs"); $result"; fi
|
||||||
|
bs_log "remote cargo ${CARGO_ARGS[0]} done in $(bs_fmt_secs "$secs") wall from the Mac; ${result#build-remote: RESULT }"
|
||||||
|
|
||||||
|
if [ "$FETCH" = 1 ] && [ -n "$ARTEFACTS" ]; then
|
||||||
|
mkdir -p "$OUT"
|
||||||
|
for a in $ARTEFACTS; do
|
||||||
|
rel="${a#"$TARGET_DIR"/}"; dest="$OUT/$rel"; mkdir -p "$(dirname "$dest")"
|
||||||
|
bs_rsync -p "$BS_HOST:$BS_REMOTE_CRATE/$a" "$dest" || bs_die "no $a on the box after the build"
|
||||||
|
bs_log "artefact $dest: $(bs_size "$dest") bytes, sha256 $(bs_sha256 "$dest"), $(file -b "$dest" | cut -c1-60)"
|
||||||
|
done
|
||||||
|
fi
|
||||||
107
tools/cross-remote.sh
Executable file
107
tools/cross-remote.sh
Executable file
|
|
@ -0,0 +1,107 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# The Windows cross-build on igneum-build-1: what proto-cuda/windows-node/cross-build.sh does on the Mac (Homebrew mingw-w64)
|
||||||
|
# and what the PC build job does in WSL2 (app/igneum-app/src/jobbuild.rs: Ubuntu 24.04 mingw-w64 posix threads, static
|
||||||
|
# libgcc), run on the box with sccache and -j 90 (sources synced and re-stamped with touch by lib.sh's bs_overlay_dir, the
|
||||||
|
# copied-sources rule). Run from a fork worktree (igneumd.exe, igneum-miner.exe) or from
|
||||||
|
# app/igneum-app (igneum-app.exe, igneum-ota-sign.exe, igneum-prove-verify.exe).
|
||||||
|
#
|
||||||
|
# tools/cross-remote.sh the default command for this crate
|
||||||
|
# tools/cross-remote.sh --compare target-integration/x86_64-pc-windows-gnu/release sha256 against the Mac's exes
|
||||||
|
# tools/cross-remote.sh -- build --release -p kaspad --features igneum-pow --target x86_64-pc-windows-gnu
|
||||||
|
# tools/cross-remote.sh --jobs 48 --target-dir target-win
|
||||||
|
#
|
||||||
|
# Output: <crate>/target-remote/x86_64-pc-windows-gnu/release/<name>.exe, one line each with size, sha256 and the DLL import
|
||||||
|
# list (x86_64-w64-mingw32-objdump -p on the box, as the Mac script prints it). With --compare <dir>, the Mac's exe of the same
|
||||||
|
# name is hashed too and the line says identical or differs.
|
||||||
|
#
|
||||||
|
# Byte identity (6 October 2026): the same rustc (1.99.0 both sides, refused otherwise) and the same flags give the same Rust
|
||||||
|
# code, but two things still differ between the Mac's exe and the box's: the C and C++ objects (rocksdb, snappy, zstd, lz4,
|
||||||
|
# secp256k1, mimalloc) come from Homebrew's mingw gcc on the Mac and Ubuntu's gcc 13 here, and source paths embedded by
|
||||||
|
# rustc (panic locations, /Users/joshm/... against /srv/builds/...) differ unless both sides pass --remap-path-prefix, which
|
||||||
|
# the Mac script does not. So: identical bytes build to build ON THE BOX (sccache does not change output), a different hash
|
||||||
|
# from the Mac's exe is expected, and the number that matters is the DLL list (must be none since the fork's database/build.rs
|
||||||
|
# links libstdc++ statically) and that the exe runs on the PC. The same holds for the PC-built exes today.
|
||||||
|
#
|
||||||
|
# Environment of the build (the PC recipe; the Mac's -static-libstdc++ added, harmless since housekeeping made the C++ runtime
|
||||||
|
# static in the fork): CC/CXX/AR_x86_64_pc_windows_gnu = the posix mingw compilers, the linker the same gcc, RUSTFLAGS
|
||||||
|
# -static -static-libgcc -static-libstdc++, LIBCLANG_PATH = Ubuntu's llvm lib dir (bindgen for librocksdb-sys),
|
||||||
|
# BINDGEN_EXTRA_CLANG_ARGS pointed at /usr/x86_64-w64-mingw32, IGNEUM_WINDRES for the app's .rc.
|
||||||
|
set -euo pipefail
|
||||||
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck disable=SC2034 # shared with lib.sh
|
||||||
|
BS_TOOL=cross-remote
|
||||||
|
# shellcheck source=../infra/build-server/lib.sh
|
||||||
|
. "$HERE/../infra/build-server/lib.sh"
|
||||||
|
|
||||||
|
TARGET=x86_64-pc-windows-gnu
|
||||||
|
JOBS="${JOBS:-90}"; OUT=""; COMPARE=""; TARGET_DIR="target"; CARGO_ARGS=()
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
case "$1" in
|
||||||
|
--jobs) JOBS="$2"; shift 2 ;;
|
||||||
|
--out) OUT="$2"; shift 2 ;;
|
||||||
|
--compare) COMPARE="$2"; shift 2 ;;
|
||||||
|
--target-dir) TARGET_DIR="$2"; shift 2 ;;
|
||||||
|
--) shift; CARGO_ARGS=("$@"); break ;;
|
||||||
|
-h|--help) sed -n '2,30p' "$0"; exit 0 ;;
|
||||||
|
*) CARGO_ARGS=("$@"); break ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
[ "${CARGO_ARGS[0]:-}" = cargo ] && CARGO_ARGS=("${CARGO_ARGS[@]:1}")
|
||||||
|
|
||||||
|
bs_host
|
||||||
|
bs_context
|
||||||
|
case "$BS_KIND:$BS_CRATE_REL" in
|
||||||
|
node:*)
|
||||||
|
[ "${#CARGO_ARGS[@]}" -gt 0 ] || CARGO_ARGS=(build --release -p kaspad -p igneum-miner --features igneum-pow --target "$TARGET")
|
||||||
|
EXES="igneumd.exe igneum-miner.exe" ;;
|
||||||
|
repo:app/igneum-app)
|
||||||
|
[ "${#CARGO_ARGS[@]}" -gt 0 ] || CARGO_ARGS=(build --release --target "$TARGET")
|
||||||
|
EXES="igneum-app.exe igneum-ota-sign.exe igneum-prove-verify.exe" ;;
|
||||||
|
*) bs_die "cross-remote builds the fork (run from a vendor/igneum-node* worktree) or app/igneum-app, not $BS_CRATE_REL" ;;
|
||||||
|
esac
|
||||||
|
[ -n "$OUT" ] || OUT="$BS_CRATE/target-remote"
|
||||||
|
[ -z "$COMPARE" ] && [ -d "$BS_CRATE/target-integration/$TARGET/release" ] && COMPARE="$BS_CRATE/target-integration/$TARGET/release"
|
||||||
|
|
||||||
|
bs_log "$BS_KIND crate $BS_WT/$BS_CRATE_REL at $BS_SHA ($BS_BRANCH) -> $BS_HOST:$BS_REMOTE_CRATE; cargo ${CARGO_ARGS[*]} -j $JOBS; target dir $TARGET_DIR"
|
||||||
|
bs_toolchain_check
|
||||||
|
t_sync0=$(date +%s)
|
||||||
|
bs_sync_sources
|
||||||
|
bs_log "sources in place after $(( $(date +%s) - t_sync0 )) s"
|
||||||
|
|
||||||
|
# the build environment, as one shell string for the remote runner (every value is a literal; nothing from the Mac's env)
|
||||||
|
env_block='LLVM_LIB=$(ls -d /usr/lib/llvm-*/lib 2>/dev/null | sort -V | tail -1); [ -n "$LLVM_LIB" ] || { echo "no /usr/lib/llvm-*/lib on the box (apt clang libclang-dev)"; exit 2; }
|
||||||
|
export CC_x86_64_pc_windows_gnu=x86_64-w64-mingw32-gcc-posix CXX_x86_64_pc_windows_gnu=x86_64-w64-mingw32-g++-posix AR_x86_64_pc_windows_gnu=x86_64-w64-mingw32-ar
|
||||||
|
export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER=x86_64-w64-mingw32-gcc-posix
|
||||||
|
export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS="-C link-arg=-static -C link-arg=-static-libgcc -C link-arg=-static-libstdc++"
|
||||||
|
export IGNEUM_WINDRES=x86_64-w64-mingw32-windres LIBCLANG_PATH="$LLVM_LIB"
|
||||||
|
export BINDGEN_EXTRA_CLANG_ARGS_x86_64_pc_windows_gnu="--target=x86_64-w64-mingw32 --sysroot=/usr/x86_64-w64-mingw32 -I/usr/x86_64-w64-mingw32/include"
|
||||||
|
echo "cross-remote: $(x86_64-w64-mingw32-gcc-posix --version | head -1); libclang $LLVM_LIB"'
|
||||||
|
cmd="$env_block
|
||||||
|
CARGO_TARGET_DIR='$TARGET_DIR' cargo $(printf '%q ' "${CARGO_ARGS[@]}")-j $JOBS 2>&1 | tee -a '$BS_REMOTE_WT/.cross-remote.log'; rc=\${PIPESTATUS[0]}
|
||||||
|
for exe in $EXES; do f='$TARGET_DIR/$TARGET/release/'\$exe; [ -f \"\$f\" ] && echo \"cross-remote: DLLS \$exe: \$(x86_64-w64-mingw32-objdump -p \"\$f\" | awk '/DLL Name/ { print \$3 }' | sort -u | tr '\n' ' ')\"; done
|
||||||
|
exit \$rc"
|
||||||
|
label="$BS_WT/$BS_CRATE_REL cross $TARGET"
|
||||||
|
t0=$(date +%s)
|
||||||
|
set +e
|
||||||
|
bs_remote_run "$BS_REMOTE_CRATE" "$label" "$cmd" 2>&1 | tee "/tmp/cross-remote-$$.log"
|
||||||
|
rc=${PIPESTATUS[0]}
|
||||||
|
set -e
|
||||||
|
secs=$(( $(date +%s) - t0 ))
|
||||||
|
result=$(grep -m1 '^build-remote: RESULT' "/tmp/cross-remote-$$.log" || true)
|
||||||
|
dlls=$(grep '^cross-remote: DLLS' "/tmp/cross-remote-$$.log" || true); rm -f "/tmp/cross-remote-$$.log"
|
||||||
|
if [ "$rc" != 0 ]; then bs_die "remote cross-build failed (rc $rc) after $(bs_fmt_secs "$secs"); $result"; fi
|
||||||
|
bs_log "remote cross-build done in $(bs_fmt_secs "$secs") wall from the Mac; ${result#build-remote: RESULT }"
|
||||||
|
|
||||||
|
mkdir -p "$OUT/$TARGET/release"
|
||||||
|
for exe in $EXES; do
|
||||||
|
dest="$OUT/$TARGET/release/$exe"
|
||||||
|
bs_rsync -p "$BS_HOST:$BS_REMOTE_CRATE/$TARGET_DIR/$TARGET/release/$exe" "$dest" || bs_die "no $exe on the box after the build"
|
||||||
|
sum=$(bs_sha256 "$dest"); line="$exe: $(bs_size "$dest") bytes, sha256 $sum"
|
||||||
|
d=$(printf '%s\n' "$dlls" | grep "DLLS $exe:" | sed 's/.*DLLS [^:]*: *//'); line="$line, DLLs: ${d:-none}"
|
||||||
|
if [ -n "$COMPARE" ] && [ -f "$COMPARE/$exe" ]; then
|
||||||
|
msum=$(bs_sha256 "$COMPARE/$exe")
|
||||||
|
if [ "$msum" = "$sum" ]; then line="$line; IDENTICAL to $COMPARE/$exe"; else line="$line; differs from $COMPARE/$exe ($(bs_size "$COMPARE/$exe") bytes, sha256 ${msum:0:16}...; expected, see the header)"; fi
|
||||||
|
fi
|
||||||
|
bs_log "$line"
|
||||||
|
done
|
||||||
|
bs_log "exes in $OUT/$TARGET/release"
|
||||||
Loading…
Reference in a new issue