97255a4e8f
The dashboard lane, 7 October 2026 10:39Z: both slots of build-1 flock-held and EMPTY while two suites ran. Cause: a run from a worktree without last night's append-mode fix opens a busy sibling's slot file with > on every probe. The holder now keeps its own line: a keeper re-writes it within BR_KEEP_S (20 s) whenever the file is empty, until release; remote-run.sh --self-test-keeper (in the gate) truncates a held line and sees it return, and sees nothing written after release; live on build-1 at 11:19Z (the line came back in 25 s). The first version deadlocked the runner's bare wait with the keeper (build-2's first run hung 15 min after its test passed): the keeper stops before the wait. The two running suites' -j 90 came from explicit --jobs 90: a bounded class now clamps it to its cap with a log line (pass --priority gate for the full set). run-from-mac.sh --box N: the host file was suffixed twice (build-server-2-2) and every box's mirror would have shared one remote name; one remote per box (build-N). build-2's first green run: a suite at nice 10 on 32 cores, jobs 32, 986 s cold. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
81 lines
6 KiB
Bash
Executable file
81 lines
6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Provision igneum-build-1 from this Mac and wire the Mac to it. Idempotent; run it again after any change to provision.sh.
|
|
#
|
|
# infra/build-server/run-from-mac.sh <ip> install (if in rescue) or provision, then wire the Mac
|
|
# infra/build-server/run-from-mac.sh <ip> --wire-only skip provision.sh: only the host file, the remotes and the mirror push
|
|
# RUST_TOOLCHAIN=1.99.0 SLOTS=2 infra/build-server/run-from-mac.sh <ip> settings pass through to provision.sh
|
|
#
|
|
# What it does: 1. ssh root@<ip> with provision.sh on stdin, WORKTREES filled from `git worktree list` of the igneum repo
|
|
# (one /srv/builds/<name> per agent worktree); 2. writes build@<ip> to ~/.config/igneum/build-server (what tools/build-remote.sh
|
|
# and tools/cross-remote.sh read); 3. adds the `build` remote to the igneum repo and to the fork vendor/igneum-node and pushes
|
|
# every branch to the bare mirrors on the box (the fork exists only on this Mac; the mirror is its first copy elsewhere);
|
|
# 4. prints the ssh line. If the box is still in the rescue system, provision.sh installs Ubuntu and reboots; run this again
|
|
# when ssh answers (the host key changes: the old entry is removed here).
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO="$(cd "$HERE/../.." && pwd)"
|
|
MAIN_REPO="${IGNEUM_MAIN_REPO:-/Users/joshm/Projects/igneum}" # the shared checkout: the fork lives under its vendor/
|
|
# shellcheck disable=SC2034 # shared with the scripts that source lib.sh
|
|
BS_TOOL=run-from-mac
|
|
# shellcheck source=lib.sh
|
|
. "$HERE/lib.sh"
|
|
|
|
BOX=1; while [ "${1:-}" = --box ]; do BOX="$2"; shift 2; done # --box N: igneum-build-N, host file build-server-N (7 Oct 2026)
|
|
IP="${1:-}"; shift || true
|
|
[ -n "$IP" ] || bs_die "usage: run-from-mac.sh [--box N] <ip> [--wire-only]"
|
|
WIRE_ONLY=0; [ "${1:-}" = --wire-only ] && WIRE_ONLY=1
|
|
# box N is igneum-build-N with its own dashboard feed name build-N.igneum.network (the dashboard lane's collector reads one server
|
|
# section per box; main adds the A record with the IP)
|
|
[ "$BOX" = 1 ] || { BOX_HOSTNAME="${BOX_HOSTNAME:-igneum-build-$BOX}"; WORKERS_HOST="${WORKERS_HOST:-build-$BOX.igneum.network}"; export BOX_HOSTNAME WORKERS_HOST; }
|
|
export BS_BOX="$BOX" # lib.sh's bs_host derives the host file from BS_BOX (box 1: build-server; box N: build-server-N); never set BS_HOST_FILE here (7 Oct 2026: a second suffix, build-server-2-2)
|
|
HOST_FILE=$(bs_box_file "$BOX")
|
|
REMOTE="build"; [ "$BOX" = 1 ] || REMOTE="build-$BOX" # one git remote per box in the two repositories
|
|
# the toolchain pin travels from rust-toolchain.toml unless RUST_TOOLCHAIN is set by hand (one file pins every side, 7 Oct 2026)
|
|
[ -n "${RUST_TOOLCHAIN:-}" ] || RUST_TOOLCHAIN=$(sed -n 's/^channel *= *"\([^"]*\)".*/\1/p' "$REPO/rust-toolchain.toml" 2>/dev/null | head -1); export RUST_TOOLCHAIN
|
|
PASS="" # a string, not an array: bash 3.2 (the Mac) treats an empty array as unbound under set -u
|
|
for v in MODE RUST_TOOLCHAIN SCCACHE_GB SCCACHE_VERSION NODE_MAJOR SLOTS P2P_PORTS BOX_HOSTNAME WORKERS_HOST SSH_PUBKEY; do
|
|
[ -n "${!v:-}" ] && PASS="$PASS $v=$(printf '%q' "${!v}")"
|
|
done
|
|
|
|
ROOT_SSH=(ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 "root@$IP")
|
|
|
|
if [ "$WIRE_ONLY" = 0 ]; then
|
|
WORKTREES=$(git -C "$MAIN_REPO" worktree list --porcelain | awk '/^worktree /{ print $2 }' | xargs -n1 basename | tr '\n' ' ')
|
|
bs_log "provisioning root@$IP with $(printf '%s\n' "$WORKTREES" | wc -w | tr -d ' ') worktree names${PASS:+ and$PASS}"
|
|
if ! "${ROOT_SSH[@]}" "WORKTREES='$WORKTREES'$PASS bash -s" < "$HERE/provision.sh"; then
|
|
bs_log "provision.sh did not finish (an install-mode run ends with a reboot and a lost connection: wait for ssh, then run this again)"
|
|
ssh-keygen -R "$IP" >/dev/null 2>&1 || true
|
|
exit 1
|
|
fi
|
|
if "${ROOT_SSH[@]}" 'hostname' 2>/dev/null | grep -q '^rescue'; then bs_die "still in the rescue system after provision.sh"; fi
|
|
fi
|
|
|
|
mkdir -p "$(dirname "$HOST_FILE")"
|
|
printf 'build@%s\n' "$IP" > "$HOST_FILE"
|
|
bs_log "wrote $HOST_FILE"
|
|
bs_host
|
|
bs_ssh 'hostname; nproc' >/dev/null || bs_die "build@$IP does not answer with $BS_KEY"
|
|
|
|
wire_remote() { # repo-dir mirror label
|
|
local dir="$1" mirror="$2" label="$3" url="$BS_HOST:$2" cur
|
|
cur=$(git -C "$dir" remote get-url "$REMOTE" 2>/dev/null || true)
|
|
if [ -z "$cur" ]; then git -C "$dir" remote add "$REMOTE" "$url"; bs_log "$label: remote $REMOTE = $url"
|
|
elif [ "$cur" != "$url" ]; then git -C "$dir" remote set-url "$REMOTE" "$url"; bs_log "$label: remote $REMOTE -> $url"
|
|
else bs_log "$label: remote $REMOTE ok"; fi
|
|
GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$dir" push -q --force "$REMOTE" --all && bs_log "$label: every branch pushed to $mirror ($(git -C "$dir" branch --list | wc -l | tr -d ' ') branches)" || bs_die "$label: push to $mirror failed"
|
|
}
|
|
wire_remote "$MAIN_REPO" "$BS_MIRROR_REPO" "igneum"
|
|
wire_remote "$MAIN_REPO/vendor/igneum-node" "$BS_MIRROR_NODE" "igneum-node (the fork)"
|
|
# every repository under vendor/ that a Cargo.toml of the repo reaches by path (6 October 2026: proving/igneum-prove ->
|
|
# vendor/igneum-node-exec, a worktree of the fork, so already on /srv/igneum-node.git; a repository of its own gets its own
|
|
# mirror /srv/<name>.git here, and lib.sh checks it out whole on the box)
|
|
grep -rhoE 'path *= *"\.\./[^"]*vendor/[^/"]+' --include=Cargo.toml "$MAIN_REPO/app" "$MAIN_REPO/proving" "$MAIN_REPO/igneum-pow" "$MAIN_REPO/igneum-census" "$MAIN_REPO/proto-vdf" 2>/dev/null \
|
|
| sed -E 's|.*vendor/||' | sort -u | while read -r name; do
|
|
dir="$MAIN_REPO/vendor/$name"; [ -d "$dir" ] || { bs_log "vendor/$name is reached by a path dependency but missing on this Mac"; continue; }
|
|
mirror=$(bs_mirror_for "$dir")
|
|
if [ "$mirror" = "$BS_MIRROR_NODE" ]; then bs_log "vendor/$name: a worktree of the fork ($(git -C "$dir" branch --show-current)), on $BS_MIRROR_NODE already"
|
|
else bs_ssh "[ -d '$mirror' ] || git init -q --bare -b master '$mirror'"; wire_remote "$dir" "$mirror" "vendor/$name"; fi
|
|
done
|
|
|
|
bs_log "ssh line: ssh -i $BS_KEY build@$IP"
|
|
bs_log "next: cd <crate> && $REPO/tools/build-remote.sh (cross: tools/cross-remote.sh)"
|