#!/usr/bin/env bash
# Provision igneum-build-1, the Hetzner dedicated build server (AX162-1-LTD: EPYC 9454P 48 cores / 96 threads, 128 GB,
# 2x 3.84 TB NVMe, Falkenstein; ordered 6 October 2026). Idempotent: every step checks before it changes anything and
# says "ok" (nothing to do) or "changed". Run it over ssh as root; nothing here reads a secret.
#
#   infra/build-server/run-from-mac.sh <ip>            the usual way (ships this file, fills WORKTREES, writes the host file)
#   ssh root@<ip> 'bash -s' < infra/build-server/provision.sh                      the bare way
#   ssh root@<ip> 'MODE=install bash -s' < infra/build-server/provision.sh         force the rescue-system path
#
# Two modes, chosen by MODE (auto, install, provision; default auto):
#   install    the box booted into Hetzner's rescue system (installimage present, hostname rescue*): run installimage in
#              batch mode for Ubuntu 24.04 with software RAID 1 over the two NVMe drives, no swap, the rescue system's
#              authorized_keys taken over, the image signature checked, then reboot. Run the script again after the reboot.
#              Ran on igneum-build-1 on 6 October 2026 at 17:16 to 17:20 UTC (16 steps, no prompt).
#   provision  a running Ubuntu 24.04: user `build` with root's key, the compiler and cross toolchains, rustup pinned to
#              RUST_TOOLCHAIN with the x86_64-pc-windows-gnu target, sccache with a 100 GB disk cache, Node 22, git, tmux,
#              a swap-free tuned sysctl, the two bare mirrors (/srv/igneum.git, /srv/igneum-node.git), /srv/builds with one
#              directory per agent worktree, sshd key-only, ufw with 22 and the seed p2p ports.
#
# Settings (environment, all optional):
#   RUST_TOOLCHAIN   1.99.0   the channel of rust-toolchain.toml at the repo root (run-from-mac.sh passes it; one file pins
#                             the Mac, the box, the PCs and CI since 7 October 2026). tools/build-remote.sh refuses a build
#                             when the pin, the Mac's rustc or the box's rustc differ.
#   SCCACHE_GB       100      the local disk cache at /srv/sccache
#   SCCACHE_VERSION  (unset)  a `cargo install sccache --version` pin; unset = the newest on crates.io
#   NODE_MAJOR       22
#   WORKTREES        ""       space-separated agent worktree names, one /srv/builds/<name> each (run-from-mac.sh fills it from
#                             `git worktree list` on the Mac; tools/build-remote.sh creates a missing one on first use)
#   SLOTS            2        remote build slots (tools/build-remote.sh takes one; remote-run.sh sets CARGO_BUILD_JOBS 90 when it holds
#                             the only taken slot and 45 when both are held; a measurement takes the `measure` file and excludes builds)
#   P2P_PORTS        "26611 26811 26621"  TCP ports ufw opens beside 22: the devnet seed's p2p (infra/seed-nodes/config.sh devnet
#                             P2P_PORT=26611) and the testnet seed's (26811). A suffixed devnet (Devnet 2, the fleet's staging
#                             chain) listens on the same 26611 (infra/cloud-devnet/config.sh P2P_PORT=26611); the Devnet 2 seed that
#                             runs on build-1 as a bare process listens on 26621. RPC ports
#                             (26610, 28610, 26790 and the 268xx set) stay on loopback as on every seed, so they are not opened.
#   BOX_HOSTNAME     igneum-build-1
#   WORKERS_HOST     build.igneum.network   Caddy serves /srv/workers/{workers,headline}.json there (read-only, all else 404)
#   SSH_PUBKEY       (unset)  a public key line for the build user when root has no authorized_keys (installimage installs it)
#
# Mirrors: the Mac pushes to them (never a clone from GitHub; the fork vendor/igneum-node exists only on the Mac):
#   git -C /Users/joshm/Projects/igneum remote add build build@<ip>:/srv/igneum.git
#   git -C /Users/joshm/Projects/igneum/vendor/igneum-node remote add build build@<ip>:/srv/igneum-node.git
#   git push build --all           (tools/build-remote.sh pushes the branch it builds before every build)
#
# What this script does NOT do: install zig or cargo-zigbuild (the Mac's glibc 2.36 Linux cross-build, infra/cross/build-linux.sh,
# stays on the Mac until the box is proven; a native build here links glibc 2.39, which Debian 13 seeds accept and HiveOS
# does not), start any node, or copy a secret. The installimage flags and the image name were read from the live rescue system
# on 6 October 2026 (`installimage -h`, /root/.oldroot/nfs/images); the script still reads the image list instead of hard-coding a name.
set -euo pipefail

MODE="${MODE:-auto}"
RUST_TOOLCHAIN="${RUST_TOOLCHAIN:-1.99.0}"
SCCACHE_GB="${SCCACHE_GB:-100}"
SCCACHE_VERSION="${SCCACHE_VERSION:-}"
NODE_MAJOR="${NODE_MAJOR:-22}"
WORKTREES="${WORKTREES:-}"
SLOTS_GIVEN="${SLOTS:-}"; SLOTS="${SLOTS:-2}"   # 2 since main's ruling of 6 October 2026 (20:3x UK); remote-run.sh gives 90 jobs alone, 45 beside another
P2P_PORTS="${P2P_PORTS:-26611 26811 26621 26631 26651}"   # 26631 the Devnet 3 seed, 26651 the hands' Devnet 3 node1 (0.3.22, placeholders until the object lands)   # 26621: the Devnet 2 seed on build-1 (blocked by ufw until 7 Oct 2026 16:40 BST; the fleet lane found it closed from outside)
BOX_HOSTNAME="${BOX_HOSTNAME:-igneum-build-1}"
case "$BOX_HOSTNAME" in *-2|*-4) [ -n "$SLOTS_GIVEN" ] || SLOTS=3 ;; esac   # build-2 and build-4 (AX162-1, 96 threads): three slots (main, 7 Oct 2026; the bounded runs take one 32-core band each)
WORKERS_HOST="${WORKERS_HOST:-build.igneum.network}"   # the dashboard feed's HTTPS name (A record in deSEC, 6 Oct 2026)
SSH_PUBKEY="${SSH_PUBKEY:-}"
BUILD_USER=build
ROLE="${ROLE:-box}"                                       # box (the default) or sweep: a rented cloud sweep worker, provisioned minus the runner (7 Oct 2026)
BUILD_HOME=/home/$BUILD_USER
# the GitHub Actions runner (step_runner, 6 October 2026 evening): a dedicated user, never build and never root
RUNNER_USER=runner
RUNNER_HOME=/home/$RUNNER_USER
RUNNER_DIR=/opt/actions-runner
RUNNER_VERSION="${RUNNER_VERSION:-2.338.0}"                 # github.com/actions/runner releases, read 6 October 2026
RUNNER_SHA256="${RUNNER_SHA256:-af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32}"   # the release note's linux-x64 line
RUNNER_REPO_URL="${RUNNER_REPO_URL:-https://github.com/igneum-network/igneum}"
RUNNER_NAME="${RUNNER_NAME:-$BOX_HOSTNAME}"
RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1,ci-red}"     # added to the defaults self-hosted, linux, x64. igneum-build-1 is the POOL label
                                                            # (every box that takes pow and sims carries it); ci-red marks the one box that
                                                            # holds the red watcher's record file and poster. A second box: BOX_HOSTNAME=igneum-build-2
                                                            # RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 (register.sh --host)
RUNNER_CPUS="${RUNNER_CPUS:-}"                              # AllowedCPUs for the runner's service when set (a second box is bounded like a suite: 32 cores, nice 10)
case "$BOX_HOSTNAME" in *-2|*-3|*-4) [ -n "$RUNNER_CPUS" ] || RUNNER_CPUS="0-31" ;; esac   # build-2, build-3 and build-4 join the pool (label igneum-build-1) bounded to 32 cores at Nice 10 (main, 7 Oct 2026)
RUNNER_JOBS="${RUNNER_JOBS:-48}"                            # cargo jobs for a CI job: half the box, the agents' builds keep the rest
RUNNER_TOKEN="${RUNNER_TOKEN:-}"                            # a registration token (1 h), from infra/build-server/runner/register.sh over stdin; never logged
RUNNER_SCCACHE_PORT="${RUNNER_SCCACHE_PORT:-4227}"          # the runner's own sccache server; 4226 is the build user's

log() { printf '%s provision: %s\n' "$(date -u +%H:%M:%S)" "$*"; }
die() { log "ERROR: $*" >&2; exit 1; }
changed() { log "$1: changed${2:+ ($2)}"; }
ok() { log "$1: ok${2:+ ($2)}"; }
as_build() { su - "$BUILD_USER" -c "$*"; }

[ "$(id -u)" = 0 ] || die "run as root"

# ----------------------------------------------------------------------------------------------------------------------
# install mode: the rescue system
# ----------------------------------------------------------------------------------------------------------------------
INSTALLIMAGE=/root/.oldroot/nfs/install/installimage   # not on PATH in a non-interactive ssh shell (read 6 Oct 2026)

in_rescue() {
  [ -x "$INSTALLIMAGE" ] || return 1
  case "$(hostname)" in rescue*) return 0 ;; esac
  [ -d /root/.oldroot/nfs/images ]
}

do_install() {
  local images drives image parts
  images=/root/.oldroot/nfs/images
  [ -d "$images" ] || die "no image directory at $images: not the Hetzner rescue system"
  # the Ubuntu 24.04 (noble) amd64 base image the rescue system offers (read on 6 October 2026: Ubuntu-2404-noble-amd64-base.tar.zst
  # with a detached .sig; read, not hard-coded, because Hetzner refreshes the names)
  image=$(find "$images" -maxdepth 1 -type f -iregex '.*/ubuntu-2404.*amd64.*\.tar\.\(zst\|gz\|xz\)' -printf '%f\n' | sort | tail -1 || true)
  [ -n "$image" ] || die "no Ubuntu 24.04 amd64 image under $images: $(find "$images" -maxdepth 1 -type f -printf '%f ' )"
  mapfile -t drives < <(lsblk -dn -o NAME,TYPE | awk '$2 == "disk" && $1 ~ /^nvme/ { print $1 }' | sort)
  [ "${#drives[@]}" = 2 ] || die "expected exactly two NVMe drives for RAID 1, found ${#drives[@]}: ${drives[*]:-none}"
  [ -s /root/.ssh/authorized_keys ] || die "/root/.ssh/authorized_keys is empty in the rescue system; -t yes would carry nothing into the image"
  [ -d /sys/firmware/efi ] || log "WARNING: no /sys/firmware/efi, the box booted in BIOS mode; the esp partition is harmless but grub goes to the MBR"
  # no swap partition: 128 GB of RAM and a swap-free sysctl (the provision mode checks no swap is active)
  parts="/boot/efi:esp:512M,/boot:ext4:1G,/:ext4:all"
  log "installimage: image $image, drives ${drives[*]} as software RAID 1, partitions $parts, hostname $BOX_HOSTNAME, rescue ssh keys taken over (-t yes), image signature checked (-g)"
  log "this WIPES ${drives[*]}"
  # flag form, read from `installimage -h` on 6 October 2026: -a batch, -n hostname, -r raid, -l level, -i image, -g verify
  # the detached signature, -p partitions mount:fs:size, -d drives, -t yes take over the rescue system's ssh keys (root's
  # authorized_keys), -G yes new host keys. The -c config form forbids every other flag, so the keys could not travel with it.
  TERM="${TERM:-xterm}" "$INSTALLIMAGE" -a -n "$BOX_HOSTNAME" -r yes -l 1 -i "$images/$image" -g -p "$parts" -d "$(IFS=,; echo "${drives[*]}")" -t yes -G yes
  log "installimage finished; rebooting into Ubuntu. Run this script again (MODE=provision or auto) once ssh answers (the host key is new: -G yes)."
  sync; reboot
}

# ----------------------------------------------------------------------------------------------------------------------
# provision mode: the installed Ubuntu
# ----------------------------------------------------------------------------------------------------------------------
step_hostname() {
  if [ "$(hostnamectl --static 2>/dev/null || hostname)" = "$BOX_HOSTNAME" ]; then ok hostname "$BOX_HOSTNAME"; return; fi
  hostnamectl set-hostname "$BOX_HOSTNAME"; grep -q "$BOX_HOSTNAME" /etc/hosts || printf '127.0.1.1 %s\n' "$BOX_HOSTNAME" >> /etc/hosts
  changed hostname "$BOX_HOSTNAME"
}

step_os_check() {
  . /etc/os-release
  [ "${ID:-}" = ubuntu ] && [ "${VERSION_ID:-}" = 24.04 ] || die "this is ${PRETTY_NAME:-unknown}, not Ubuntu 24.04"
  ok os "$PRETTY_NAME, $(nproc) threads, $(awk '/MemTotal/ { printf "%d GB", $2 / 1024 / 1024 }' /proc/meminfo)"
}

# libprotobuf-dev: the well-known .proto files (google/protobuf/empty.proto) that sp1-prover-types's build script imports
# (6 October 2026, the first proving build on the box: "protoc failed: google/protobuf/empty.proto: File not found").
# the proven Ubuntu 24.04 set: the PC build job's APT lists (app/igneum-app/src/jobbuild.rs: mingw-w64 posix threads so
# libstdc++ has std::thread for rocksdb, clang for librocksdb-sys's bindgen, protoc for the node's proto crates) plus the
# task's list (build-essential, clang, lld, pkg-config, libssl-dev, cmake, git, tmux) and what the scripts here call
APT_PACKAGES=(
  build-essential clang lld llvm libclang-dev pkg-config libssl-dev cmake protobuf-compiler libprotobuf-dev
  gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-x86-64-dev mingw-w64-tools
  git tmux curl ca-certificates xz-utils zstd unzip rsync jq python3 ufw htop file caddy docker.io
  # the GitHub Actions runner's .NET runtime needs libicu (bin/installdependencies.sh would install it); the two Python
  # simulators (sim/finality_v2.py, sim/difficulty/sim.py) need numpy, which ci.yml pip-installs on GitHub's runners
  libicu74 python3-numpy
  # innoextract: tools/repro reads the shipped igneumd.exe and igneum-miner.exe out of the public Inno Setup installer
  innoextract
  # headless Chromium (the dashboard lane's site captures): the 16 system libraries it dlopens, found missing on both boxes
  # on 7 October 2026 (installed by hand at 14:5x UK; step_headless_check launches the shell once and prints its version)
  libatk1.0-0t64 libatk-bridge2.0-0t64 libatspi2.0-0t64 libcairo2 libcups2t64 libgbm1 libpango-1.0-0 libx11-6 libxcb1
  libxcomposite1 libxdamage1 libxext6 libxfixes3 libxrandr2 libasound2t64 libxkbcommon0 fonts-liberation
)
step_apt() {
  local need=() p
  for p in "${APT_PACKAGES[@]}"; do dpkg -s "$p" >/dev/null 2>&1 || need+=("$p"); done
  if [ "${#need[@]}" = 0 ]; then ok apt "${#APT_PACKAGES[@]} packages present"; return; fi
  export DEBIAN_FRONTEND=noninteractive
  apt-get update -qq
  apt-get install -y -qq --no-install-recommends "${need[@]}"
  changed apt "installed ${need[*]}"
}

step_mingw_alternatives() {
  # Ubuntu ships -posix and -win32 variants behind update-alternatives; the Windows exes want posix threads (jobbuild.rs)
  local tool want cur any=0
  for tool in gcc g++; do
    want="/usr/bin/x86_64-w64-mingw32-$tool-posix"
    cur=$(readlink -f "/etc/alternatives/x86_64-w64-mingw32-$tool" 2>/dev/null || true)
    [ -x "$want" ] || die "no $want after apt"
    if [ "$cur" != "$want" ]; then update-alternatives --set "x86_64-w64-mingw32-$tool" "$want" >/dev/null; any=1; fi
  done
  [ "$any" = 1 ] && changed mingw-alternatives "posix threads" || ok mingw-alternatives "posix threads"
}

step_no_swap() {
  local any=0
  if [ -n "$(swapon --noheadings --show 2>/dev/null)" ]; then swapoff -a; any=1; fi
  if grep -qE '^[^#].*\sswap\s' /etc/fstab; then sed -i -E 's/^([^#].*\sswap\s.*)$/# \1  (disabled by infra\/build-server\/provision.sh)/' /etc/fstab; any=1; fi
  [ "$any" = 1 ] && changed swap "off, fstab entry commented" || ok swap "none"
}

step_sysctl() {
  local f=/etc/sysctl.d/90-igneum-build.conf tmp
  tmp=$(mktemp)
  cat > "$tmp" <<'EOF'
# igneum-build-1: a compile box with no swap (infra/build-server/provision.sh)
vm.swappiness = 0
vm.overcommit_memory = 0
vm.dirty_ratio = 20
vm.dirty_background_ratio = 5
vm.max_map_count = 1048576
fs.file-max = 4194304
fs.inotify.max_user_watches = 1048576
fs.inotify.max_user_instances = 8192
kernel.pid_max = 4194304
kernel.threads-max = 1048576
net.core.somaxconn = 4096
net.ipv4.tcp_fin_timeout = 15
EOF
  if [ -f "$f" ] && cmp -s "$tmp" "$f"; then rm -f "$tmp"; ok sysctl "$f"; return; fi
  install -m 644 "$tmp" "$f"; rm -f "$tmp"; sysctl --system >/dev/null
  changed sysctl "$f applied"
}

step_limits() {
  local f=/etc/security/limits.d/90-igneum-build.conf
  if [ -f "$f" ]; then ok limits; return; fi
  printf '%s soft nofile 1048576\n%s hard nofile 1048576\n%s soft nproc unlimited\n' "$BUILD_USER" "$BUILD_USER" "$BUILD_USER" > "$f"
  changed limits "$f"
}

step_user() {
  local keys
  if ! id -u "$BUILD_USER" >/dev/null 2>&1; then useradd -m -s /bin/bash -G users "$BUILD_USER"; changed user "$BUILD_USER created"; else ok user "$BUILD_USER"; fi
  install -d -m 700 -o "$BUILD_USER" -g "$BUILD_USER" "$BUILD_HOME/.ssh"
  if [ -n "$SSH_PUBKEY" ]; then keys="$SSH_PUBKEY"; elif [ -s /root/.ssh/authorized_keys ]; then keys=$(cat /root/.ssh/authorized_keys); else die "no key for $BUILD_USER: root has no authorized_keys and SSH_PUBKEY is unset"; fi
  if [ -f "$BUILD_HOME/.ssh/authorized_keys" ] && [ "$(cat "$BUILD_HOME/.ssh/authorized_keys")" = "$keys" ]; then ok authorized_keys; else
    printf '%s\n' "$keys" > "$BUILD_HOME/.ssh/authorized_keys"; chmod 600 "$BUILD_HOME/.ssh/authorized_keys"; chown "$BUILD_USER:$BUILD_USER" "$BUILD_HOME/.ssh/authorized_keys"
    changed authorized_keys "$(printf '%s\n' "$keys" | grep -c .) key(s) from root"
  fi
}

worktree_count() { find /srv/builds -mindepth 1 -maxdepth 1 -type d -not -name '_*' | wc -l | tr -d ' '; }

# docker (7 October 2026): the glibc proof runs a shipped binary inside ubuntu:20.04 and ubuntu:22.04 on the box (tools/build-remote.sh
# --ship's proof); user build may run containers. Nothing else of the box runs in docker.
step_docker() {
  command -v docker >/dev/null 2>&1 || die "docker is not installed (apt docker.io)"
  systemctl is-active --quiet docker || systemctl enable --now docker >/dev/null 2>&1
  if id -nG "$BUILD_USER" | tr ' ' '\n' | grep -qx docker; then ok docker "$(docker --version | cut -d, -f1), $BUILD_USER in the docker group"; else usermod -aG docker "$BUILD_USER"; changed docker "$(docker --version | cut -d, -f1), $BUILD_USER added to the docker group (new ssh sessions see it)"; fi
}

step_dirs() {
  local d any=0
  for d in /srv/builds /srv/builds/_locks /srv/sccache /srv/artefacts; do
    if [ ! -d "$d" ]; then install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "$d"; any=1; fi
  done
  if [ ! -f /srv/builds/_locks/slots ] || [ "$(cat /srv/builds/_locks/slots)" != "$SLOTS" ]; then printf '%s\n' "$SLOTS" > /srv/builds/_locks/slots; chown "$BUILD_USER:$BUILD_USER" /srv/builds/_locks/slots; any=1; fi
  for d in $WORKTREES; do
    case "$d" in */*|.*|_*) die "worktree name '$d' is not a plain directory name" ;; esac
    if [ ! -d "/srv/builds/$d" ]; then install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "/srv/builds/$d"; any=1; fi
  done
  [ "$any" = 1 ] && changed dirs "/srv/builds ($(worktree_count) worktree dirs), /srv/sccache, slots=$SLOTS" || ok dirs "$(worktree_count) worktree dirs, slots=$SLOTS"
}
# the lease tool (infra/build-server/lease.sh: per-core measurement leases, the quiet class, the reaper; 7 October 2026) from the
# mirror's master at /srv/builds/_bin/lease, which remote-run.sh's keeper calls; the mirror is pushed by step_mirrors' caller
step_lease_tool() {
  local src="/srv/igneum.git" want have=""
  install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" /srv/builds/_bin
  want=$(git -C "$src" show master:infra/build-server/lease.sh 2>/dev/null) || { ok lease-tool "mirror has no master yet; run-from-mac.sh installs it on the next provision"; return; }
  [ -f /srv/builds/_bin/lease ] && have=$(cat /srv/builds/_bin/lease)
  if [ "$want" = "$have" ]; then ok lease-tool "/srv/builds/_bin/lease is the mirror's master copy"; return; fi
  printf '%s\n' "$want" > /srv/builds/_bin/lease.new; chmod 755 /srv/builds/_bin/lease.new; chown "$BUILD_USER:$BUILD_USER" /srv/builds/_bin/lease.new
  mv /srv/builds/_bin/lease.new /srv/builds/_bin/lease; changed lease-tool "/srv/builds/_bin/lease installed from the mirror's master"
}
# headless Chromium self-test: the shell the dashboard lane's node tooling downloads (playwright or puppeteer cache of the build
# user) launches once with --headless and prints its version; when no shell is downloaded yet the libraries are checked by ldd of
# nothing, so the step only says so (the apt list above carries them)
step_headless_check() {
  local shell="" v
  shell=$(find "/home/$BUILD_USER/.cache/ms-playwright" "/home/$BUILD_USER/.cache/puppeteer" /srv -maxdepth 6 -type f \( -name headless_shell -o -name chrome-headless-shell -o -name chrome \) 2>/dev/null | head -1)
  [ -n "$shell" ] || { ok headless "no headless shell downloaded yet (the 16 libraries are installed; the lane's first capture downloads it)"; return; }
  if v=$(sudo -u "$BUILD_USER" timeout 60 "$shell" --headless --no-sandbox --disable-gpu --version 2>&1 | head -1) && [ -n "$v" ]; then ok headless "$shell: $v"
  else die "headless shell $shell does not launch: $v"; fi
}

step_mirrors() {
  local r any=0
  for r in /srv/igneum.git /srv/igneum-node.git; do
    if [ ! -d "$r" ]; then install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "$r"; as_build "git init -q --bare -b master $r"; any=1; fi
  done
  as_build "git config --global --get safe.directory >/dev/null 2>&1 || git config --global --add safe.directory '*'"
  as_build "git config --global init.defaultBranch master; git config --global gc.auto 0"
  [ "$any" = 1 ] && changed mirrors "bare /srv/igneum.git and /srv/igneum-node.git (push from the Mac, see the header)" || ok mirrors
}

step_rustup() {
  local cargo="$BUILD_HOME/.cargo/bin/cargo" rustup="$BUILD_HOME/.cargo/bin/rustup" any=0 t
  if [ ! -x "$rustup" ]; then
    as_build "curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --no-modify-path --default-toolchain $RUST_TOOLCHAIN" >/dev/null
    any=1
  fi
  if ! as_build "$rustup toolchain list" | grep -q "^$RUST_TOOLCHAIN-"; then as_build "$rustup toolchain install $RUST_TOOLCHAIN --profile minimal" >/dev/null; any=1; fi
  if [ "$(as_build "$rustup default" | cut -d- -f1)" != "$RUST_TOOLCHAIN" ]; then as_build "$rustup default $RUST_TOOLCHAIN" >/dev/null; any=1; fi
  for t in x86_64-pc-windows-gnu x86_64-unknown-linux-gnu; do
    as_build "$rustup target list --installed --toolchain $RUST_TOOLCHAIN" | grep -qx "$t" || { as_build "$rustup target add $t --toolchain $RUST_TOOLCHAIN" >/dev/null; any=1; }
  done
  as_build "$rustup component list --installed --toolchain $RUST_TOOLCHAIN" | grep -q '^clippy' || { as_build "$rustup component add clippy rustfmt --toolchain $RUST_TOOLCHAIN" >/dev/null; any=1; }
  [ "$any" = 1 ] && changed rustup "$(as_build "$cargo --version"), targets: $(as_build "$rustup target list --installed" | tr '\n' ' ')" || ok rustup "$(as_build "$cargo --version"), targets: $(as_build "$rustup target list --installed" | tr '\n' ' ')"
}

step_sccache() {
  local cargo="$BUILD_HOME/.cargo/bin/cargo" bin="$BUILD_HOME/.cargo/bin/sccache" cfgdir="$BUILD_HOME/.config/sccache" any=0 bytes tmp
  if [ ! -x "$bin" ] || { [ -n "$SCCACHE_VERSION" ] && ! "$bin" --version | grep -q " $SCCACHE_VERSION\$"; }; then
    as_build "$cargo install sccache --locked ${SCCACHE_VERSION:+--version $SCCACHE_VERSION}" >/dev/null 2>&1 || as_build "$cargo install sccache ${SCCACHE_VERSION:+--version $SCCACHE_VERSION}" >/dev/null
    any=1
  fi
  bytes=$(( SCCACHE_GB * 1024 * 1024 * 1024 ))
  install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "$cfgdir"
  tmp=$(mktemp)
  printf '[cache.disk]\ndir = "/srv/sccache"\nsize = %s\n' "$bytes" > "$tmp"
  if ! cmp -s "$tmp" "$cfgdir/config"; then install -m 644 -o "$BUILD_USER" -g "$BUILD_USER" "$tmp" "$cfgdir/config"; any=1; fi
  rm -f "$tmp"
  [ "$any" = 1 ] && changed sccache "$(as_build "$bin --version"), disk cache /srv/sccache, $SCCACHE_GB GB" || ok sccache "$(as_build "$bin --version"), /srv/sccache $SCCACHE_GB GB"
}

step_cargo_config() {
  # the build user's cargo defaults: sccache in front of rustc, 90 jobs (96 threads, 6 left for ssh, rsync and the
  # system), lld for the native target through clang. The Windows target's compilers and flags are NOT here: they are
  # set per build by tools/cross-remote.sh, the same variables as the Mac's proto-cuda/windows-node/cross-build.sh and
  # the PC's jobbuild.rs, so a build's flags are visible in the script that runs it.
  local f="$BUILD_HOME/.cargo/config.toml" tmp
  tmp=$(mktemp)
  cat > "$tmp" <<'EOF'
# igneum-build-1 (infra/build-server/provision.sh)
[build]
rustc-wrapper = "/home/build/.cargo/bin/sccache"
jobs = 90

[target.x86_64-unknown-linux-gnu]
linker = "clang"
rustflags = ["-C", "link-arg=-fuse-ld=lld"]

[net]
git-fetch-with-cli = true
EOF
  if cmp -s "$tmp" "$f"; then rm -f "$tmp"; ok cargo-config "$f"; return; fi
  install -m 644 -o "$BUILD_USER" -g "$BUILD_USER" "$tmp" "$f"; rm -f "$tmp"
  changed cargo-config "$f"
}

step_profile() {
  # sourced by tools/build-remote.sh's remote script (a non-login ssh shell reads no profile) and by login shells
  local f=/etc/profile.d/igneum-build.sh tmp
  tmp=$(mktemp)
  cat > "$tmp" <<EOF
# igneum-build-1 (infra/build-server/provision.sh)
export PATH="/home/build/.cargo/bin:/usr/local/bin:\$PATH"
export SCCACHE_DIR=/srv/sccache
export SCCACHE_CACHE_SIZE=${SCCACHE_GB}G
export CARGO_INCREMENTAL=0
export IGNEUM_BUILD_SLOTS_DIR=/srv/builds/_locks
export IGNEUM_BUILD_ROOT=/srv/builds
export IGNEUM_RUST_TOOLCHAIN=$RUST_TOOLCHAIN
EOF
  if [ -f "$f" ] && cmp -s "$tmp" "$f"; then rm -f "$tmp"; ok profile "$f"; return; fi
  install -m 644 "$tmp" "$f"; rm -f "$tmp"; changed profile "$f"
}

step_node() {
  local want have shasums tarball ver dir
  have=$(/usr/local/bin/node --version 2>/dev/null || true)
  case "$have" in v$NODE_MAJOR.*) ok node "$have"; return ;; esac
  # the newest $NODE_MAJOR release from nodejs.org, checked against its SHASUMS256.txt (https, the official host)
  shasums=$(curl -fsSL "https://nodejs.org/dist/latest-v$NODE_MAJOR.x/SHASUMS256.txt")
  tarball=$(printf '%s\n' "$shasums" | awk '$2 ~ /linux-x64\.tar\.xz$/ { print $2 }' | head -1)
  [ -n "$tarball" ] || die "no linux-x64 tarball in the Node $NODE_MAJOR SHASUMS"
  ver=${tarball#node-}; ver=${ver%-linux-x64.tar.xz}
  dir=/usr/local/lib/nodejs
  install -d "$dir"
  ( cd "$dir" && curl -fsSLO "https://nodejs.org/dist/latest-v$NODE_MAJOR.x/$tarball" && printf '%s\n' "$shasums" | grep " $tarball\$" | sha256sum -c --quiet - && tar -xJf "$tarball" && rm -f "$tarball" )
  ln -sfn "$dir/node-$ver-linux-x64/bin/node" /usr/local/bin/node
  ln -sfn "$dir/node-$ver-linux-x64/bin/npm" /usr/local/bin/npm
  ln -sfn "$dir/node-$ver-linux-x64/bin/npx" /usr/local/bin/npx
  changed node "$(/usr/local/bin/node --version) from nodejs.org (sha256 checked)"
}

# zig (main, 7 October 2026): the glibc 2.36 Linux artefacts for Debian 12 seeds and HiveOS rigs come from cargo-zigbuild with zig as
# the C/C++ toolchain, as infra/cross/build-linux.sh does on the Mac (tonight's seed took 14 restarts and three minutes down on a
# glibc 2.39 native build). The release the Mac uses (0.17.0), from ziglang.org with the sha256 of the official download index.
ZIG_VERSION="${ZIG_VERSION:-0.17.0}"
ZIG_SHA256="${ZIG_SHA256:-1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026}"   # zig-x86_64-linux-0.17.0.tar.xz, index.json 7 Oct 2026
step_zig() {
  local have dir tar
  have=$(/usr/local/bin/zig version 2>/dev/null || true)
  if [ "$have" = "$ZIG_VERSION" ]; then ok zig "$have"; return; fi
  dir=/usr/local/lib/zig; tar="zig-x86_64-linux-$ZIG_VERSION.tar.xz"
  install -d "$dir"
  ( cd "$dir" && curl -fsSLO "https://ziglang.org/download/$ZIG_VERSION/$tar" && echo "$ZIG_SHA256  $tar" | sha256sum -c --quiet - && tar -xJf "$tar" && rm -f "$tar" )
  ln -sfn "$dir/zig-x86_64-linux-$ZIG_VERSION/zig" /usr/local/bin/zig
  changed zig "$(/usr/local/bin/zig version) from ziglang.org (sha256 checked) at /usr/local/bin/zig"
}

step_sshd() {
  local f=/etc/ssh/sshd_config.d/10-igneum-build.conf tmp
  tmp=$(mktemp)
  cat > "$tmp" <<'EOF'
# igneum-build-1 (infra/build-server/provision.sh): keys only
PasswordAuthentication no
KbdInteractiveAuthentication no
ChallengeResponseAuthentication no
PubkeyAuthentication yes
PermitRootLogin prohibit-password
PermitEmptyPasswords no
X11Forwarding no
MaxAuthTries 4
ClientAliveInterval 60
ClientAliveCountMax 10
EOF
  if [ -f "$f" ] && cmp -s "$tmp" "$f"; then rm -f "$tmp"; ok sshd "$f"; return; fi
  install -m 644 "$tmp" "$f"; rm -f "$tmp"
  # Hetzner's installimage may leave a cloud-init drop-in that sets PasswordAuthentication yes; the lowest-numbered file wins
  if [ -f /etc/ssh/sshd_config.d/50-cloud-init.conf ] && grep -qi '^PasswordAuthentication yes' /etc/ssh/sshd_config.d/50-cloud-init.conf; then
    sed -i 's/^PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config.d/50-cloud-init.conf
  fi
  sshd -t || die "sshd -t rejected the configuration; the drop-in $f was NOT activated"
  systemctl reload ssh 2>/dev/null || systemctl reload sshd
  changed sshd "key-only, root prohibit-password"
}

# the worker dashboard's feed (asked for on 6 October 2026, approved by main): Caddy serves exactly two files of /srv/workers
# over HTTPS at build.igneum.network (A record in deSEC, set by main), read-only, no directory listing, every other path 404,
# CORS for dl.igneum.network, no caching. Nothing under /srv/workers is a secret (workers.json and headline.json are written
# by the dashboard collector and remote-run.sh); the Caddyfile refuses every other file name anyway. Issuer pinned to Let's
# Encrypt: Ubuntu's Caddy 2.6.2 fails the ZeroSSL fallback (HTTP 422 caddy_legacy_user_removed, 6 Oct 2026) and would retry it for ever.
step_caddy() {
  local f=/etc/caddy/Caddyfile tmp
  command -v caddy >/dev/null 2>&1 || die "caddy is not installed (apt)"
  install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" /srv/workers
  tmp=$(mktemp)
  cat > "$tmp" <<EOF
# igneum-build-1 (infra/build-server/provision.sh): the worker dashboard's two JSON files, nothing else
$WORKERS_HOST {
	tls {
		issuer acme
	}
	root * /srv/workers
	header Access-Control-Allow-Origin https://dl.igneum.network
	header Cache-Control "no-store"
	@notjson not path /workers.json /headline.json
	respond @notjson 404
	file_server
}
EOF
  if [ -f "$f" ] && cmp -s "$tmp" "$f"; then rm -f "$tmp"; systemctl is-active --quiet caddy || systemctl start caddy; ok caddy "$WORKERS_HOST"; return; fi
  caddy validate --config "$tmp" --adapter caddyfile >/dev/null 2>&1 || { rm -f "$tmp"; die "caddy validate rejected the Caddyfile"; }
  install -m 644 "$tmp" "$f"; rm -f "$tmp"
  systemctl enable --quiet caddy 2>/dev/null || true
  systemctl reload caddy 2>/dev/null || systemctl restart caddy
  changed caddy "$WORKERS_HOST serving /srv/workers/{workers,headline}.json"
}

# CUDA headers and stubs for the GPU workers' Linux build (main, 6 October 2026, the class v4 rehearsal): proto-cuda/nvrtc/worker.cpp
# and proto-opencl/host.c need cuda.h, nvrtc.h and CL/cl.h at compile time only and dlopen libcuda, libnvrtc and libOpenCL at run
# time (infra/cross/build-workers-linux.sh links -ldl -lpthread, no nvcc anywhere in the build files). NVIDIA's apt repository for
# Ubuntu 24.04, the 12.8 minor the repo's fetch-redist.sh pins (nvrtc 12.8.93, cudart 12.8.90); no driver (the box has no GPU),
# no nvcc. CUDA_MINOR overrides the minor.
CUDA_MINOR="${CUDA_MINOR:-12-8}"
step_cuda() {
  local keyring=/usr/share/keyrings/cuda-archive-keyring.gpg pkgs p need=() tmp
  pkgs=("cuda-nvrtc-dev-$CUDA_MINOR" "cuda-cudart-dev-$CUDA_MINOR" "cuda-driver-dev-$CUDA_MINOR" opencl-c-headers)
  for p in "${pkgs[@]}"; do dpkg -s "$p" >/dev/null 2>&1 || need+=("$p"); done
  if [ "${#need[@]}" = 0 ]; then ok cuda "${pkgs[*]} (headers and stubs, no nvcc, no driver)"; return; fi
  if [ ! -f "$keyring" ] && [ ! -f /etc/apt/sources.list.d/cuda-ubuntu2404-x86_64.list ]; then
    tmp=$(mktemp -d)
    curl -fsSL -o "$tmp/cuda-keyring.deb" https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
    DEBIAN_FRONTEND=noninteractive dpkg -i "$tmp/cuda-keyring.deb" >/dev/null; rm -rf "$tmp"
  fi
  export DEBIAN_FRONTEND=noninteractive
  apt-get update -qq
  apt-get install -y -qq --no-install-recommends "${need[@]}"
  changed cuda "installed ${need[*]} (headers under /usr/local/cuda-${CUDA_MINOR/-/.}/include, libcuda stub under .../lib64/stubs)"
}

step_ufw() {
  local p want=() any=0 status
  status=$(ufw status verbose 2>/dev/null || true)
  grep -q 'Default: deny (incoming), allow (outgoing)' <<<"$status" || { ufw --force default deny incoming >/dev/null; ufw --force default allow outgoing >/dev/null; any=1; }
  want=(22 80 443)
  for p in $P2P_PORTS; do want+=("$p"); done
  for p in "${want[@]}"; do
    grep -qE "^$p/tcp +ALLOW IN +Anywhere *$" <<<"$status" || { ufw allow "$p/tcp" >/dev/null; any=1; }
  done
  grep -q '^Status: active' <<<"$status" || { ufw --force enable >/dev/null; any=1; }
  [ "$any" = 1 ] && changed ufw "22, 80, 443 and ${P2P_PORTS} open, everything else denied" || ok ufw "22, 80, 443 and ${P2P_PORTS}"
}

# The GitHub Actions self-hosted runner for igneum-network/igneum (asked for 6 October 2026, "what else can the box work on").
# A dedicated user `runner` (no sudo, not in the build group), the runner at /opt/actions-runner (sha256 of the tarball checked
# against the release note), its own rustup pinned to RUST_TOOLCHAIN with the two targets and clippy, Node 22 and mingw from
# the system, and sccache against the box's cache in READ-ONLY mode (`rw_mode = "READ_ONLY"`, accepted by sccache 0.18 on
# 6 October 2026): a CI job may take hits from what the agents built, never write a line into their cache, and talks to its
# own sccache server on RUNNER_SCCACHE_PORT so the build user's server never compiles as the wrong user. Registration needs
# RUNNER_TOKEN (infra/build-server/runner/register.sh fetches one with gh as igneum-labs and pipes it over stdin); without
# it the step installs everything and says what is missing. The service is GitHub's own `svc.sh install runner` unit plus a
# drop-in with Nice=10 (agents' builds through build-remote.sh win the CPU; a CI job is a check, not a release build).
# Not ephemeral: GitHub recommends `--ephemeral` for autoscaled fleets that register a fresh runner per job; one standing
# runner on a private repository keeps its registration and cleans `_work` per job (approximate: from GitHub's runner
# documentation as remembered on 6 October 2026, the docs host answered 404 to the fetch that evening).
runner_env_file() {
  cat <<EOF
# $BOX_HOSTNAME (infra/build-server/provision.sh step_runner): the environment every CI job on this runner starts with
RUSTC_WRAPPER=/usr/local/bin/sccache
SCCACHE_CONF=$RUNNER_HOME/.config/sccache/config
SCCACHE_SERVER_PORT=$RUNNER_SCCACHE_PORT
CARGO_INCREMENTAL=0
CARGO_BUILD_JOBS=$RUNNER_JOBS
CARGO_NET_GIT_FETCH_WITH_CLI=true
IGNEUM_RUST_TOOLCHAIN=$RUST_TOOLCHAIN
EOF
}

step_runner() {
  local any=0 tarball cargo="$RUNNER_HOME/.cargo/bin/cargo" rustup="$RUNNER_HOME/.cargo/bin/rustup" t tmp svc unit dropin
  as_runner() { su - "$RUNNER_USER" -c "$*"; }
  # 1. the user: own group, no sudo, no membership of build; /home/runner 750 like the other homes
  if ! id -u "$RUNNER_USER" >/dev/null 2>&1; then useradd -m -s /bin/bash "$RUNNER_USER"; any=1; fi
  id -nG "$RUNNER_USER" | tr ' ' '\n' | grep -qx sudo && die "the runner user must never be in sudo"
  # 2. the runner itself, from the GitHub release with the published sha256
  if [ ! -x "$RUNNER_DIR/run.sh" ] || ! grep -q "\"$RUNNER_VERSION\"" "$RUNNER_DIR/.runner_version" 2>/dev/null; then
    [ ! -d "$RUNNER_DIR" ] || [ ! -f "$RUNNER_DIR/.runner" ] || log "runner: a configured runner is in place; the tarball is updated underneath it (the service restarts below)"
    install -d -m 755 -o "$RUNNER_USER" -g "$RUNNER_USER" "$RUNNER_DIR"
    tarball="actions-runner-linux-x64-$RUNNER_VERSION.tar.gz"
    tmp=$(mktemp -d)
    ( cd "$tmp" && curl -fsSLO "https://github.com/actions/runner/releases/download/v$RUNNER_VERSION/$tarball" \
      && printf '%s  %s\n' "$RUNNER_SHA256" "$tarball" | sha256sum -c --quiet - ) || { rm -rf "$tmp"; die "runner tarball download or sha256 check failed (version $RUNNER_VERSION)"; }
    tar -xzf "$tmp/$tarball" -C "$RUNNER_DIR"      # as root: the temp dir is root-only; ownership handed over below
    chown -R "$RUNNER_USER:$RUNNER_USER" "$RUNNER_DIR"
    rm -rf "$tmp"
    printf '"%s"\n' "$RUNNER_VERSION" > "$RUNNER_DIR/.runner_version"; chown "$RUNNER_USER:$RUNNER_USER" "$RUNNER_DIR/.runner_version"
    any=1
  fi
  # 3. toolchains for the runner user: rustup pinned like the box's, both targets, clippy and rustfmt
  if [ ! -x "$rustup" ]; then
    as_runner "curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --no-modify-path --default-toolchain $RUST_TOOLCHAIN" >/dev/null; any=1
  fi
  as_runner "$rustup toolchain list" | grep -q "^$RUST_TOOLCHAIN-" || { as_runner "$rustup toolchain install $RUST_TOOLCHAIN --profile minimal" >/dev/null; any=1; }
  [ "$(as_runner "$rustup default" | cut -d- -f1)" = "$RUST_TOOLCHAIN" ] || { as_runner "$rustup default $RUST_TOOLCHAIN" >/dev/null; any=1; }
  for t in x86_64-pc-windows-gnu x86_64-unknown-linux-gnu; do
    as_runner "$rustup target list --installed --toolchain $RUST_TOOLCHAIN" | grep -qx "$t" || { as_runner "$rustup target add $t --toolchain $RUST_TOOLCHAIN" >/dev/null; any=1; }
  done
  as_runner "$rustup component list --installed --toolchain $RUST_TOOLCHAIN" | grep -q '^clippy' || { as_runner "$rustup component add clippy rustfmt --toolchain $RUST_TOOLCHAIN" >/dev/null; any=1; }
  as_runner "git config --global --get safe.directory >/dev/null 2>&1 || git config --global --add safe.directory '*'"   # the mirrors are owned by build
  # 3b. the CI red watcher's record file: the workflow's `red` job appends one line per failed master or release run here
  #     (tools/ci/red-watch.mjs record); the poster (infra/build-server/ci-red) reads it as build
  #     (tools/ci/red-watch.mjs record); remote-run.sh appends the box's own red rows as build; the shared group `cired` lets
  #     both write one file (664 in a 2775 directory), and the poster (infra/build-server/ci-red) reads it as build
  getent group cired >/dev/null || { groupadd cired; any=1; }
  id -nG "$RUNNER_USER" | tr ' ' '\n' | grep -qx cired || { usermod -aG cired "$RUNNER_USER"; any=1; }
  id -nG "$BUILD_USER" | tr ' ' '\n' | grep -qx cired || { usermod -aG cired "$BUILD_USER"; any=1; }
  if [ ! -d /srv/ci-red ]; then install -d -o root -g cired -m 2775 /srv/ci-red; any=1; fi
  [ -f /srv/ci-red/red.jsonl ] || { install -o root -g cired -m 664 /dev/null /srv/ci-red/red.jsonl; any=1; }
  # 4. sccache: the build user's binary copied system-wide (the runner cannot read /home/build), a read-only view of /srv/sccache
  if [ ! -x /usr/local/bin/sccache ] || ! cmp -s "$BUILD_HOME/.cargo/bin/sccache" /usr/local/bin/sccache; then
    install -m 755 "$BUILD_HOME/.cargo/bin/sccache" /usr/local/bin/sccache; any=1
  fi
  install -d -m 755 -o "$RUNNER_USER" -g "$RUNNER_USER" "$RUNNER_HOME/.config" "$RUNNER_HOME/.config/sccache"
  tmp=$(mktemp)
  printf '[cache.disk]\ndir = "/srv/sccache"\nsize = %s\nrw_mode = "READ_ONLY"\n' "$(( SCCACHE_GB * 1024 * 1024 * 1024 ))" > "$tmp"
  if ! cmp -s "$tmp" "$RUNNER_HOME/.config/sccache/config"; then install -m 644 -o "$RUNNER_USER" -g "$RUNNER_USER" "$tmp" "$RUNNER_HOME/.config/sccache/config"; any=1; fi
  rm -f "$tmp"
  # 6. registration (once; --replace re-registers under the same name after a token is given again)
  if [ ! -f "$RUNNER_DIR/.runner" ]; then
    if [ -z "$RUNNER_TOKEN" ]; then
      log "runner: NOT registered: no RUNNER_TOKEN. From the Mac: infra/build-server/runner/register.sh (gh as igneum-labs fetches a registration token and pipes it here)"
      return
    fi
    # the token goes to config.sh as an argument of a process owned by runner for a second; it is a one-hour registration
    # token (not the runner's credential, which config.sh writes to .credentials, mode 600, owner runner), never logged here
    RUNNER_TOKEN="$RUNNER_TOKEN" runuser -u "$RUNNER_USER" -- bash -c "cd '$RUNNER_DIR' && ./config.sh --unattended --replace --url '$RUNNER_REPO_URL' --token \"\$RUNNER_TOKEN\" --name '$RUNNER_NAME' --labels '$RUNNER_LABELS' --work _work" >/dev/null \
      || die "runner: config.sh failed (an expired token? register.sh fetches a fresh one)"
    any=1
    log "runner: registered as $RUNNER_NAME with labels self-hosted, linux, x64, $RUNNER_LABELS${RUNNER_CPUS:+, AllowedCPUs $RUNNER_CPUS}"
  fi
  # 7. the service: GitHub's unit (User=runner, KillMode=process) plus Nice and a restart on failure
  svc="actions.runner.$(sed -n 's/.*"gitHubUrl": *"https:\/\/github.com\/\([^"]*\)".*/\1/p' "$RUNNER_DIR/.runner" | tr '/' '-').$RUNNER_NAME.service"
  unit="/etc/systemd/system/$svc"
  if [ ! -f "$unit" ]; then ( cd "$RUNNER_DIR" && ./svc.sh install "$RUNNER_USER" >/dev/null ) || die "runner: svc.sh install failed"; any=1; fi
  # 8. the job environment, AFTER svc.sh install: its env.sh rewrites .env and .path from the installing shell (6 October 2026:
  #    the second provision run found them changed and restarted the service for nothing); the runner reads both at start
  tmp=$(mktemp); runner_env_file > "$tmp"
  if ! cmp -s "$tmp" "$RUNNER_DIR/.env"; then install -m 644 -o "$RUNNER_USER" -g "$RUNNER_USER" "$tmp" "$RUNNER_DIR/.env"; any=1; fi
  rm -f "$tmp"
  tmp=$(mktemp); printf '%s\n' "$RUNNER_HOME/.cargo/bin:/usr/local/bin:/usr/bin:/bin" > "$tmp"
  if ! cmp -s "$tmp" "$RUNNER_DIR/.path"; then install -m 644 -o "$RUNNER_USER" -g "$RUNNER_USER" "$tmp" "$RUNNER_DIR/.path"; any=1; fi
  rm -f "$tmp"
  dropin="/etc/systemd/system/$svc.d/igneum.conf"
  tmp=$(mktemp)
  printf '# %s (infra/build-server/provision.sh step_runner)\n[Service]\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nRestart=on-failure\nRestartSec=30\n' "$BOX_HOSTNAME" > "$tmp"
  [ -z "$RUNNER_CPUS" ] || printf 'AllowedCPUs=%s\n' "$RUNNER_CPUS" >> "$tmp"
  install -d -m 755 "$(dirname "$dropin")"
  if ! cmp -s "$tmp" "$dropin"; then install -m 644 "$tmp" "$dropin"; systemctl daemon-reload; any=1; fi
  rm -f "$tmp"
  systemctl enable --quiet "$svc" 2>/dev/null || true
  if [ "$any" = 1 ]; then systemctl restart "$svc"; else systemctl is-active --quiet "$svc" || systemctl start "$svc"; fi
  sleep 2
  systemctl is-active --quiet "$svc" || die "runner: $svc is not active: journalctl -u '$svc' -n 30"
  [ "$any" = 1 ] && changed runner "$svc active as $RUNNER_USER, runner $RUNNER_VERSION, $(as_runner "$cargo --version"), sccache read-only on /srv/sccache, jobs $RUNNER_JOBS" \
                 || ok runner "$svc active, runner $RUNNER_VERSION, $(as_runner "$cargo --version")"
}

# cargo tools the night battery needs (infra/build-server/night): cargo-audit for the advisory check of every Cargo.lock
step_cargo_tools() {
  local cargo="$BUILD_HOME/.cargo/bin/cargo" any=0
  if [ ! -x "$BUILD_HOME/.cargo/bin/cargo-audit" ]; then as_build "$cargo install cargo-audit --locked" >/dev/null 2>&1 || as_build "$cargo install cargo-audit" >/dev/null; any=1; fi
  # cargo-zigbuild (main, 7 October 2026): the glibc 2.36 Linux artefacts for Debian 12 seeds and HiveOS rigs (tools/build-remote.sh --ship)
  if [ ! -x "$BUILD_HOME/.cargo/bin/cargo-zigbuild" ]; then as_build "$cargo install cargo-zigbuild --locked" >/dev/null 2>&1 || as_build "$cargo install cargo-zigbuild" >/dev/null; any=1; fi
  [ "$any" = 1 ] && changed cargo-tools "$(as_build "$BUILD_HOME/.cargo/bin/cargo-audit --version"), $(as_build "$BUILD_HOME/.cargo/bin/cargo-zigbuild --version")" || ok cargo-tools "$(as_build "$BUILD_HOME/.cargo/bin/cargo-audit --version"), $(as_build "$BUILD_HOME/.cargo/bin/cargo-zigbuild --version")"
}

# the night battery (infra/build-server/night): the script and remote-run.sh into /srv/builds/_bin, the two units, the timer
# enabled. The files come out of the bare mirror /srv/igneum.git at NIGHT_REF (master; a branch while the work is unmerged),
# so provision.sh stays one piped file and the box runs what the repository holds. The battery re-execs itself from master's
# checkout at run time, so the copy here only has to be good enough to check out.
NIGHT_REF="${NIGHT_REF:-master}"
step_night() {
  local any=0 f tmp u
  install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" /srv/builds/_bin /srv/builds/_night /srv/builds/_log
  as_build "git -C /srv/igneum.git cat-file -e '$NIGHT_REF:infra/build-server/night/night-battery.sh'" 2>/dev/null || { log "night: $NIGHT_REF on /srv/igneum.git has no infra/build-server/night/night-battery.sh (push the branch, or NIGHT_REF=<branch>)"; return; }
  for f in infra/build-server/night/night-battery.sh infra/build-server/remote-run.sh; do
    tmp=$(mktemp); as_build "git -C /srv/igneum.git show '$NIGHT_REF:$f'" > "$tmp"
    if ! cmp -s "$tmp" "/srv/builds/_bin/$(basename "$f")"; then install -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "$tmp" "/srv/builds/_bin/$(basename "$f")"; any=1; fi
    rm -f "$tmp"
  done
  for u in igneum-night-battery.service igneum-night-battery.timer; do
    tmp=$(mktemp); as_build "git -C /srv/igneum.git show '$NIGHT_REF:infra/build-server/night/$u'" > "$tmp"
    if ! cmp -s "$tmp" "/etc/systemd/system/$u"; then install -m 644 "$tmp" "/etc/systemd/system/$u"; any=1; fi
    rm -f "$tmp"
  done
  [ "$any" = 1 ] && systemctl daemon-reload
  systemctl is-enabled --quiet igneum-night-battery.timer 2>/dev/null || { systemctl enable --now --quiet igneum-night-battery.timer; any=1; }
  systemctl is-active --quiet igneum-night-battery.timer || systemctl start igneum-night-battery.timer
  [ "$any" = 1 ] && changed night "timer $(systemctl list-timers igneum-night-battery.timer --no-pager --no-legend | awk '{ print $1, $2, $3, $4 }'), files from $NIGHT_REF" \
                 || ok night "next $(systemctl list-timers igneum-night-battery.timer --no-pager --no-legend | awk '{ print $1, $2, $3, $4 }')"
}

step_summary() {
  log "summary:"
  {
    printf 'host %s, %s threads, %s RAM, root fs %s free\n' "$(hostname)" "$(nproc)" "$(awk '/MemTotal/ { printf "%d GB", $2 / 1024 / 1024 }' /proc/meminfo)" "$(df -h / | awk 'NR == 2 { print $4 }')"
    printf 'raid: %s\n' "$(grep -E '^md' /proc/mdstat 2>/dev/null | tr '\n' ';' || echo none)"
    printf 'rust: %s | %s | targets %s\n' "$(as_build "$BUILD_HOME/.cargo/bin/rustc --version")" "$(as_build "$BUILD_HOME/.cargo/bin/cargo --version")" "$(as_build "$BUILD_HOME/.cargo/bin/rustup target list --installed" | tr '\n' ' ')"
    printf 'sccache: %s, %s\n' "$(as_build "$BUILD_HOME/.cargo/bin/sccache --version")" "$(cat "$BUILD_HOME/.config/sccache/config" | tr '\n' ' ')"
    printf 'zig: %s, cargo-zigbuild %s (glibc 2.36 Linux artefacts: tools/build-remote.sh --ship, tools/workers-remote.sh)\n' "$(/usr/local/bin/zig version 2>/dev/null || echo missing)" "$(as_build "$BUILD_HOME/.cargo/bin/cargo-zigbuild --version 2>/dev/null | awk '{ print \$NF }'" || echo missing)"
    printf 'mingw: %s\n' "$(x86_64-w64-mingw32-gcc-posix --version | head -1)"
    printf 'clang: %s | lld: %s\n' "$(clang --version | head -1)" "$(ld.lld --version | head -1)"
    printf 'node: %s | git: %s | tmux: %s\n' "$(/usr/local/bin/node --version)" "$(git --version)" "$(tmux -V)"
    printf 'swap: %s\n' "$(swapon --noheadings --show 2>/dev/null | wc -l | awk '{ print ($1 == 0) ? "none" : $1 " device(s) ACTIVE" }')"
    printf 'mirrors: /srv/igneum.git (%s) /srv/igneum-node.git (%s)\n' "$(as_build 'git -C /srv/igneum.git branch --list | wc -l') branches" "$(as_build 'git -C /srv/igneum-node.git branch --list | wc -l') branches"
    printf 'builds: %s worktree dirs under /srv/builds, %s slot(s)\n' "$(worktree_count)" "$(cat /srv/builds/_locks/slots)"
    printf 'ufw: %s\n' "$(ufw status | grep -E 'ALLOW' | awk '{ print $1 }' | tr '\n' ' ')"
    printf 'caddy: %s, %s\n' "$(caddy version 2>/dev/null | cut -d' ' -f1)" "$(systemctl is-active caddy 2>/dev/null) at https://$WORKERS_HOST/headline.json"
    printf 'cuda headers: %s\n' "$(ls -d /usr/local/cuda-*/include 2>/dev/null | tr '\n' ' ')$( [ -f /usr/include/CL/cl.h ] && echo '+ CL/cl.h' )"
    printf 'runner: %s\n' "$( [ -f "$RUNNER_DIR/.runner" ] && printf '%s, %s, user %s' "$(systemctl list-units --type=service --no-legend 'actions.runner.*' | awk '{ print $1 ": " $4 }' | head -1)" "v$(tr -d '"' < "$RUNNER_DIR/.runner_version" 2>/dev/null)" "$RUNNER_USER" || echo "installed, NOT registered (infra/build-server/runner/register.sh)" )"
    printf 'night battery: %s\n' "$(systemctl list-timers igneum-night-battery.timer --no-pager --no-legend 2>/dev/null | awk '{ print "next " $1, $2, $3, $4 }')"
    printf 'ssh line: ssh -i ~/.ssh/igneum_ed25519 build@%s\n' "$(hostname -I 2>/dev/null | awk '{ print $1 }')"
  } | sed 's/^/  /'
}

do_provision() {
  step_os_check
  step_hostname
  step_apt
  step_mingw_alternatives
  step_no_swap
  step_sysctl
  step_limits
  step_user
  step_docker
  step_dirs
  step_mirrors
  step_lease_tool
  step_rustup
  step_sccache
  step_cargo_config
  step_profile
  step_node
  step_sshd
  if [ "$ROLE" = sweep ]; then
    # a rented SWEEP WORKER (Hetzner Cloud CCX, by the hour, main's order 7 Oct 2026): the build user, toolchain, sccache, mirrors,
    # lease tool and firewall only; no dashboard feed, no CUDA, no CI runner, no night battery, no Chromium, no zig
    step_ufw
    step_cargo_tools
  else
    step_caddy
    step_cuda
    step_ufw
    step_runner
    step_cargo_tools
    step_zig
    step_night
    step_headless_check
  fi
  step_summary
  log "done"
}

case "$MODE" in
  install) do_install ;;
  provision) do_provision ;;
  auto) if in_rescue; then log "rescue system detected: install mode"; do_install; else do_provision; fi ;;
  *) die "MODE must be auto, install or provision" ;;
esac