693 lines
48 KiB
Bash
Executable file
693 lines
48 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Provision igneum-build-1, the Hetzner dedicated build server (AX162-1-LTD: EPYC 9454P 48 cores / 96 threads, 128 GB,
|
|
# 2x 3.84 TB NVMe, Falkenstein; ordered 6 October 2026). Idempotent: every step checks before it changes anything and
|
|
# says "ok" (nothing to do) or "changed". Run it over ssh as root; nothing here reads a secret.
|
|
#
|
|
# infra/build-server/run-from-mac.sh <ip> the usual way (ships this file, fills WORKTREES, writes the host file)
|
|
# ssh root@<ip> 'bash -s' < infra/build-server/provision.sh the bare way
|
|
# ssh root@<ip> 'MODE=install bash -s' < infra/build-server/provision.sh force the rescue-system path
|
|
#
|
|
# Two modes, chosen by MODE (auto, install, provision; default auto):
|
|
# install the box booted into Hetzner's rescue system (installimage present, hostname rescue*): run installimage in
|
|
# batch mode for Ubuntu 24.04 with software RAID 1 over the two NVMe drives, no swap, the rescue system's
|
|
# authorized_keys taken over, the image signature checked, then reboot. Run the script again after the reboot.
|
|
# Ran on igneum-build-1 on 6 October 2026 at 17:16 to 17:20 UTC (16 steps, no prompt).
|
|
# provision a running Ubuntu 24.04: user `build` with root's key, the compiler and cross toolchains, rustup pinned to
|
|
# RUST_TOOLCHAIN with the x86_64-pc-windows-gnu target, sccache with a 100 GB disk cache, Node 22, git, tmux,
|
|
# a swap-free tuned sysctl, the two bare mirrors (/srv/igneum.git, /srv/igneum-node.git), /srv/builds with one
|
|
# directory per agent worktree, sshd key-only, ufw with 22 and the seed p2p ports.
|
|
#
|
|
# Settings (environment, all optional):
|
|
# RUST_TOOLCHAIN 1.99.0 the channel of rust-toolchain.toml at the repo root (run-from-mac.sh passes it; one file pins
|
|
# the Mac, the box, the PCs and CI since 7 October 2026). tools/build-remote.sh refuses a build
|
|
# when the pin, the Mac's rustc or the box's rustc differ.
|
|
# SCCACHE_GB 100 the local disk cache at /srv/sccache
|
|
# SCCACHE_VERSION (unset) a `cargo install sccache --version` pin; unset = the newest on crates.io
|
|
# NODE_MAJOR 22
|
|
# WORKTREES "" space-separated agent worktree names, one /srv/builds/<name> each (run-from-mac.sh fills it from
|
|
# `git worktree list` on the Mac; tools/build-remote.sh creates a missing one on first use)
|
|
# SLOTS 2 remote build slots (tools/build-remote.sh takes one; remote-run.sh sets CARGO_BUILD_JOBS 90 when it holds
|
|
# the only taken slot and 45 when both are held; a measurement takes the `measure` file and excludes builds)
|
|
# P2P_PORTS "26611 26811 26621" TCP ports ufw opens beside 22: the devnet seed's p2p (infra/seed-nodes/config.sh devnet
|
|
# P2P_PORT=26611) and the testnet seed's (26811). A suffixed devnet (Devnet 2, the fleet's staging
|
|
# chain) listens on the same 26611 (infra/cloud-devnet/config.sh P2P_PORT=26611); the Devnet 2 seed that
|
|
# runs on build-1 as a bare process listens on 26621. RPC ports
|
|
# (26610, 28610, 26790 and the 268xx set) stay on loopback as on every seed, so they are not opened.
|
|
# BOX_HOSTNAME igneum-build-1
|
|
# WORKERS_HOST build.igneum.network Caddy serves /srv/workers/{workers,headline}.json there (read-only, all else 404)
|
|
# SSH_PUBKEY (unset) a public key line for the build user when root has no authorized_keys (installimage installs it)
|
|
#
|
|
# Mirrors: the Mac pushes to them (never a clone from GitHub; the fork vendor/igneum-node exists only on the Mac):
|
|
# git -C /Users/joshm/Projects/igneum remote add build build@<ip>:/srv/igneum.git
|
|
# git -C /Users/joshm/Projects/igneum/vendor/igneum-node remote add build build@<ip>:/srv/igneum-node.git
|
|
# git push build --all (tools/build-remote.sh pushes the branch it builds before every build)
|
|
#
|
|
# What this script does NOT do: install zig or cargo-zigbuild (the Mac's glibc 2.36 Linux cross-build, infra/cross/build-linux.sh,
|
|
# stays on the Mac until the box is proven; a native build here links glibc 2.39, which Debian 13 seeds accept and HiveOS
|
|
# does not), start any node, or copy a secret. The installimage flags and the image name were read from the live rescue system
|
|
# on 6 October 2026 (`installimage -h`, /root/.oldroot/nfs/images); the script still reads the image list instead of hard-coding a name.
|
|
set -euo pipefail
|
|
|
|
MODE="${MODE:-auto}"
|
|
RUST_TOOLCHAIN="${RUST_TOOLCHAIN:-1.99.0}"
|
|
SCCACHE_GB="${SCCACHE_GB:-100}"
|
|
SCCACHE_VERSION="${SCCACHE_VERSION:-}"
|
|
NODE_MAJOR="${NODE_MAJOR:-22}"
|
|
WORKTREES="${WORKTREES:-}"
|
|
SLOTS_GIVEN="${SLOTS:-}"; SLOTS="${SLOTS:-2}" # 2 since main's ruling of 6 October 2026 (20:3x UK); remote-run.sh gives 90 jobs alone, 45 beside another
|
|
P2P_PORTS="${P2P_PORTS:-26611 26811 26621}" # 26621: the Devnet 2 seed on build-1 (blocked by ufw until 7 Oct 2026 16:40 BST; the fleet lane found it closed from outside)
|
|
BOX_HOSTNAME="${BOX_HOSTNAME:-igneum-build-1}"
|
|
case "$BOX_HOSTNAME" in *-2|*-4) [ -n "$SLOTS_GIVEN" ] || SLOTS=3 ;; esac # build-2 and build-4 (AX162-1, 96 threads): three slots (main, 7 Oct 2026; the bounded runs take one 32-core band each)
|
|
WORKERS_HOST="${WORKERS_HOST:-build.igneum.network}" # the dashboard feed's HTTPS name (A record in deSEC, 6 Oct 2026)
|
|
SSH_PUBKEY="${SSH_PUBKEY:-}"
|
|
BUILD_USER=build
|
|
BUILD_HOME=/home/$BUILD_USER
|
|
# the GitHub Actions runner (step_runner, 6 October 2026 evening): a dedicated user, never build and never root
|
|
RUNNER_USER=runner
|
|
RUNNER_HOME=/home/$RUNNER_USER
|
|
RUNNER_DIR=/opt/actions-runner
|
|
RUNNER_VERSION="${RUNNER_VERSION:-2.338.0}" # github.com/actions/runner releases, read 6 October 2026
|
|
RUNNER_SHA256="${RUNNER_SHA256:-af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32}" # the release note's linux-x64 line
|
|
RUNNER_REPO_URL="${RUNNER_REPO_URL:-https://github.com/igneum-network/igneum}"
|
|
RUNNER_NAME="${RUNNER_NAME:-$BOX_HOSTNAME}"
|
|
RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1,ci-red}" # added to the defaults self-hosted, linux, x64. igneum-build-1 is the POOL label
|
|
# (every box that takes pow and sims carries it); ci-red marks the one box that
|
|
# holds the red watcher's record file and poster. A second box: BOX_HOSTNAME=igneum-build-2
|
|
# RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 (register.sh --host)
|
|
RUNNER_CPUS="${RUNNER_CPUS:-}" # AllowedCPUs for the runner's service when set (a second box is bounded like a suite: 32 cores, nice 10)
|
|
case "$BOX_HOSTNAME" in *-2|*-3|*-4) [ -n "$RUNNER_CPUS" ] || RUNNER_CPUS="0-31" ;; esac # build-2, build-3 and build-4 join the pool (label igneum-build-1) bounded to 32 cores at Nice 10 (main, 7 Oct 2026)
|
|
RUNNER_JOBS="${RUNNER_JOBS:-48}" # cargo jobs for a CI job: half the box, the agents' builds keep the rest
|
|
RUNNER_TOKEN="${RUNNER_TOKEN:-}" # a registration token (1 h), from infra/build-server/runner/register.sh over stdin; never logged
|
|
RUNNER_SCCACHE_PORT="${RUNNER_SCCACHE_PORT:-4227}" # the runner's own sccache server; 4226 is the build user's
|
|
|
|
log() { printf '%s provision: %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
|
die() { log "ERROR: $*" >&2; exit 1; }
|
|
changed() { log "$1: changed${2:+ ($2)}"; }
|
|
ok() { log "$1: ok${2:+ ($2)}"; }
|
|
as_build() { su - "$BUILD_USER" -c "$*"; }
|
|
|
|
[ "$(id -u)" = 0 ] || die "run as root"
|
|
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
# install mode: the rescue system
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
INSTALLIMAGE=/root/.oldroot/nfs/install/installimage # not on PATH in a non-interactive ssh shell (read 6 Oct 2026)
|
|
|
|
in_rescue() {
|
|
[ -x "$INSTALLIMAGE" ] || return 1
|
|
case "$(hostname)" in rescue*) return 0 ;; esac
|
|
[ -d /root/.oldroot/nfs/images ]
|
|
}
|
|
|
|
do_install() {
|
|
local images drives image parts
|
|
images=/root/.oldroot/nfs/images
|
|
[ -d "$images" ] || die "no image directory at $images: not the Hetzner rescue system"
|
|
# the Ubuntu 24.04 (noble) amd64 base image the rescue system offers (read on 6 October 2026: Ubuntu-2404-noble-amd64-base.tar.zst
|
|
# with a detached .sig; read, not hard-coded, because Hetzner refreshes the names)
|
|
image=$(find "$images" -maxdepth 1 -type f -iregex '.*/ubuntu-2404.*amd64.*\.tar\.\(zst\|gz\|xz\)' -printf '%f\n' | sort | tail -1 || true)
|
|
[ -n "$image" ] || die "no Ubuntu 24.04 amd64 image under $images: $(find "$images" -maxdepth 1 -type f -printf '%f ' )"
|
|
mapfile -t drives < <(lsblk -dn -o NAME,TYPE | awk '$2 == "disk" && $1 ~ /^nvme/ { print $1 }' | sort)
|
|
[ "${#drives[@]}" = 2 ] || die "expected exactly two NVMe drives for RAID 1, found ${#drives[@]}: ${drives[*]:-none}"
|
|
[ -s /root/.ssh/authorized_keys ] || die "/root/.ssh/authorized_keys is empty in the rescue system; -t yes would carry nothing into the image"
|
|
[ -d /sys/firmware/efi ] || log "WARNING: no /sys/firmware/efi, the box booted in BIOS mode; the esp partition is harmless but grub goes to the MBR"
|
|
# no swap partition: 128 GB of RAM and a swap-free sysctl (the provision mode checks no swap is active)
|
|
parts="/boot/efi:esp:512M,/boot:ext4:1G,/:ext4:all"
|
|
log "installimage: image $image, drives ${drives[*]} as software RAID 1, partitions $parts, hostname $BOX_HOSTNAME, rescue ssh keys taken over (-t yes), image signature checked (-g)"
|
|
log "this WIPES ${drives[*]}"
|
|
# flag form, read from `installimage -h` on 6 October 2026: -a batch, -n hostname, -r raid, -l level, -i image, -g verify
|
|
# the detached signature, -p partitions mount:fs:size, -d drives, -t yes take over the rescue system's ssh keys (root's
|
|
# authorized_keys), -G yes new host keys. The -c config form forbids every other flag, so the keys could not travel with it.
|
|
TERM="${TERM:-xterm}" "$INSTALLIMAGE" -a -n "$BOX_HOSTNAME" -r yes -l 1 -i "$images/$image" -g -p "$parts" -d "$(IFS=,; echo "${drives[*]}")" -t yes -G yes
|
|
log "installimage finished; rebooting into Ubuntu. Run this script again (MODE=provision or auto) once ssh answers (the host key is new: -G yes)."
|
|
sync; reboot
|
|
}
|
|
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
# provision mode: the installed Ubuntu
|
|
# ----------------------------------------------------------------------------------------------------------------------
|
|
step_hostname() {
|
|
if [ "$(hostnamectl --static 2>/dev/null || hostname)" = "$BOX_HOSTNAME" ]; then ok hostname "$BOX_HOSTNAME"; return; fi
|
|
hostnamectl set-hostname "$BOX_HOSTNAME"; grep -q "$BOX_HOSTNAME" /etc/hosts || printf '127.0.1.1 %s\n' "$BOX_HOSTNAME" >> /etc/hosts
|
|
changed hostname "$BOX_HOSTNAME"
|
|
}
|
|
|
|
step_os_check() {
|
|
. /etc/os-release
|
|
[ "${ID:-}" = ubuntu ] && [ "${VERSION_ID:-}" = 24.04 ] || die "this is ${PRETTY_NAME:-unknown}, not Ubuntu 24.04"
|
|
ok os "$PRETTY_NAME, $(nproc) threads, $(awk '/MemTotal/ { printf "%d GB", $2 / 1024 / 1024 }' /proc/meminfo)"
|
|
}
|
|
|
|
# libprotobuf-dev: the well-known .proto files (google/protobuf/empty.proto) that sp1-prover-types's build script imports
|
|
# (6 October 2026, the first proving build on the box: "protoc failed: google/protobuf/empty.proto: File not found").
|
|
# the proven Ubuntu 24.04 set: the PC build job's APT lists (app/igneum-app/src/jobbuild.rs: mingw-w64 posix threads so
|
|
# libstdc++ has std::thread for rocksdb, clang for librocksdb-sys's bindgen, protoc for the node's proto crates) plus the
|
|
# task's list (build-essential, clang, lld, pkg-config, libssl-dev, cmake, git, tmux) and what the scripts here call
|
|
APT_PACKAGES=(
|
|
build-essential clang lld llvm libclang-dev pkg-config libssl-dev cmake protobuf-compiler libprotobuf-dev
|
|
gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-x86-64-dev mingw-w64-tools
|
|
git tmux curl ca-certificates xz-utils zstd unzip rsync jq python3 ufw htop file caddy docker.io
|
|
# the GitHub Actions runner's .NET runtime needs libicu (bin/installdependencies.sh would install it); the two Python
|
|
# simulators (sim/finality_v2.py, sim/difficulty/sim.py) need numpy, which ci.yml pip-installs on GitHub's runners
|
|
libicu74 python3-numpy
|
|
# innoextract: tools/repro reads the shipped igneumd.exe and igneum-miner.exe out of the public Inno Setup installer
|
|
innoextract
|
|
# headless Chromium (the dashboard lane's site captures): the 16 system libraries it dlopens, found missing on both boxes
|
|
# on 7 October 2026 (installed by hand at 14:5x UK; step_headless_check launches the shell once and prints its version)
|
|
libatk1.0-0t64 libatk-bridge2.0-0t64 libatspi2.0-0t64 libcairo2 libcups2t64 libgbm1 libpango-1.0-0 libx11-6 libxcb1
|
|
libxcomposite1 libxdamage1 libxext6 libxfixes3 libxrandr2 libasound2t64 libxkbcommon0 fonts-liberation
|
|
)
|
|
step_apt() {
|
|
local need=() p
|
|
for p in "${APT_PACKAGES[@]}"; do dpkg -s "$p" >/dev/null 2>&1 || need+=("$p"); done
|
|
if [ "${#need[@]}" = 0 ]; then ok apt "${#APT_PACKAGES[@]} packages present"; return; fi
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
apt-get update -qq
|
|
apt-get install -y -qq --no-install-recommends "${need[@]}"
|
|
changed apt "installed ${need[*]}"
|
|
}
|
|
|
|
step_mingw_alternatives() {
|
|
# Ubuntu ships -posix and -win32 variants behind update-alternatives; the Windows exes want posix threads (jobbuild.rs)
|
|
local tool want cur any=0
|
|
for tool in gcc g++; do
|
|
want="/usr/bin/x86_64-w64-mingw32-$tool-posix"
|
|
cur=$(readlink -f "/etc/alternatives/x86_64-w64-mingw32-$tool" 2>/dev/null || true)
|
|
[ -x "$want" ] || die "no $want after apt"
|
|
if [ "$cur" != "$want" ]; then update-alternatives --set "x86_64-w64-mingw32-$tool" "$want" >/dev/null; any=1; fi
|
|
done
|
|
[ "$any" = 1 ] && changed mingw-alternatives "posix threads" || ok mingw-alternatives "posix threads"
|
|
}
|
|
|
|
step_no_swap() {
|
|
local any=0
|
|
if [ -n "$(swapon --noheadings --show 2>/dev/null)" ]; then swapoff -a; any=1; fi
|
|
if grep -qE '^[^#].*\sswap\s' /etc/fstab; then sed -i -E 's/^([^#].*\sswap\s.*)$/# \1 (disabled by infra\/build-server\/provision.sh)/' /etc/fstab; any=1; fi
|
|
[ "$any" = 1 ] && changed swap "off, fstab entry commented" || ok swap "none"
|
|
}
|
|
|
|
step_sysctl() {
|
|
local f=/etc/sysctl.d/90-igneum-build.conf tmp
|
|
tmp=$(mktemp)
|
|
cat > "$tmp" <<'EOF'
|
|
# igneum-build-1: a compile box with no swap (infra/build-server/provision.sh)
|
|
vm.swappiness = 0
|
|
vm.overcommit_memory = 0
|
|
vm.dirty_ratio = 20
|
|
vm.dirty_background_ratio = 5
|
|
vm.max_map_count = 1048576
|
|
fs.file-max = 4194304
|
|
fs.inotify.max_user_watches = 1048576
|
|
fs.inotify.max_user_instances = 8192
|
|
kernel.pid_max = 4194304
|
|
kernel.threads-max = 1048576
|
|
net.core.somaxconn = 4096
|
|
net.ipv4.tcp_fin_timeout = 15
|
|
EOF
|
|
if [ -f "$f" ] && cmp -s "$tmp" "$f"; then rm -f "$tmp"; ok sysctl "$f"; return; fi
|
|
install -m 644 "$tmp" "$f"; rm -f "$tmp"; sysctl --system >/dev/null
|
|
changed sysctl "$f applied"
|
|
}
|
|
|
|
step_limits() {
|
|
local f=/etc/security/limits.d/90-igneum-build.conf
|
|
if [ -f "$f" ]; then ok limits; return; fi
|
|
printf '%s soft nofile 1048576\n%s hard nofile 1048576\n%s soft nproc unlimited\n' "$BUILD_USER" "$BUILD_USER" "$BUILD_USER" > "$f"
|
|
changed limits "$f"
|
|
}
|
|
|
|
step_user() {
|
|
local keys
|
|
if ! id -u "$BUILD_USER" >/dev/null 2>&1; then useradd -m -s /bin/bash -G users "$BUILD_USER"; changed user "$BUILD_USER created"; else ok user "$BUILD_USER"; fi
|
|
install -d -m 700 -o "$BUILD_USER" -g "$BUILD_USER" "$BUILD_HOME/.ssh"
|
|
if [ -n "$SSH_PUBKEY" ]; then keys="$SSH_PUBKEY"; elif [ -s /root/.ssh/authorized_keys ]; then keys=$(cat /root/.ssh/authorized_keys); else die "no key for $BUILD_USER: root has no authorized_keys and SSH_PUBKEY is unset"; fi
|
|
if [ -f "$BUILD_HOME/.ssh/authorized_keys" ] && [ "$(cat "$BUILD_HOME/.ssh/authorized_keys")" = "$keys" ]; then ok authorized_keys; else
|
|
printf '%s\n' "$keys" > "$BUILD_HOME/.ssh/authorized_keys"; chmod 600 "$BUILD_HOME/.ssh/authorized_keys"; chown "$BUILD_USER:$BUILD_USER" "$BUILD_HOME/.ssh/authorized_keys"
|
|
changed authorized_keys "$(printf '%s\n' "$keys" | grep -c .) key(s) from root"
|
|
fi
|
|
}
|
|
|
|
worktree_count() { find /srv/builds -mindepth 1 -maxdepth 1 -type d -not -name '_*' | wc -l | tr -d ' '; }
|
|
|
|
# docker (7 October 2026): the glibc proof runs a shipped binary inside ubuntu:20.04 and ubuntu:22.04 on the box (tools/build-remote.sh
|
|
# --ship's proof); user build may run containers. Nothing else of the box runs in docker.
|
|
step_docker() {
|
|
command -v docker >/dev/null 2>&1 || die "docker is not installed (apt docker.io)"
|
|
systemctl is-active --quiet docker || systemctl enable --now docker >/dev/null 2>&1
|
|
if id -nG "$BUILD_USER" | tr ' ' '\n' | grep -qx docker; then ok docker "$(docker --version | cut -d, -f1), $BUILD_USER in the docker group"; else usermod -aG docker "$BUILD_USER"; changed docker "$(docker --version | cut -d, -f1), $BUILD_USER added to the docker group (new ssh sessions see it)"; fi
|
|
}
|
|
|
|
step_dirs() {
|
|
local d any=0
|
|
for d in /srv/builds /srv/builds/_locks /srv/sccache /srv/artefacts; do
|
|
if [ ! -d "$d" ]; then install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "$d"; any=1; fi
|
|
done
|
|
if [ ! -f /srv/builds/_locks/slots ] || [ "$(cat /srv/builds/_locks/slots)" != "$SLOTS" ]; then printf '%s\n' "$SLOTS" > /srv/builds/_locks/slots; chown "$BUILD_USER:$BUILD_USER" /srv/builds/_locks/slots; any=1; fi
|
|
for d in $WORKTREES; do
|
|
case "$d" in */*|.*|_*) die "worktree name '$d' is not a plain directory name" ;; esac
|
|
if [ ! -d "/srv/builds/$d" ]; then install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "/srv/builds/$d"; any=1; fi
|
|
done
|
|
[ "$any" = 1 ] && changed dirs "/srv/builds ($(worktree_count) worktree dirs), /srv/sccache, slots=$SLOTS" || ok dirs "$(worktree_count) worktree dirs, slots=$SLOTS"
|
|
}
|
|
# the lease tool (infra/build-server/lease.sh: per-core measurement leases, the quiet class, the reaper; 7 October 2026) from the
|
|
# mirror's master at /srv/builds/_bin/lease, which remote-run.sh's keeper calls; the mirror is pushed by step_mirrors' caller
|
|
step_lease_tool() {
|
|
local src="/srv/igneum.git" want have=""
|
|
install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" /srv/builds/_bin
|
|
want=$(git -C "$src" show master:infra/build-server/lease.sh 2>/dev/null) || { ok lease-tool "mirror has no master yet; run-from-mac.sh installs it on the next provision"; return; }
|
|
[ -f /srv/builds/_bin/lease ] && have=$(cat /srv/builds/_bin/lease)
|
|
if [ "$want" = "$have" ]; then ok lease-tool "/srv/builds/_bin/lease is the mirror's master copy"; return; fi
|
|
printf '%s\n' "$want" > /srv/builds/_bin/lease.new; chmod 755 /srv/builds/_bin/lease.new; chown "$BUILD_USER:$BUILD_USER" /srv/builds/_bin/lease.new
|
|
mv /srv/builds/_bin/lease.new /srv/builds/_bin/lease; changed lease-tool "/srv/builds/_bin/lease installed from the mirror's master"
|
|
}
|
|
# headless Chromium self-test: the shell the dashboard lane's node tooling downloads (playwright or puppeteer cache of the build
|
|
# user) launches once with --headless and prints its version; when no shell is downloaded yet the libraries are checked by ldd of
|
|
# nothing, so the step only says so (the apt list above carries them)
|
|
step_headless_check() {
|
|
local shell="" v
|
|
shell=$(find "/home/$BUILD_USER/.cache/ms-playwright" "/home/$BUILD_USER/.cache/puppeteer" /srv -maxdepth 6 -type f \( -name headless_shell -o -name chrome-headless-shell -o -name chrome \) 2>/dev/null | head -1)
|
|
[ -n "$shell" ] || { ok headless "no headless shell downloaded yet (the 16 libraries are installed; the lane's first capture downloads it)"; return; }
|
|
if v=$(sudo -u "$BUILD_USER" timeout 60 "$shell" --headless --no-sandbox --disable-gpu --version 2>&1 | head -1) && [ -n "$v" ]; then ok headless "$shell: $v"
|
|
else die "headless shell $shell does not launch: $v"; fi
|
|
}
|
|
|
|
step_mirrors() {
|
|
local r any=0
|
|
for r in /srv/igneum.git /srv/igneum-node.git; do
|
|
if [ ! -d "$r" ]; then install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "$r"; as_build "git init -q --bare -b master $r"; any=1; fi
|
|
done
|
|
as_build "git config --global --get safe.directory >/dev/null 2>&1 || git config --global --add safe.directory '*'"
|
|
as_build "git config --global init.defaultBranch master; git config --global gc.auto 0"
|
|
[ "$any" = 1 ] && changed mirrors "bare /srv/igneum.git and /srv/igneum-node.git (push from the Mac, see the header)" || ok mirrors
|
|
}
|
|
|
|
step_rustup() {
|
|
local cargo="$BUILD_HOME/.cargo/bin/cargo" rustup="$BUILD_HOME/.cargo/bin/rustup" any=0 t
|
|
if [ ! -x "$rustup" ]; then
|
|
as_build "curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --no-modify-path --default-toolchain $RUST_TOOLCHAIN" >/dev/null
|
|
any=1
|
|
fi
|
|
if ! as_build "$rustup toolchain list" | grep -q "^$RUST_TOOLCHAIN-"; then as_build "$rustup toolchain install $RUST_TOOLCHAIN --profile minimal" >/dev/null; any=1; fi
|
|
if [ "$(as_build "$rustup default" | cut -d- -f1)" != "$RUST_TOOLCHAIN" ]; then as_build "$rustup default $RUST_TOOLCHAIN" >/dev/null; any=1; fi
|
|
for t in x86_64-pc-windows-gnu x86_64-unknown-linux-gnu; do
|
|
as_build "$rustup target list --installed --toolchain $RUST_TOOLCHAIN" | grep -qx "$t" || { as_build "$rustup target add $t --toolchain $RUST_TOOLCHAIN" >/dev/null; any=1; }
|
|
done
|
|
as_build "$rustup component list --installed --toolchain $RUST_TOOLCHAIN" | grep -q '^clippy' || { as_build "$rustup component add clippy rustfmt --toolchain $RUST_TOOLCHAIN" >/dev/null; any=1; }
|
|
[ "$any" = 1 ] && changed rustup "$(as_build "$cargo --version"), targets: $(as_build "$rustup target list --installed" | tr '\n' ' ')" || ok rustup "$(as_build "$cargo --version"), targets: $(as_build "$rustup target list --installed" | tr '\n' ' ')"
|
|
}
|
|
|
|
step_sccache() {
|
|
local cargo="$BUILD_HOME/.cargo/bin/cargo" bin="$BUILD_HOME/.cargo/bin/sccache" cfgdir="$BUILD_HOME/.config/sccache" any=0 bytes tmp
|
|
if [ ! -x "$bin" ] || { [ -n "$SCCACHE_VERSION" ] && ! "$bin" --version | grep -q " $SCCACHE_VERSION\$"; }; then
|
|
as_build "$cargo install sccache --locked ${SCCACHE_VERSION:+--version $SCCACHE_VERSION}" >/dev/null 2>&1 || as_build "$cargo install sccache ${SCCACHE_VERSION:+--version $SCCACHE_VERSION}" >/dev/null
|
|
any=1
|
|
fi
|
|
bytes=$(( SCCACHE_GB * 1024 * 1024 * 1024 ))
|
|
install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "$cfgdir"
|
|
tmp=$(mktemp)
|
|
printf '[cache.disk]\ndir = "/srv/sccache"\nsize = %s\n' "$bytes" > "$tmp"
|
|
if ! cmp -s "$tmp" "$cfgdir/config"; then install -m 644 -o "$BUILD_USER" -g "$BUILD_USER" "$tmp" "$cfgdir/config"; any=1; fi
|
|
rm -f "$tmp"
|
|
[ "$any" = 1 ] && changed sccache "$(as_build "$bin --version"), disk cache /srv/sccache, $SCCACHE_GB GB" || ok sccache "$(as_build "$bin --version"), /srv/sccache $SCCACHE_GB GB"
|
|
}
|
|
|
|
step_cargo_config() {
|
|
# the build user's cargo defaults: sccache in front of rustc, 90 jobs (96 threads, 6 left for ssh, rsync and the
|
|
# system), lld for the native target through clang. The Windows target's compilers and flags are NOT here: they are
|
|
# set per build by tools/cross-remote.sh, the same variables as the Mac's proto-cuda/windows-node/cross-build.sh and
|
|
# the PC's jobbuild.rs, so a build's flags are visible in the script that runs it.
|
|
local f="$BUILD_HOME/.cargo/config.toml" tmp
|
|
tmp=$(mktemp)
|
|
cat > "$tmp" <<'EOF'
|
|
# igneum-build-1 (infra/build-server/provision.sh)
|
|
[build]
|
|
rustc-wrapper = "/home/build/.cargo/bin/sccache"
|
|
jobs = 90
|
|
|
|
[target.x86_64-unknown-linux-gnu]
|
|
linker = "clang"
|
|
rustflags = ["-C", "link-arg=-fuse-ld=lld"]
|
|
|
|
[net]
|
|
git-fetch-with-cli = true
|
|
EOF
|
|
if cmp -s "$tmp" "$f"; then rm -f "$tmp"; ok cargo-config "$f"; return; fi
|
|
install -m 644 -o "$BUILD_USER" -g "$BUILD_USER" "$tmp" "$f"; rm -f "$tmp"
|
|
changed cargo-config "$f"
|
|
}
|
|
|
|
step_profile() {
|
|
# sourced by tools/build-remote.sh's remote script (a non-login ssh shell reads no profile) and by login shells
|
|
local f=/etc/profile.d/igneum-build.sh tmp
|
|
tmp=$(mktemp)
|
|
cat > "$tmp" <<EOF
|
|
# igneum-build-1 (infra/build-server/provision.sh)
|
|
export PATH="/home/build/.cargo/bin:/usr/local/bin:\$PATH"
|
|
export SCCACHE_DIR=/srv/sccache
|
|
export SCCACHE_CACHE_SIZE=${SCCACHE_GB}G
|
|
export CARGO_INCREMENTAL=0
|
|
export IGNEUM_BUILD_SLOTS_DIR=/srv/builds/_locks
|
|
export IGNEUM_BUILD_ROOT=/srv/builds
|
|
export IGNEUM_RUST_TOOLCHAIN=$RUST_TOOLCHAIN
|
|
EOF
|
|
if [ -f "$f" ] && cmp -s "$tmp" "$f"; then rm -f "$tmp"; ok profile "$f"; return; fi
|
|
install -m 644 "$tmp" "$f"; rm -f "$tmp"; changed profile "$f"
|
|
}
|
|
|
|
step_node() {
|
|
local want have shasums tarball ver dir
|
|
have=$(/usr/local/bin/node --version 2>/dev/null || true)
|
|
case "$have" in v$NODE_MAJOR.*) ok node "$have"; return ;; esac
|
|
# the newest $NODE_MAJOR release from nodejs.org, checked against its SHASUMS256.txt (https, the official host)
|
|
shasums=$(curl -fsSL "https://nodejs.org/dist/latest-v$NODE_MAJOR.x/SHASUMS256.txt")
|
|
tarball=$(printf '%s\n' "$shasums" | awk '$2 ~ /linux-x64\.tar\.xz$/ { print $2 }' | head -1)
|
|
[ -n "$tarball" ] || die "no linux-x64 tarball in the Node $NODE_MAJOR SHASUMS"
|
|
ver=${tarball#node-}; ver=${ver%-linux-x64.tar.xz}
|
|
dir=/usr/local/lib/nodejs
|
|
install -d "$dir"
|
|
( cd "$dir" && curl -fsSLO "https://nodejs.org/dist/latest-v$NODE_MAJOR.x/$tarball" && printf '%s\n' "$shasums" | grep " $tarball\$" | sha256sum -c --quiet - && tar -xJf "$tarball" && rm -f "$tarball" )
|
|
ln -sfn "$dir/node-$ver-linux-x64/bin/node" /usr/local/bin/node
|
|
ln -sfn "$dir/node-$ver-linux-x64/bin/npm" /usr/local/bin/npm
|
|
ln -sfn "$dir/node-$ver-linux-x64/bin/npx" /usr/local/bin/npx
|
|
changed node "$(/usr/local/bin/node --version) from nodejs.org (sha256 checked)"
|
|
}
|
|
|
|
# zig (main, 7 October 2026): the glibc 2.36 Linux artefacts for Debian 12 seeds and HiveOS rigs come from cargo-zigbuild with zig as
|
|
# the C/C++ toolchain, as infra/cross/build-linux.sh does on the Mac (tonight's seed took 14 restarts and three minutes down on a
|
|
# glibc 2.39 native build). The release the Mac uses (0.17.0), from ziglang.org with the sha256 of the official download index.
|
|
ZIG_VERSION="${ZIG_VERSION:-0.17.0}"
|
|
ZIG_SHA256="${ZIG_SHA256:-1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026}" # zig-x86_64-linux-0.17.0.tar.xz, index.json 7 Oct 2026
|
|
step_zig() {
|
|
local have dir tar
|
|
have=$(/usr/local/bin/zig version 2>/dev/null || true)
|
|
if [ "$have" = "$ZIG_VERSION" ]; then ok zig "$have"; return; fi
|
|
dir=/usr/local/lib/zig; tar="zig-x86_64-linux-$ZIG_VERSION.tar.xz"
|
|
install -d "$dir"
|
|
( cd "$dir" && curl -fsSLO "https://ziglang.org/download/$ZIG_VERSION/$tar" && echo "$ZIG_SHA256 $tar" | sha256sum -c --quiet - && tar -xJf "$tar" && rm -f "$tar" )
|
|
ln -sfn "$dir/zig-x86_64-linux-$ZIG_VERSION/zig" /usr/local/bin/zig
|
|
changed zig "$(/usr/local/bin/zig version) from ziglang.org (sha256 checked) at /usr/local/bin/zig"
|
|
}
|
|
|
|
step_sshd() {
|
|
local f=/etc/ssh/sshd_config.d/10-igneum-build.conf tmp
|
|
tmp=$(mktemp)
|
|
cat > "$tmp" <<'EOF'
|
|
# igneum-build-1 (infra/build-server/provision.sh): keys only
|
|
PasswordAuthentication no
|
|
KbdInteractiveAuthentication no
|
|
ChallengeResponseAuthentication no
|
|
PubkeyAuthentication yes
|
|
PermitRootLogin prohibit-password
|
|
PermitEmptyPasswords no
|
|
X11Forwarding no
|
|
MaxAuthTries 4
|
|
ClientAliveInterval 60
|
|
ClientAliveCountMax 10
|
|
EOF
|
|
if [ -f "$f" ] && cmp -s "$tmp" "$f"; then rm -f "$tmp"; ok sshd "$f"; return; fi
|
|
install -m 644 "$tmp" "$f"; rm -f "$tmp"
|
|
# Hetzner's installimage may leave a cloud-init drop-in that sets PasswordAuthentication yes; the lowest-numbered file wins
|
|
if [ -f /etc/ssh/sshd_config.d/50-cloud-init.conf ] && grep -qi '^PasswordAuthentication yes' /etc/ssh/sshd_config.d/50-cloud-init.conf; then
|
|
sed -i 's/^PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config.d/50-cloud-init.conf
|
|
fi
|
|
sshd -t || die "sshd -t rejected the configuration; the drop-in $f was NOT activated"
|
|
systemctl reload ssh 2>/dev/null || systemctl reload sshd
|
|
changed sshd "key-only, root prohibit-password"
|
|
}
|
|
|
|
# the worker dashboard's feed (asked for on 6 October 2026, approved by main): Caddy serves exactly two files of /srv/workers
|
|
# over HTTPS at build.igneum.network (A record in deSEC, set by main), read-only, no directory listing, every other path 404,
|
|
# CORS for dl.igneum.network, no caching. Nothing under /srv/workers is a secret (workers.json and headline.json are written
|
|
# by the dashboard collector and remote-run.sh); the Caddyfile refuses every other file name anyway. Issuer pinned to Let's
|
|
# Encrypt: Ubuntu's Caddy 2.6.2 fails the ZeroSSL fallback (HTTP 422 caddy_legacy_user_removed, 6 Oct 2026) and would retry it for ever.
|
|
step_caddy() {
|
|
local f=/etc/caddy/Caddyfile tmp
|
|
command -v caddy >/dev/null 2>&1 || die "caddy is not installed (apt)"
|
|
install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" /srv/workers
|
|
tmp=$(mktemp)
|
|
cat > "$tmp" <<EOF
|
|
# igneum-build-1 (infra/build-server/provision.sh): the worker dashboard's two JSON files, nothing else
|
|
$WORKERS_HOST {
|
|
tls {
|
|
issuer acme
|
|
}
|
|
root * /srv/workers
|
|
header Access-Control-Allow-Origin https://dl.igneum.network
|
|
header Cache-Control "no-store"
|
|
@notjson not path /workers.json /headline.json
|
|
respond @notjson 404
|
|
file_server
|
|
}
|
|
EOF
|
|
if [ -f "$f" ] && cmp -s "$tmp" "$f"; then rm -f "$tmp"; systemctl is-active --quiet caddy || systemctl start caddy; ok caddy "$WORKERS_HOST"; return; fi
|
|
caddy validate --config "$tmp" --adapter caddyfile >/dev/null 2>&1 || { rm -f "$tmp"; die "caddy validate rejected the Caddyfile"; }
|
|
install -m 644 "$tmp" "$f"; rm -f "$tmp"
|
|
systemctl enable --quiet caddy 2>/dev/null || true
|
|
systemctl reload caddy 2>/dev/null || systemctl restart caddy
|
|
changed caddy "$WORKERS_HOST serving /srv/workers/{workers,headline}.json"
|
|
}
|
|
|
|
# CUDA headers and stubs for the GPU workers' Linux build (main, 6 October 2026, the class v4 rehearsal): proto-cuda/nvrtc/worker.cpp
|
|
# and proto-opencl/host.c need cuda.h, nvrtc.h and CL/cl.h at compile time only and dlopen libcuda, libnvrtc and libOpenCL at run
|
|
# time (infra/cross/build-workers-linux.sh links -ldl -lpthread, no nvcc anywhere in the build files). NVIDIA's apt repository for
|
|
# Ubuntu 24.04, the 12.8 minor the repo's fetch-redist.sh pins (nvrtc 12.8.93, cudart 12.8.90); no driver (the box has no GPU),
|
|
# no nvcc. CUDA_MINOR overrides the minor.
|
|
CUDA_MINOR="${CUDA_MINOR:-12-8}"
|
|
step_cuda() {
|
|
local keyring=/usr/share/keyrings/cuda-archive-keyring.gpg pkgs p need=() tmp
|
|
pkgs=("cuda-nvrtc-dev-$CUDA_MINOR" "cuda-cudart-dev-$CUDA_MINOR" "cuda-driver-dev-$CUDA_MINOR" opencl-c-headers)
|
|
for p in "${pkgs[@]}"; do dpkg -s "$p" >/dev/null 2>&1 || need+=("$p"); done
|
|
if [ "${#need[@]}" = 0 ]; then ok cuda "${pkgs[*]} (headers and stubs, no nvcc, no driver)"; return; fi
|
|
if [ ! -f "$keyring" ] && [ ! -f /etc/apt/sources.list.d/cuda-ubuntu2404-x86_64.list ]; then
|
|
tmp=$(mktemp -d)
|
|
curl -fsSL -o "$tmp/cuda-keyring.deb" https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
|
|
DEBIAN_FRONTEND=noninteractive dpkg -i "$tmp/cuda-keyring.deb" >/dev/null; rm -rf "$tmp"
|
|
fi
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
apt-get update -qq
|
|
apt-get install -y -qq --no-install-recommends "${need[@]}"
|
|
changed cuda "installed ${need[*]} (headers under /usr/local/cuda-${CUDA_MINOR/-/.}/include, libcuda stub under .../lib64/stubs)"
|
|
}
|
|
|
|
step_ufw() {
|
|
local p want=() any=0 status
|
|
status=$(ufw status verbose 2>/dev/null || true)
|
|
grep -q 'Default: deny (incoming), allow (outgoing)' <<<"$status" || { ufw --force default deny incoming >/dev/null; ufw --force default allow outgoing >/dev/null; any=1; }
|
|
want=(22 80 443)
|
|
for p in $P2P_PORTS; do want+=("$p"); done
|
|
for p in "${want[@]}"; do
|
|
grep -qE "^$p/tcp +ALLOW IN +Anywhere *$" <<<"$status" || { ufw allow "$p/tcp" >/dev/null; any=1; }
|
|
done
|
|
grep -q '^Status: active' <<<"$status" || { ufw --force enable >/dev/null; any=1; }
|
|
[ "$any" = 1 ] && changed ufw "22, 80, 443 and ${P2P_PORTS} open, everything else denied" || ok ufw "22, 80, 443 and ${P2P_PORTS}"
|
|
}
|
|
|
|
# The GitHub Actions self-hosted runner for igneum-network/igneum (asked for 6 October 2026, "what else can the box work on").
|
|
# A dedicated user `runner` (no sudo, not in the build group), the runner at /opt/actions-runner (sha256 of the tarball checked
|
|
# against the release note), its own rustup pinned to RUST_TOOLCHAIN with the two targets and clippy, Node 22 and mingw from
|
|
# the system, and sccache against the box's cache in READ-ONLY mode (`rw_mode = "READ_ONLY"`, accepted by sccache 0.18 on
|
|
# 6 October 2026): a CI job may take hits from what the agents built, never write a line into their cache, and talks to its
|
|
# own sccache server on RUNNER_SCCACHE_PORT so the build user's server never compiles as the wrong user. Registration needs
|
|
# RUNNER_TOKEN (infra/build-server/runner/register.sh fetches one with gh as igneum-labs and pipes it over stdin); without
|
|
# it the step installs everything and says what is missing. The service is GitHub's own `svc.sh install runner` unit plus a
|
|
# drop-in with Nice=10 (agents' builds through build-remote.sh win the CPU; a CI job is a check, not a release build).
|
|
# Not ephemeral: GitHub recommends `--ephemeral` for autoscaled fleets that register a fresh runner per job; one standing
|
|
# runner on a private repository keeps its registration and cleans `_work` per job (approximate: from GitHub's runner
|
|
# documentation as remembered on 6 October 2026, the docs host answered 404 to the fetch that evening).
|
|
runner_env_file() {
|
|
cat <<EOF
|
|
# $BOX_HOSTNAME (infra/build-server/provision.sh step_runner): the environment every CI job on this runner starts with
|
|
RUSTC_WRAPPER=/usr/local/bin/sccache
|
|
SCCACHE_CONF=$RUNNER_HOME/.config/sccache/config
|
|
SCCACHE_SERVER_PORT=$RUNNER_SCCACHE_PORT
|
|
CARGO_INCREMENTAL=0
|
|
CARGO_BUILD_JOBS=$RUNNER_JOBS
|
|
CARGO_NET_GIT_FETCH_WITH_CLI=true
|
|
IGNEUM_RUST_TOOLCHAIN=$RUST_TOOLCHAIN
|
|
EOF
|
|
}
|
|
|
|
step_runner() {
|
|
local any=0 tarball cargo="$RUNNER_HOME/.cargo/bin/cargo" rustup="$RUNNER_HOME/.cargo/bin/rustup" t tmp svc unit dropin
|
|
as_runner() { su - "$RUNNER_USER" -c "$*"; }
|
|
# 1. the user: own group, no sudo, no membership of build; /home/runner 750 like the other homes
|
|
if ! id -u "$RUNNER_USER" >/dev/null 2>&1; then useradd -m -s /bin/bash "$RUNNER_USER"; any=1; fi
|
|
id -nG "$RUNNER_USER" | tr ' ' '\n' | grep -qx sudo && die "the runner user must never be in sudo"
|
|
# 2. the runner itself, from the GitHub release with the published sha256
|
|
if [ ! -x "$RUNNER_DIR/run.sh" ] || ! grep -q "\"$RUNNER_VERSION\"" "$RUNNER_DIR/.runner_version" 2>/dev/null; then
|
|
[ ! -d "$RUNNER_DIR" ] || [ ! -f "$RUNNER_DIR/.runner" ] || log "runner: a configured runner is in place; the tarball is updated underneath it (the service restarts below)"
|
|
install -d -m 755 -o "$RUNNER_USER" -g "$RUNNER_USER" "$RUNNER_DIR"
|
|
tarball="actions-runner-linux-x64-$RUNNER_VERSION.tar.gz"
|
|
tmp=$(mktemp -d)
|
|
( cd "$tmp" && curl -fsSLO "https://github.com/actions/runner/releases/download/v$RUNNER_VERSION/$tarball" \
|
|
&& printf '%s %s\n' "$RUNNER_SHA256" "$tarball" | sha256sum -c --quiet - ) || { rm -rf "$tmp"; die "runner tarball download or sha256 check failed (version $RUNNER_VERSION)"; }
|
|
tar -xzf "$tmp/$tarball" -C "$RUNNER_DIR" # as root: the temp dir is root-only; ownership handed over below
|
|
chown -R "$RUNNER_USER:$RUNNER_USER" "$RUNNER_DIR"
|
|
rm -rf "$tmp"
|
|
printf '"%s"\n' "$RUNNER_VERSION" > "$RUNNER_DIR/.runner_version"; chown "$RUNNER_USER:$RUNNER_USER" "$RUNNER_DIR/.runner_version"
|
|
any=1
|
|
fi
|
|
# 3. toolchains for the runner user: rustup pinned like the box's, both targets, clippy and rustfmt
|
|
if [ ! -x "$rustup" ]; then
|
|
as_runner "curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --no-modify-path --default-toolchain $RUST_TOOLCHAIN" >/dev/null; any=1
|
|
fi
|
|
as_runner "$rustup toolchain list" | grep -q "^$RUST_TOOLCHAIN-" || { as_runner "$rustup toolchain install $RUST_TOOLCHAIN --profile minimal" >/dev/null; any=1; }
|
|
[ "$(as_runner "$rustup default" | cut -d- -f1)" = "$RUST_TOOLCHAIN" ] || { as_runner "$rustup default $RUST_TOOLCHAIN" >/dev/null; any=1; }
|
|
for t in x86_64-pc-windows-gnu x86_64-unknown-linux-gnu; do
|
|
as_runner "$rustup target list --installed --toolchain $RUST_TOOLCHAIN" | grep -qx "$t" || { as_runner "$rustup target add $t --toolchain $RUST_TOOLCHAIN" >/dev/null; any=1; }
|
|
done
|
|
as_runner "$rustup component list --installed --toolchain $RUST_TOOLCHAIN" | grep -q '^clippy' || { as_runner "$rustup component add clippy rustfmt --toolchain $RUST_TOOLCHAIN" >/dev/null; any=1; }
|
|
as_runner "git config --global --get safe.directory >/dev/null 2>&1 || git config --global --add safe.directory '*'" # the mirrors are owned by build
|
|
# 3b. the CI red watcher's record file: the workflow's `red` job appends one line per failed master or release run here
|
|
# (tools/ci/red-watch.mjs record); the poster (infra/build-server/ci-red) reads it as build
|
|
# (tools/ci/red-watch.mjs record); remote-run.sh appends the box's own red rows as build; the shared group `cired` lets
|
|
# both write one file (664 in a 2775 directory), and the poster (infra/build-server/ci-red) reads it as build
|
|
getent group cired >/dev/null || { groupadd cired; any=1; }
|
|
id -nG "$RUNNER_USER" | tr ' ' '\n' | grep -qx cired || { usermod -aG cired "$RUNNER_USER"; any=1; }
|
|
id -nG "$BUILD_USER" | tr ' ' '\n' | grep -qx cired || { usermod -aG cired "$BUILD_USER"; any=1; }
|
|
if [ ! -d /srv/ci-red ]; then install -d -o root -g cired -m 2775 /srv/ci-red; any=1; fi
|
|
[ -f /srv/ci-red/red.jsonl ] || { install -o root -g cired -m 664 /dev/null /srv/ci-red/red.jsonl; any=1; }
|
|
# 4. sccache: the build user's binary copied system-wide (the runner cannot read /home/build), a read-only view of /srv/sccache
|
|
if [ ! -x /usr/local/bin/sccache ] || ! cmp -s "$BUILD_HOME/.cargo/bin/sccache" /usr/local/bin/sccache; then
|
|
install -m 755 "$BUILD_HOME/.cargo/bin/sccache" /usr/local/bin/sccache; any=1
|
|
fi
|
|
install -d -m 755 -o "$RUNNER_USER" -g "$RUNNER_USER" "$RUNNER_HOME/.config" "$RUNNER_HOME/.config/sccache"
|
|
tmp=$(mktemp)
|
|
printf '[cache.disk]\ndir = "/srv/sccache"\nsize = %s\nrw_mode = "READ_ONLY"\n' "$(( SCCACHE_GB * 1024 * 1024 * 1024 ))" > "$tmp"
|
|
if ! cmp -s "$tmp" "$RUNNER_HOME/.config/sccache/config"; then install -m 644 -o "$RUNNER_USER" -g "$RUNNER_USER" "$tmp" "$RUNNER_HOME/.config/sccache/config"; any=1; fi
|
|
rm -f "$tmp"
|
|
# 6. registration (once; --replace re-registers under the same name after a token is given again)
|
|
if [ ! -f "$RUNNER_DIR/.runner" ]; then
|
|
if [ -z "$RUNNER_TOKEN" ]; then
|
|
log "runner: NOT registered: no RUNNER_TOKEN. From the Mac: infra/build-server/runner/register.sh (gh as igneum-labs fetches a registration token and pipes it here)"
|
|
return
|
|
fi
|
|
# the token goes to config.sh as an argument of a process owned by runner for a second; it is a one-hour registration
|
|
# token (not the runner's credential, which config.sh writes to .credentials, mode 600, owner runner), never logged here
|
|
RUNNER_TOKEN="$RUNNER_TOKEN" runuser -u "$RUNNER_USER" -- bash -c "cd '$RUNNER_DIR' && ./config.sh --unattended --replace --url '$RUNNER_REPO_URL' --token \"\$RUNNER_TOKEN\" --name '$RUNNER_NAME' --labels '$RUNNER_LABELS' --work _work" >/dev/null \
|
|
|| die "runner: config.sh failed (an expired token? register.sh fetches a fresh one)"
|
|
any=1
|
|
log "runner: registered as $RUNNER_NAME with labels self-hosted, linux, x64, $RUNNER_LABELS${RUNNER_CPUS:+, AllowedCPUs $RUNNER_CPUS}"
|
|
fi
|
|
# 7. the service: GitHub's unit (User=runner, KillMode=process) plus Nice and a restart on failure
|
|
svc="actions.runner.$(sed -n 's/.*"gitHubUrl": *"https:\/\/github.com\/\([^"]*\)".*/\1/p' "$RUNNER_DIR/.runner" | tr '/' '-').$RUNNER_NAME.service"
|
|
unit="/etc/systemd/system/$svc"
|
|
if [ ! -f "$unit" ]; then ( cd "$RUNNER_DIR" && ./svc.sh install "$RUNNER_USER" >/dev/null ) || die "runner: svc.sh install failed"; any=1; fi
|
|
# 8. the job environment, AFTER svc.sh install: its env.sh rewrites .env and .path from the installing shell (6 October 2026:
|
|
# the second provision run found them changed and restarted the service for nothing); the runner reads both at start
|
|
tmp=$(mktemp); runner_env_file > "$tmp"
|
|
if ! cmp -s "$tmp" "$RUNNER_DIR/.env"; then install -m 644 -o "$RUNNER_USER" -g "$RUNNER_USER" "$tmp" "$RUNNER_DIR/.env"; any=1; fi
|
|
rm -f "$tmp"
|
|
tmp=$(mktemp); printf '%s\n' "$RUNNER_HOME/.cargo/bin:/usr/local/bin:/usr/bin:/bin" > "$tmp"
|
|
if ! cmp -s "$tmp" "$RUNNER_DIR/.path"; then install -m 644 -o "$RUNNER_USER" -g "$RUNNER_USER" "$tmp" "$RUNNER_DIR/.path"; any=1; fi
|
|
rm -f "$tmp"
|
|
dropin="/etc/systemd/system/$svc.d/igneum.conf"
|
|
tmp=$(mktemp)
|
|
printf '# %s (infra/build-server/provision.sh step_runner)\n[Service]\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nRestart=on-failure\nRestartSec=30\n' "$BOX_HOSTNAME" > "$tmp"
|
|
[ -z "$RUNNER_CPUS" ] || printf 'AllowedCPUs=%s\n' "$RUNNER_CPUS" >> "$tmp"
|
|
install -d -m 755 "$(dirname "$dropin")"
|
|
if ! cmp -s "$tmp" "$dropin"; then install -m 644 "$tmp" "$dropin"; systemctl daemon-reload; any=1; fi
|
|
rm -f "$tmp"
|
|
systemctl enable --quiet "$svc" 2>/dev/null || true
|
|
if [ "$any" = 1 ]; then systemctl restart "$svc"; else systemctl is-active --quiet "$svc" || systemctl start "$svc"; fi
|
|
sleep 2
|
|
systemctl is-active --quiet "$svc" || die "runner: $svc is not active: journalctl -u '$svc' -n 30"
|
|
[ "$any" = 1 ] && changed runner "$svc active as $RUNNER_USER, runner $RUNNER_VERSION, $(as_runner "$cargo --version"), sccache read-only on /srv/sccache, jobs $RUNNER_JOBS" \
|
|
|| ok runner "$svc active, runner $RUNNER_VERSION, $(as_runner "$cargo --version")"
|
|
}
|
|
|
|
# cargo tools the night battery needs (infra/build-server/night): cargo-audit for the advisory check of every Cargo.lock
|
|
step_cargo_tools() {
|
|
local cargo="$BUILD_HOME/.cargo/bin/cargo" any=0
|
|
if [ ! -x "$BUILD_HOME/.cargo/bin/cargo-audit" ]; then as_build "$cargo install cargo-audit --locked" >/dev/null 2>&1 || as_build "$cargo install cargo-audit" >/dev/null; any=1; fi
|
|
# cargo-zigbuild (main, 7 October 2026): the glibc 2.36 Linux artefacts for Debian 12 seeds and HiveOS rigs (tools/build-remote.sh --ship)
|
|
if [ ! -x "$BUILD_HOME/.cargo/bin/cargo-zigbuild" ]; then as_build "$cargo install cargo-zigbuild --locked" >/dev/null 2>&1 || as_build "$cargo install cargo-zigbuild" >/dev/null; any=1; fi
|
|
[ "$any" = 1 ] && changed cargo-tools "$(as_build "$BUILD_HOME/.cargo/bin/cargo-audit --version"), $(as_build "$BUILD_HOME/.cargo/bin/cargo-zigbuild --version")" || ok cargo-tools "$(as_build "$BUILD_HOME/.cargo/bin/cargo-audit --version"), $(as_build "$BUILD_HOME/.cargo/bin/cargo-zigbuild --version")"
|
|
}
|
|
|
|
# the night battery (infra/build-server/night): the script and remote-run.sh into /srv/builds/_bin, the two units, the timer
|
|
# enabled. The files come out of the bare mirror /srv/igneum.git at NIGHT_REF (master; a branch while the work is unmerged),
|
|
# so provision.sh stays one piped file and the box runs what the repository holds. The battery re-execs itself from master's
|
|
# checkout at run time, so the copy here only has to be good enough to check out.
|
|
NIGHT_REF="${NIGHT_REF:-master}"
|
|
step_night() {
|
|
local any=0 f tmp u
|
|
install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" /srv/builds/_bin /srv/builds/_night /srv/builds/_log
|
|
as_build "git -C /srv/igneum.git cat-file -e '$NIGHT_REF:infra/build-server/night/night-battery.sh'" 2>/dev/null || { log "night: $NIGHT_REF on /srv/igneum.git has no infra/build-server/night/night-battery.sh (push the branch, or NIGHT_REF=<branch>)"; return; }
|
|
for f in infra/build-server/night/night-battery.sh infra/build-server/remote-run.sh; do
|
|
tmp=$(mktemp); as_build "git -C /srv/igneum.git show '$NIGHT_REF:$f'" > "$tmp"
|
|
if ! cmp -s "$tmp" "/srv/builds/_bin/$(basename "$f")"; then install -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "$tmp" "/srv/builds/_bin/$(basename "$f")"; any=1; fi
|
|
rm -f "$tmp"
|
|
done
|
|
for u in igneum-night-battery.service igneum-night-battery.timer; do
|
|
tmp=$(mktemp); as_build "git -C /srv/igneum.git show '$NIGHT_REF:infra/build-server/night/$u'" > "$tmp"
|
|
if ! cmp -s "$tmp" "/etc/systemd/system/$u"; then install -m 644 "$tmp" "/etc/systemd/system/$u"; any=1; fi
|
|
rm -f "$tmp"
|
|
done
|
|
[ "$any" = 1 ] && systemctl daemon-reload
|
|
systemctl is-enabled --quiet igneum-night-battery.timer 2>/dev/null || { systemctl enable --now --quiet igneum-night-battery.timer; any=1; }
|
|
systemctl is-active --quiet igneum-night-battery.timer || systemctl start igneum-night-battery.timer
|
|
[ "$any" = 1 ] && changed night "timer $(systemctl list-timers igneum-night-battery.timer --no-pager --no-legend | awk '{ print $1, $2, $3, $4 }'), files from $NIGHT_REF" \
|
|
|| ok night "next $(systemctl list-timers igneum-night-battery.timer --no-pager --no-legend | awk '{ print $1, $2, $3, $4 }')"
|
|
}
|
|
|
|
step_summary() {
|
|
log "summary:"
|
|
{
|
|
printf 'host %s, %s threads, %s RAM, root fs %s free\n' "$(hostname)" "$(nproc)" "$(awk '/MemTotal/ { printf "%d GB", $2 / 1024 / 1024 }' /proc/meminfo)" "$(df -h / | awk 'NR == 2 { print $4 }')"
|
|
printf 'raid: %s\n' "$(grep -E '^md' /proc/mdstat 2>/dev/null | tr '\n' ';' || echo none)"
|
|
printf 'rust: %s | %s | targets %s\n' "$(as_build "$BUILD_HOME/.cargo/bin/rustc --version")" "$(as_build "$BUILD_HOME/.cargo/bin/cargo --version")" "$(as_build "$BUILD_HOME/.cargo/bin/rustup target list --installed" | tr '\n' ' ')"
|
|
printf 'sccache: %s, %s\n' "$(as_build "$BUILD_HOME/.cargo/bin/sccache --version")" "$(cat "$BUILD_HOME/.config/sccache/config" | tr '\n' ' ')"
|
|
printf 'zig: %s, cargo-zigbuild %s (glibc 2.36 Linux artefacts: tools/build-remote.sh --ship, tools/workers-remote.sh)\n' "$(/usr/local/bin/zig version 2>/dev/null || echo missing)" "$(as_build "$BUILD_HOME/.cargo/bin/cargo-zigbuild --version 2>/dev/null | awk '{ print \$NF }'" || echo missing)"
|
|
printf 'mingw: %s\n' "$(x86_64-w64-mingw32-gcc-posix --version | head -1)"
|
|
printf 'clang: %s | lld: %s\n' "$(clang --version | head -1)" "$(ld.lld --version | head -1)"
|
|
printf 'node: %s | git: %s | tmux: %s\n' "$(/usr/local/bin/node --version)" "$(git --version)" "$(tmux -V)"
|
|
printf 'swap: %s\n' "$(swapon --noheadings --show 2>/dev/null | wc -l | awk '{ print ($1 == 0) ? "none" : $1 " device(s) ACTIVE" }')"
|
|
printf 'mirrors: /srv/igneum.git (%s) /srv/igneum-node.git (%s)\n' "$(as_build 'git -C /srv/igneum.git branch --list | wc -l') branches" "$(as_build 'git -C /srv/igneum-node.git branch --list | wc -l') branches"
|
|
printf 'builds: %s worktree dirs under /srv/builds, %s slot(s)\n' "$(worktree_count)" "$(cat /srv/builds/_locks/slots)"
|
|
printf 'ufw: %s\n' "$(ufw status | grep -E 'ALLOW' | awk '{ print $1 }' | tr '\n' ' ')"
|
|
printf 'caddy: %s, %s\n' "$(caddy version 2>/dev/null | cut -d' ' -f1)" "$(systemctl is-active caddy 2>/dev/null) at https://$WORKERS_HOST/headline.json"
|
|
printf 'cuda headers: %s\n' "$(ls -d /usr/local/cuda-*/include 2>/dev/null | tr '\n' ' ')$( [ -f /usr/include/CL/cl.h ] && echo '+ CL/cl.h' )"
|
|
printf 'runner: %s\n' "$( [ -f "$RUNNER_DIR/.runner" ] && printf '%s, %s, user %s' "$(systemctl list-units --type=service --no-legend 'actions.runner.*' | awk '{ print $1 ": " $4 }' | head -1)" "v$(tr -d '"' < "$RUNNER_DIR/.runner_version" 2>/dev/null)" "$RUNNER_USER" || echo "installed, NOT registered (infra/build-server/runner/register.sh)" )"
|
|
printf 'night battery: %s\n' "$(systemctl list-timers igneum-night-battery.timer --no-pager --no-legend 2>/dev/null | awk '{ print "next " $1, $2, $3, $4 }')"
|
|
printf 'ssh line: ssh -i ~/.ssh/igneum_ed25519 build@%s\n' "$(hostname -I 2>/dev/null | awk '{ print $1 }')"
|
|
} | sed 's/^/ /'
|
|
}
|
|
|
|
do_provision() {
|
|
step_os_check
|
|
step_hostname
|
|
step_apt
|
|
step_mingw_alternatives
|
|
step_no_swap
|
|
step_sysctl
|
|
step_limits
|
|
step_user
|
|
step_docker
|
|
step_dirs
|
|
step_mirrors
|
|
step_lease_tool
|
|
step_rustup
|
|
step_sccache
|
|
step_cargo_config
|
|
step_profile
|
|
step_node
|
|
step_sshd
|
|
step_caddy
|
|
step_cuda
|
|
step_ufw
|
|
step_runner
|
|
step_cargo_tools
|
|
step_zig
|
|
step_night
|
|
step_headless_check
|
|
step_summary
|
|
log "done"
|
|
}
|
|
|
|
case "$MODE" in
|
|
install) do_install ;;
|
|
provision) do_provision ;;
|
|
auto) if in_rescue; then log "rescue system detected: install mode"; do_install; else do_provision; fi ;;
|
|
*) die "MODE must be auto, install or provision" ;;
|
|
esac
|