diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md
index c04ce7cb1..4a4f1f32e 100644
--- a/docs/plans/build-server.md
+++ b/docs/plans/build-server.md
@@ -99,6 +99,16 @@ Also logged for context: the Mac's Linux cross-build with zig (`infra/cross/buil
| The 0.3.15 prover pair for the PCs needs `--features igneum-prove-host/cuda` (the PCs run SP1_PROVER=cuda) | my first proving build named no feature | built on the box from master e1b5bc9: igneum-prove-host 73,161,528 B sha256 71bc2438856bb141f6cad3d18489f708568144fad5a06a002fbadefb9ce256f9, igneum-prove-export 3,609,360 B sha256 263bf4cef70af4a13a45b2e79b8dbab373282ea4c571f624d02ddb5791935361 (52 s warm, no CUDA needed at build time); handed to the shipper |
| Let's Encrypt saw NXDOMAIN for build.igneum.network | the deSEC record was minutes old; Ubuntu's Caddy then fell back to ZeroSSL and failed with HTTP 422 for ever | issuer pinned to Let's Encrypt; the retry got the certificate |
+## 5a. The GPU workers (added 6 October 2026, 19:10 UTC, for the class v4 rehearsal)
+
+| What | Fact |
+|---|---|
+| Headers on the box | provision.sh `step_cuda`: NVIDIA's ubuntu2404 apt repository, `cuda-nvrtc-dev-12-8`, `cuda-cudart-dev-12-8`, `cuda-driver-dev-12-8` (the libcuda stub) and Ubuntu's `opencl-c-headers`; no nvcc (no build file calls it: both workers compile from C/C++ sources with the headers and dlopen libcuda, libnvrtc and libOpenCL at run time), no driver (no GPU here). 12.8 is the minor proto-cuda/nvrtc/fetch-redist.sh pins |
+| Command | `tools/workers-remote.sh [--out
]` from any igneum worktree: proto-cuda and proto-opencl through the mirror and overlay, clang++ 18 with `-static-libstdc++ -static-libgcc`, both workers, sha256 and the glibc ceiling printed, artefacts in `/infra/cross/out-workers-box/` |
+| Proof (master worker.cpp at 3b2c840) | igneum-worker-cuda 1,613,992 B sha256 6db8a9ad295a9f7598a8b5500f1c271fbfdb9a8df90848b95ff340664007031e; igneum-worker-opencl 124,904 B sha256 0dea75bb8d2d54721ee44b55a3ba486241d3c6053e72a133f8f5e40d2355cbbd; 3 s on the box |
+| Consequence: glibc ceiling 2.38 | runs on the fleet (Ubuntu 22.04 containers are glibc 2.35: NO, 2.38 > 2.35; Ubuntu 24.04 hosts yes). The Mac's zig build (`infra/cross/build-workers-linux.sh`, glibc 2.36) is the one for Debian 12 and HiveOS; the fleet agent must check its boxes' glibc before swapping the worker. Fix if needed: zig on the box (open row in section 6) or `clang -target x86_64-linux-gnu.2.35` via zig; both a day's work, not done |
+| Runner fix found on the way | a command string carrying `set -e` leaked into remote-run.sh through `eval` and killed the runner before its RESULT line (reported as rc 101); the runner now evaluates the command in a subshell |
+
## 6. What the box does not do yet
| Gap | Why it matters | Next step |
diff --git a/infra/build-server/provision.sh b/infra/build-server/provision.sh
index 8c4d35aa3..ee789e9fc 100755
--- a/infra/build-server/provision.sh
+++ b/infra/build-server/provision.sh
@@ -373,6 +373,28 @@ EOF
changed caddy "$WORKERS_HOST serving /srv/workers/{workers,headline}.json"
}
+# CUDA headers and stubs for the GPU workers' Linux build (main, 6 October 2026, the class v4 rehearsal): proto-cuda/nvrtc/worker.cpp
+# and proto-opencl/host.c need cuda.h, nvrtc.h and CL/cl.h at compile time only and dlopen libcuda, libnvrtc and libOpenCL at run
+# time (infra/cross/build-workers-linux.sh links -ldl -lpthread, no nvcc anywhere in the build files). NVIDIA's apt repository for
+# Ubuntu 24.04, the 12.8 minor the repo's fetch-redist.sh pins (nvrtc 12.8.93, cudart 12.8.90); no driver (the box has no GPU),
+# no nvcc. CUDA_MINOR overrides the minor.
+CUDA_MINOR="${CUDA_MINOR:-12-8}"
+step_cuda() {
+ local keyring=/usr/share/keyrings/cuda-archive-keyring.gpg pkgs p need=() tmp
+ pkgs=("cuda-nvrtc-dev-$CUDA_MINOR" "cuda-cudart-dev-$CUDA_MINOR" "cuda-driver-dev-$CUDA_MINOR" opencl-c-headers)
+ for p in "${pkgs[@]}"; do dpkg -s "$p" >/dev/null 2>&1 || need+=("$p"); done
+ if [ "${#need[@]}" = 0 ]; then ok cuda "${pkgs[*]} (headers and stubs, no nvcc, no driver)"; return; fi
+ if [ ! -f "$keyring" ] && [ ! -f /etc/apt/sources.list.d/cuda-ubuntu2404-x86_64.list ]; then
+ tmp=$(mktemp -d)
+ curl -fsSL -o "$tmp/cuda-keyring.deb" https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb
+ DEBIAN_FRONTEND=noninteractive dpkg -i "$tmp/cuda-keyring.deb" >/dev/null; rm -rf "$tmp"
+ fi
+ export DEBIAN_FRONTEND=noninteractive
+ apt-get update -qq
+ apt-get install -y -qq --no-install-recommends "${need[@]}"
+ changed cuda "installed ${need[*]} (headers under /usr/local/cuda-${CUDA_MINOR/-/.}/include, libcuda stub under .../lib64/stubs)"
+}
+
step_ufw() {
local p want=() any=0 status
status=$(ufw status verbose 2>/dev/null || true)
@@ -401,6 +423,7 @@ step_summary() {
printf 'builds: %s worktree dirs under /srv/builds, %s slot(s)\n' "$(worktree_count)" "$(cat /srv/builds/_locks/slots)"
printf 'ufw: %s\n' "$(ufw status | grep -E 'ALLOW' | awk '{ print $1 }' | tr '\n' ' ')"
printf 'caddy: %s, %s\n' "$(caddy version 2>/dev/null | cut -d' ' -f1)" "$(systemctl is-active caddy 2>/dev/null) at https://$WORKERS_HOST/headline.json"
+ printf 'cuda headers: %s\n' "$(ls -d /usr/local/cuda-*/include 2>/dev/null | tr '\n' ' ')$( [ -f /usr/include/CL/cl.h ] && echo '+ CL/cl.h' )"
printf 'ssh line: ssh -i ~/.ssh/igneum_ed25519 build@%s\n' "$(hostname -I 2>/dev/null | awk '{ print $1 }')"
} | sed 's/^/ /'
}
@@ -423,6 +446,7 @@ do_provision() {
step_node
step_sshd
step_caddy
+ step_cuda
step_ufw
step_summary
log "done"
diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh
index 2a600fcdf..bb626a2f6 100755
--- a/infra/build-server/remote-run.sh
+++ b/infra/build-server/remote-run.sh
@@ -176,7 +176,9 @@ sccache --start-server >/dev/null 2>&1 || true
stat_field() { sccache --show-stats 2>/dev/null | awk -v key="$1" 'index($0, key) == 1 { print $NF; exit }'; }
exec_before=$(stat_field "Compile requests executed"); hits_before=$(stat_field "Cache hits "); misses_before=$(stat_field "Cache misses ")
t1=$(date +%s)
-eval "$BR_CMD"
+# in a subshell: a command string that carries `set -e` or `exit` ends only the subshell, never this runner (6 October 2026,
+# workers-remote.sh: both workers built, then the leaked set -e killed the runner before its RESULT line, reported as rc 101)
+( eval "$BR_CMD" )
rc=$?
t2=$(date +%s); secs=$(( t2 - t1 ))
exec_after=$(stat_field "Compile requests executed"); hits_after=$(stat_field "Cache hits "); misses_after=$(stat_field "Cache misses ")
diff --git a/tools/workers-remote.sh b/tools/workers-remote.sh
new file mode 100755
index 000000000..b97bdc052
--- /dev/null
+++ b/tools/workers-remote.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2034 # the BS_* context variables are read by lib.sh
+# Build the two Linux GPU workers (igneum-worker-cuda from proto-cuda/nvrtc/worker.cpp, igneum-worker-opencl from
+# proto-opencl/host.c) on igneum-build-1 instead of the Mac's zig cross-build (infra/cross/build-workers-linux.sh). Same sources
+# and defines as that script; the box's clang++ and the CUDA 12.8 headers that provision.sh installs (no nvcc: both workers
+# dlopen libcuda, libnvrtc and libOpenCL at run time). libstdc++ and libgcc are linked statically so a fleet box with an older
+# libstdc++ (Ubuntu 22.04) runs them; the glibc ceiling of each binary is printed (a native Ubuntu 24.04 build; the HiveOS
+# package keeps the Mac's zig build at glibc 2.36 until zig is on the box).
+#
+# tools/workers-remote.sh [--out ] from any worktree of the igneum repo; artefacts in /infra/cross/out-workers-box/
+#
+# Sources travel as HEAD through the mirror plus the overlay of proto-cuda and proto-opencl (lib.sh); the build takes a remote
+# slot like every other build. Asked for by main on 6 October 2026 for the class v4 rehearsal (the fleet agent's generator-4 worker).
+set -euo pipefail
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck disable=SC2034
+BS_TOOL=workers-remote
+# shellcheck source=../infra/build-server/lib.sh
+. "$HERE/../infra/build-server/lib.sh"
+OUT=""; while [ $# -gt 0 ]; do case "$1" in --out) OUT="$2"; shift 2 ;; *) bs_die "unknown argument $1" ;; esac; done
+bs_host
+# the context by hand (bs_context wants a Cargo.toml): the igneum worktree root is the repo; lib.sh reads these
+BS_TOP=$(git -C "$HERE" rev-parse --show-toplevel); BS_WT_ROOT="$BS_TOP"; BS_KIND=repo; BS_MIRROR="$BS_MIRROR_REPO"; BS_TOP_REL=.
+BS_WT=$(basename "$BS_WT_ROOT"); BS_CRATE_REL=proto-cuda; BS_REMOTE_WT="$BS_ROOT_REMOTE/$BS_WT"; BS_REMOTE_CRATE="$BS_REMOTE_WT"
+BS_BRANCH=$(git -C "$BS_TOP" branch --show-current 2>/dev/null || true); BS_SHA=$(git -C "$BS_TOP" rev-parse HEAD); [ -n "$BS_BRANCH" ] || BS_BRANCH="detached-$(git -C "$BS_TOP" rev-parse --short HEAD)"
+BS_LOCAL_DIRS="proto-cuda proto-opencl"; BS_VENDOR_REPOS=""
+[ -n "$OUT" ] || OUT="$BS_WT_ROOT/infra/cross/out-workers-box"
+bs_log "workers from $BS_WT at $BS_SHA ($BS_BRANCH) -> $BS_HOST:$BS_REMOTE_WT (proto-cuda, proto-opencl)"
+bs_sync_sources
+PLACEHOLDER=proto-cuda/packs/igneum-devnet-v4-epoch0 # the OpenCL worker's compile-time pack, as build-workers-linux.sh
+cmd='. /etc/profile.d/igneum-build.sh
+CUDA=$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1); [ -n "$CUDA" ] || { echo "no CUDA headers under /usr/local/cuda-12.*: provision.sh step_cuda"; exit 2; }
+[ -f /usr/include/CL/cl.h ] || { echo "no /usr/include/CL/cl.h: apt opencl-c-headers"; exit 2; }
+mkdir -p out-workers-box
+echo "workers-remote: $(clang++ --version | head -1); CUDA headers $CUDA/include; $(git rev-parse --short HEAD)"
+clang++ -std=c++17 -O2 -Wall -Wextra -I proto-cuda/nvrtc -I "$CUDA/include" -static-libstdc++ -static-libgcc -o out-workers-box/igneum-worker-cuda proto-cuda/nvrtc/worker.cpp -ldl -lpthread || exit 1
+clang -std=gnu99 -D_GNU_SOURCE -O2 -Wall -Wextra -Wno-format-truncation -DIGNEUM_CL_DYNAMIC -DCL_TARGET_OPENCL_VERSION=120 -I /usr/include -I '"$PLACEHOLDER"' -DIGNEUM_KERNEL_PATH='"'"'"kernel_bound.cl"'"'"' -static-libgcc -o out-workers-box/igneum-worker-opencl proto-opencl/host.c -ldl -lm -lpthread || exit 1
+for b in igneum-worker-cuda igneum-worker-opencl; do printf "workers-remote: %s glibc ceiling %s, needs %s\n" "$b" "$(objdump -T out-workers-box/$b | grep -oE "GLIBC_[0-9.]+" | sort -V | tail -1)" "$(readelf -d out-workers-box/$b | grep -oE "\[lib[^]]+\]" | tr -d "[]" | tr "\n" " ")"; done'
+BR_KIND=other BR_COMMAND="clang++ worker.cpp; clang host.c" BR_TARGET=x86_64-unknown-linux-gnu BR_ARTEFACTS="out-workers-box/igneum-worker-cuda out-workers-box/igneum-worker-opencl"; export BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS
+t0=$(date +%s); set +e; bs_remote_run "$BS_REMOTE_WT" "$BS_WT/proto-cuda workers" "$cmd" 2>&1 | tee "/tmp/workers-remote-$$.log"; rc=${PIPESTATUS[0]}; set -e
+grep -E '^workers-remote:' "/tmp/workers-remote-$$.log" | sed 's/^/ /' >&2; rm -f "/tmp/workers-remote-$$.log"
+[ "$rc" = 0 ] || bs_die "the worker build failed (rc $rc)"
+bs_log "built in $(bs_fmt_secs $(( $(date +%s) - t0 )))"
+mkdir -p "$OUT"
+for b in igneum-worker-cuda igneum-worker-opencl; do
+ bs_rsync -p "$BS_HOST:$BS_REMOTE_WT/out-workers-box/$b" "$OUT/$b" || bs_die "no $b on the box"
+ bs_log "artefact $OUT/$b: $(bs_size "$OUT/$b") bytes, sha256 $(bs_sha256 "$OUT/$b"), $(file -b "$OUT/$b" | cut -c1-50)"
+done
+{ printf 'igneum workers, linux x86_64, built on igneum-build-1 with clang++ (static libstdc++) on %s from %s (%s); run-time libraries dlopen-ed\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$BS_SHA" "$BS_BRANCH"; } > "$OUT/version.txt"
+bs_wt_unlock