From 9508c939c8fb873a2d1ee8d3bfefee203f645dd0 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:11:19 +0000 Subject: [PATCH] Build server: CUDA 12.8 headers on the box and tools/workers-remote.sh (the GPU workers' Linux build for the class v4 rehearsal) provision.sh step_cuda: NVIDIA's ubuntu2404 apt repository, cuda-nvrtc-dev-12-8, cuda-cudart-dev-12-8, cuda-driver-dev-12-8 (the libcuda stub) and opencl-c-headers; no nvcc (no build file calls it), no driver. tools/workers-remote.sh builds igneum-worker-cuda and igneum-worker-opencl on the box from proto-cuda and proto-opencl with clang++ (static libstdc++), prints sha256 and the glibc ceiling (2.38: fine for Ubuntu 24.04 hosts, not for 22.04 containers or HiveOS, where the Mac's zig build stays). Proof: igneum-worker-cuda 1,613,992 B sha256 6db8a9ad..., igneum-worker-opencl 124,904 B sha256 0dea75bb... remote-run.sh evaluates the command string in a subshell (a leaked set -e killed the runner after a successful build). Co-Authored-By: Claude Fable 5.1 --- docs/plans/build-server.md | 10 +++++++ infra/build-server/provision.sh | 24 +++++++++++++++ infra/build-server/remote-run.sh | 4 ++- tools/workers-remote.sh | 50 ++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100755 tools/workers-remote.sh diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index c04ce7cb..4a4f1f32 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -99,6 +99,16 @@ Also logged for context: the Mac's Linux cross-build with zig (`infra/cross/buil | The 0.3.15 prover pair for the PCs needs `--features igneum-prove-host/cuda` (the PCs run SP1_PROVER=cuda) | my first proving build named no feature | built on the box from master e1b5bc9: igneum-prove-host 73,161,528 B sha256 71bc2438856bb141f6cad3d18489f708568144fad5a06a002fbadefb9ce256f9, igneum-prove-export 3,609,360 B sha256 263bf4cef70af4a13a45b2e79b8dbab373282ea4c571f624d02ddb5791935361 (52 s warm, no CUDA needed at build time); handed to the shipper | | Let's Encrypt saw NXDOMAIN for build.igneum.network | the deSEC record was minutes old; Ubuntu's Caddy then fell back to ZeroSSL and failed with HTTP 422 for ever | issuer pinned to Let's Encrypt; the retry got the certificate | +## 5a. The GPU workers (added 6 October 2026, 19:10 UTC, for the class v4 rehearsal) + +| What | Fact | +|---|---| +| Headers on the box | provision.sh `step_cuda`: NVIDIA's ubuntu2404 apt repository, `cuda-nvrtc-dev-12-8`, `cuda-cudart-dev-12-8`, `cuda-driver-dev-12-8` (the libcuda stub) and Ubuntu's `opencl-c-headers`; no nvcc (no build file calls it: both workers compile from C/C++ sources with the headers and dlopen libcuda, libnvrtc and libOpenCL at run time), no driver (no GPU here). 12.8 is the minor proto-cuda/nvrtc/fetch-redist.sh pins | +| Command | `tools/workers-remote.sh [--out ]` from any igneum worktree: proto-cuda and proto-opencl through the mirror and overlay, clang++ 18 with `-static-libstdc++ -static-libgcc`, both workers, sha256 and the glibc ceiling printed, artefacts in `/infra/cross/out-workers-box/` | +| Proof (master worker.cpp at 3b2c840) | igneum-worker-cuda 1,613,992 B sha256 6db8a9ad295a9f7598a8b5500f1c271fbfdb9a8df90848b95ff340664007031e; igneum-worker-opencl 124,904 B sha256 0dea75bb8d2d54721ee44b55a3ba486241d3c6053e72a133f8f5e40d2355cbbd; 3 s on the box | +| Consequence: glibc ceiling 2.38 | runs on the fleet (Ubuntu 22.04 containers are glibc 2.35: NO, 2.38 > 2.35; Ubuntu 24.04 hosts yes). The Mac's zig build (`infra/cross/build-workers-linux.sh`, glibc 2.36) is the one for Debian 12 and HiveOS; the fleet agent must check its boxes' glibc before swapping the worker. Fix if needed: zig on the box (open row in section 6) or `clang -target x86_64-linux-gnu.2.35` via zig; both a day's work, not done | +| Runner fix found on the way | a command string carrying `set -e` leaked into remote-run.sh through `eval` and killed the runner before its RESULT line (reported as rc 101); the runner now evaluates the command in a subshell | + ## 6. What the box does not do yet | Gap | Why it matters | Next step | diff --git a/infra/build-server/provision.sh b/infra/build-server/provision.sh index 8c4d35aa..ee789e9f 100755 --- a/infra/build-server/provision.sh +++ b/infra/build-server/provision.sh @@ -373,6 +373,28 @@ EOF changed caddy "$WORKERS_HOST serving /srv/workers/{workers,headline}.json" } +# CUDA headers and stubs for the GPU workers' Linux build (main, 6 October 2026, the class v4 rehearsal): proto-cuda/nvrtc/worker.cpp +# and proto-opencl/host.c need cuda.h, nvrtc.h and CL/cl.h at compile time only and dlopen libcuda, libnvrtc and libOpenCL at run +# time (infra/cross/build-workers-linux.sh links -ldl -lpthread, no nvcc anywhere in the build files). NVIDIA's apt repository for +# Ubuntu 24.04, the 12.8 minor the repo's fetch-redist.sh pins (nvrtc 12.8.93, cudart 12.8.90); no driver (the box has no GPU), +# no nvcc. CUDA_MINOR overrides the minor. +CUDA_MINOR="${CUDA_MINOR:-12-8}" +step_cuda() { + local keyring=/usr/share/keyrings/cuda-archive-keyring.gpg pkgs p need=() tmp + pkgs=("cuda-nvrtc-dev-$CUDA_MINOR" "cuda-cudart-dev-$CUDA_MINOR" "cuda-driver-dev-$CUDA_MINOR" opencl-c-headers) + for p in "${pkgs[@]}"; do dpkg -s "$p" >/dev/null 2>&1 || need+=("$p"); done + if [ "${#need[@]}" = 0 ]; then ok cuda "${pkgs[*]} (headers and stubs, no nvcc, no driver)"; return; fi + if [ ! -f "$keyring" ] && [ ! -f /etc/apt/sources.list.d/cuda-ubuntu2404-x86_64.list ]; then + tmp=$(mktemp -d) + curl -fsSL -o "$tmp/cuda-keyring.deb" https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb + DEBIAN_FRONTEND=noninteractive dpkg -i "$tmp/cuda-keyring.deb" >/dev/null; rm -rf "$tmp" + fi + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq --no-install-recommends "${need[@]}" + changed cuda "installed ${need[*]} (headers under /usr/local/cuda-${CUDA_MINOR/-/.}/include, libcuda stub under .../lib64/stubs)" +} + step_ufw() { local p want=() any=0 status status=$(ufw status verbose 2>/dev/null || true) @@ -401,6 +423,7 @@ step_summary() { printf 'builds: %s worktree dirs under /srv/builds, %s slot(s)\n' "$(worktree_count)" "$(cat /srv/builds/_locks/slots)" printf 'ufw: %s\n' "$(ufw status | grep -E 'ALLOW' | awk '{ print $1 }' | tr '\n' ' ')" printf 'caddy: %s, %s\n' "$(caddy version 2>/dev/null | cut -d' ' -f1)" "$(systemctl is-active caddy 2>/dev/null) at https://$WORKERS_HOST/headline.json" + printf 'cuda headers: %s\n' "$(ls -d /usr/local/cuda-*/include 2>/dev/null | tr '\n' ' ')$( [ -f /usr/include/CL/cl.h ] && echo '+ CL/cl.h' )" printf 'ssh line: ssh -i ~/.ssh/igneum_ed25519 build@%s\n' "$(hostname -I 2>/dev/null | awk '{ print $1 }')" } | sed 's/^/ /' } @@ -423,6 +446,7 @@ do_provision() { step_node step_sshd step_caddy + step_cuda step_ufw step_summary log "done" diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh index 2a600fcd..bb626a2f 100755 --- a/infra/build-server/remote-run.sh +++ b/infra/build-server/remote-run.sh @@ -176,7 +176,9 @@ sccache --start-server >/dev/null 2>&1 || true stat_field() { sccache --show-stats 2>/dev/null | awk -v key="$1" 'index($0, key) == 1 { print $NF; exit }'; } exec_before=$(stat_field "Compile requests executed"); hits_before=$(stat_field "Cache hits "); misses_before=$(stat_field "Cache misses ") t1=$(date +%s) -eval "$BR_CMD" +# in a subshell: a command string that carries `set -e` or `exit` ends only the subshell, never this runner (6 October 2026, +# workers-remote.sh: both workers built, then the leaked set -e killed the runner before its RESULT line, reported as rc 101) +( eval "$BR_CMD" ) rc=$? t2=$(date +%s); secs=$(( t2 - t1 )) exec_after=$(stat_field "Compile requests executed"); hits_after=$(stat_field "Cache hits "); misses_after=$(stat_field "Cache misses ") diff --git a/tools/workers-remote.sh b/tools/workers-remote.sh new file mode 100755 index 00000000..b97bdc05 --- /dev/null +++ b/tools/workers-remote.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# shellcheck disable=SC2034 # the BS_* context variables are read by lib.sh +# Build the two Linux GPU workers (igneum-worker-cuda from proto-cuda/nvrtc/worker.cpp, igneum-worker-opencl from +# proto-opencl/host.c) on igneum-build-1 instead of the Mac's zig cross-build (infra/cross/build-workers-linux.sh). Same sources +# and defines as that script; the box's clang++ and the CUDA 12.8 headers that provision.sh installs (no nvcc: both workers +# dlopen libcuda, libnvrtc and libOpenCL at run time). libstdc++ and libgcc are linked statically so a fleet box with an older +# libstdc++ (Ubuntu 22.04) runs them; the glibc ceiling of each binary is printed (a native Ubuntu 24.04 build; the HiveOS +# package keeps the Mac's zig build at glibc 2.36 until zig is on the box). +# +# tools/workers-remote.sh [--out ] from any worktree of the igneum repo; artefacts in /infra/cross/out-workers-box/ +# +# Sources travel as HEAD through the mirror plus the overlay of proto-cuda and proto-opencl (lib.sh); the build takes a remote +# slot like every other build. Asked for by main on 6 October 2026 for the class v4 rehearsal (the fleet agent's generator-4 worker). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC2034 +BS_TOOL=workers-remote +# shellcheck source=../infra/build-server/lib.sh +. "$HERE/../infra/build-server/lib.sh" +OUT=""; while [ $# -gt 0 ]; do case "$1" in --out) OUT="$2"; shift 2 ;; *) bs_die "unknown argument $1" ;; esac; done +bs_host +# the context by hand (bs_context wants a Cargo.toml): the igneum worktree root is the repo; lib.sh reads these +BS_TOP=$(git -C "$HERE" rev-parse --show-toplevel); BS_WT_ROOT="$BS_TOP"; BS_KIND=repo; BS_MIRROR="$BS_MIRROR_REPO"; BS_TOP_REL=. +BS_WT=$(basename "$BS_WT_ROOT"); BS_CRATE_REL=proto-cuda; BS_REMOTE_WT="$BS_ROOT_REMOTE/$BS_WT"; BS_REMOTE_CRATE="$BS_REMOTE_WT" +BS_BRANCH=$(git -C "$BS_TOP" branch --show-current 2>/dev/null || true); BS_SHA=$(git -C "$BS_TOP" rev-parse HEAD); [ -n "$BS_BRANCH" ] || BS_BRANCH="detached-$(git -C "$BS_TOP" rev-parse --short HEAD)" +BS_LOCAL_DIRS="proto-cuda proto-opencl"; BS_VENDOR_REPOS="" +[ -n "$OUT" ] || OUT="$BS_WT_ROOT/infra/cross/out-workers-box" +bs_log "workers from $BS_WT at $BS_SHA ($BS_BRANCH) -> $BS_HOST:$BS_REMOTE_WT (proto-cuda, proto-opencl)" +bs_sync_sources +PLACEHOLDER=proto-cuda/packs/igneum-devnet-v4-epoch0 # the OpenCL worker's compile-time pack, as build-workers-linux.sh +cmd='. /etc/profile.d/igneum-build.sh +CUDA=$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1); [ -n "$CUDA" ] || { echo "no CUDA headers under /usr/local/cuda-12.*: provision.sh step_cuda"; exit 2; } +[ -f /usr/include/CL/cl.h ] || { echo "no /usr/include/CL/cl.h: apt opencl-c-headers"; exit 2; } +mkdir -p out-workers-box +echo "workers-remote: $(clang++ --version | head -1); CUDA headers $CUDA/include; $(git rev-parse --short HEAD)" +clang++ -std=c++17 -O2 -Wall -Wextra -I proto-cuda/nvrtc -I "$CUDA/include" -static-libstdc++ -static-libgcc -o out-workers-box/igneum-worker-cuda proto-cuda/nvrtc/worker.cpp -ldl -lpthread || exit 1 +clang -std=gnu99 -D_GNU_SOURCE -O2 -Wall -Wextra -Wno-format-truncation -DIGNEUM_CL_DYNAMIC -DCL_TARGET_OPENCL_VERSION=120 -I /usr/include -I '"$PLACEHOLDER"' -DIGNEUM_KERNEL_PATH='"'"'"kernel_bound.cl"'"'"' -static-libgcc -o out-workers-box/igneum-worker-opencl proto-opencl/host.c -ldl -lm -lpthread || exit 1 +for b in igneum-worker-cuda igneum-worker-opencl; do printf "workers-remote: %s glibc ceiling %s, needs %s\n" "$b" "$(objdump -T out-workers-box/$b | grep -oE "GLIBC_[0-9.]+" | sort -V | tail -1)" "$(readelf -d out-workers-box/$b | grep -oE "\[lib[^]]+\]" | tr -d "[]" | tr "\n" " ")"; done' +BR_KIND=other BR_COMMAND="clang++ worker.cpp; clang host.c" BR_TARGET=x86_64-unknown-linux-gnu BR_ARTEFACTS="out-workers-box/igneum-worker-cuda out-workers-box/igneum-worker-opencl"; export BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS +t0=$(date +%s); set +e; bs_remote_run "$BS_REMOTE_WT" "$BS_WT/proto-cuda workers" "$cmd" 2>&1 | tee "/tmp/workers-remote-$$.log"; rc=${PIPESTATUS[0]}; set -e +grep -E '^workers-remote:' "/tmp/workers-remote-$$.log" | sed 's/^/ /' >&2; rm -f "/tmp/workers-remote-$$.log" +[ "$rc" = 0 ] || bs_die "the worker build failed (rc $rc)" +bs_log "built in $(bs_fmt_secs $(( $(date +%s) - t0 )))" +mkdir -p "$OUT" +for b in igneum-worker-cuda igneum-worker-opencl; do + bs_rsync -p "$BS_HOST:$BS_REMOTE_WT/out-workers-box/$b" "$OUT/$b" || bs_die "no $b on the box" + bs_log "artefact $OUT/$b: $(bs_size "$OUT/$b") bytes, sha256 $(bs_sha256 "$OUT/$b"), $(file -b "$OUT/$b" | cut -c1-50)" +done +{ printf 'igneum workers, linux x86_64, built on igneum-build-1 with clang++ (static libstdc++) on %s from %s (%s); run-time libraries dlopen-ed\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$BS_SHA" "$BS_BRANCH"; } > "$OUT/version.txt" +bs_wt_unlock