diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 9ecf02a6..65e5320a 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -142,6 +142,7 @@ so a revoked key never stops the observer; it only makes it follow the mirror ag | Gap | Why it matters | Next step | |---|---|---| | zig / cargo-zigbuild: DONE 7 Oct 2026 (main's order after a seed took 14 restarts and three minutes down on a glibc 2.39 binary) | provision.sh `step_zig` (zig 0.17.0 from ziglang.org, sha256 of the official index) and cargo-zigbuild 0.23.4 in `step_cargo_tools`; `tools/build-remote.sh --ship [--glibc 2.36]` runs `cargo zigbuild --target x86_64-unknown-linux-gnu.2.36`, fetches from the target-triple dir and runs `tools/ci/glibc-ceiling-check.sh` (need at most 2.36) on every artefact; `tools/workers-remote.sh` builds with `zig cc/c++ -target x86_64-linux-gnu.2.36` by default (`GLIBC=native` for clang). Rule: anything that ships to a seed or a HiveOS rig is built with `--ship`; a plain build is glibc 2.39 for the box and Ubuntu 24.04 hosts only. Proof (fork 3bfe346f, cold through zig, 3 min 17 s): igneumd 47,023,120 B sha256 345dfb95... needs GLIBC_2.34; igneum-miner 9,248,808 B sha256 d09dc27b... needs GLIBC_2.34; the workers through zig at 2.36 (6 s): igneum-worker-cuda 6,759,272 B sha256 690c8e91... and igneum-worker-opencl 307,264 B sha256 329fb6a9..., each needing GLIBC_2.34 and libc only (the first zig attempt died on __isoc23_strtol because `-I /usr/include` for CL/cl.h put the host's glibc 2.39 headers before zig's; the OpenCL headers are reached through a CL-only symlink dir now); the check's self-test fires on 2.38 against 2.36 and passes 2.34 and 2.36 | the Mac's infra/cross/build-linux.sh is the same recipe and can retire once two seed releases shipped from the box | +| Ceilings per class (main, 7 Oct 2026 00:2x UTC, after RunPod's Ubuntu 22.04 canaries, glibc 2.35, refused the GLIBC_2.38 box binaries, and HiveOS turned out Ubuntu 20.04 based, glibc 2.31) | one table in `tools/ci/glibc-ceiling-check.sh` (`--class`, `--ceiling-of`): hive and rig 2.31, seed and linux 2.35, native unchecked; `tools/build-remote.sh --ship hive|rig|seed|linux` (default seed) and `tools/workers-remote.sh --class` (default rig) build with zig at the class's glibc and check against it. Proof on the box: fork 3bfe346f at class hive: igneumd 47,023,760 B sha256 2a07dbf1... needs GLIBC_2.30, igneum-miner 9,249,024 B sha256 c113c... needs GLIBC_2.30 (4 min 13 s cold, zig recompiles the C++ for the new glibc); workers at class rig: igneum-worker-cuda 6,761,120 B sha256 db8ed18a..., igneum-worker-opencl 309,096 B sha256 3e18a4f8..., each needing GLIBC_2.17. Run inside docker on the box: ubuntu:20.04 (ldd 2.31) and ubuntu:22.04 (ldd 2.35) both print `igneumd 2.1.0`, the miner's `usage: igneum-miner mine ...` line, `igneum-worker-cuda 1.0 (4 October 2026)` and the OpenCL worker's own `FAIL: libOpenCL.so.1 is not installed (the GPU driver provides it ...)`, which is the binary running in a GPU-less container, not glibc refusing it | the 0.3.17 HiveOS package is the class-hive build (shipper told); docker.io is on the box for this proof (user build in the docker group) | | No macOS target | agents who run nodes on the Mac still build there | out of scope (needs the macOS SDK on Linux); the fleet or the box's own Devnet 2 seed takes the test-network runs instead | | CI runner: DONE by the box-work agent (actions.runner.igneum-network-igneum.igneum-build-1.service) | | | | Byte identity with the Mac's exes | different C/C++ toolchain (Homebrew mingw vs Ubuntu GCC 13) and embedded source paths | not a goal; the box is identical with itself build to build, cross-remote.sh reports sha256 and the DLL list per exe | diff --git a/infra/build-server/provision.sh b/infra/build-server/provision.sh index eb010bd0..6fdeb90e 100755 --- a/infra/build-server/provision.sh +++ b/infra/build-server/provision.sh @@ -139,7 +139,7 @@ step_os_check() { APT_PACKAGES=( build-essential clang lld llvm libclang-dev pkg-config libssl-dev cmake protobuf-compiler libprotobuf-dev gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-x86-64-dev mingw-w64-tools - git tmux curl ca-certificates xz-utils zstd unzip rsync jq python3 ufw htop file caddy + git tmux curl ca-certificates xz-utils zstd unzip rsync jq python3 ufw htop file caddy docker.io # the GitHub Actions runner's .NET runtime needs libicu (bin/installdependencies.sh would install it); the two Python # simulators (sim/finality_v2.py, sim/difficulty/sim.py) need numpy, which ci.yml pip-installs on GitHub's runners libicu74 python3-numpy @@ -218,6 +218,14 @@ step_user() { worktree_count() { find /srv/builds -mindepth 1 -maxdepth 1 -type d -not -name '_*' | wc -l | tr -d ' '; } +# docker (7 October 2026): the glibc proof runs a shipped binary inside ubuntu:20.04 and ubuntu:22.04 on the box (tools/build-remote.sh +# --ship's proof); user build may run containers. Nothing else of the box runs in docker. +step_docker() { + command -v docker >/dev/null 2>&1 || die "docker is not installed (apt docker.io)" + systemctl is-active --quiet docker || systemctl enable --now docker >/dev/null 2>&1 + if id -nG "$BUILD_USER" | tr ' ' '\n' | grep -qx docker; then ok docker "$(docker --version | cut -d, -f1), $BUILD_USER in the docker group"; else usermod -aG docker "$BUILD_USER"; changed docker "$(docker --version | cut -d, -f1), $BUILD_USER added to the docker group (new ssh sessions see it)"; fi +} + step_dirs() { local d any=0 for d in /srv/builds /srv/builds/_locks /srv/sccache /srv/artefacts; do @@ -622,6 +630,7 @@ do_provision() { step_sysctl step_limits step_user + step_docker step_dirs step_mirrors step_rustup diff --git a/tools/build-remote.sh b/tools/build-remote.sh index 8f22dd94..2eea1bf9 100755 --- a/tools/build-remote.sh +++ b/tools/build-remote.sh @@ -12,8 +12,11 @@ # tools/build-remote.sh --jobs 48 -- check # tools/build-remote.sh --target-dir target-exp -- build --release another persistent target dir on the box # tools/build-remote.sh --no-fetch -- clippy --all-targets nothing comes back (tests, check, clippy) -# tools/build-remote.sh --ship [--glibc 2.36] anything that SHIPS to a seed (Debian 12) or a HiveOS -# rig: cargo zigbuild for x86_64-unknown-linux-gnu. +# tools/build-remote.sh --ship [hive|rig|seed|linux] [--glibc X.Y] anything that SHIPS: cargo zigbuild for +# x86_64-unknown-linux-gnu. where the glibc comes +# from the class (hive/rig 2.31: HiveOS is Ubuntu 20.04 +# based; seed/linux 2.35: Debian 12 and Ubuntu 22.04; +# default seed; --glibc overrides) # (zig as the C/C++ toolchain, as the Mac's # infra/cross/build-linux.sh), artefacts from # target/x86_64-unknown-linux-gnu/release, each checked by @@ -57,7 +60,7 @@ BS_TOOL=build-remote . "$HERE/../infra/build-server/lib.sh" # JOBS empty = the box decides: 90 alone, 45 beside another slot holder (remote-run.sh, main's ruling 6 Oct 2026) -JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0; SHIP=0; GLIBC="${GLIBC:-2.36}" +JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0; SHIP=0; SHIP_CLASS="${SHIP_CLASS:-seed}"; GLIBC="${GLIBC:-}" while [ $# -gt 0 ]; do case "$1" in --jobs) JOBS="$2"; shift 2 ;; @@ -66,7 +69,7 @@ while [ $# -gt 0 ]; do --target-dir) TARGET_DIR="$2"; shift 2 ;; --no-fetch) FETCH=0; shift ;; --self-test-repro) SELFTEST=1; shift ;; - --ship) SHIP=1; shift ;; + --ship) SHIP=1; case "${2:-}" in hive|rig|seed|linux|native) SHIP_CLASS="$2"; shift 2 ;; *) shift ;; esac ;; --glibc) GLIBC="$2"; shift 2 ;; --full) FULL=1; shift ;; --) shift; CARGO_ARGS=("$@"); break ;; @@ -116,7 +119,11 @@ fi # --ship: the zig path and the target triple dir for the artefacts SHIP_TARGET=x86_64-unknown-linux-gnu -if [ "$SHIP" = 1 ]; then TARGET_SUB="$SHIP_TARGET/release"; else TARGET_SUB="release"; fi +if [ "$SHIP" = 1 ]; then + [ -n "$GLIBC" ] || GLIBC=$("$HERE/ci/glibc-ceiling-check.sh" --ceiling-of "$SHIP_CLASS") || bs_die "unknown ship class $SHIP_CLASS" + [ "$GLIBC" != native ] || bs_die "--ship native is a plain build: drop --ship" + TARGET_SUB="$SHIP_TARGET/release" +else TARGET_SUB="release"; fi # defaults per crate case "$BS_KIND:$BS_CRATE_REL" in node:*) diff --git a/tools/ci/glibc-ceiling-check.sh b/tools/ci/glibc-ceiling-check.sh index e163ef13..979d315a 100755 --- a/tools/ci/glibc-ceiling-check.sh +++ b/tools/ci/glibc-ceiling-check.sh @@ -5,10 +5,19 @@ # this on every artefact they fetch. Reads the versioned symbol needs (`objdump -T`, or `nm -D` where objdump is absent) and compares # the highest GLIBC_x.y with the ceiling. # +# Classes (main, 7 October 2026, after RunPod's Ubuntu 22.04 canaries refused GLIBC_2.38 binaries and a 2.36 HiveOS package would +# not run on a real rig): the ceiling is per artefact class, in this one table, read with --class: +# hive, rig 2.31 HiveOS images are Ubuntu 20.04 based; the HiveOS package and anything for a rig +# seed, linux 2.35 Debian 12 seeds (2.36) and Ubuntu 22.04 hosts and containers (2.35); generic Linux +# native none the box and Ubuntu 24.04 hosts; a native build, not checked +# # tools/ci/glibc-ceiling-check.sh [ceiling, default 2.36] exit 0 when the need is at or under the ceiling +# tools/ci/glibc-ceiling-check.sh --class the ceiling from the class table above +# tools/ci/glibc-ceiling-check.sh --ceiling-of print the class's ceiling (the build tools read it here) # tools/ci/glibc-ceiling-check.sh --symbols [ceiling] the same from a saved `objdump -T` text (the self-test, CI without ELF tools) # tools/ci/glibc-ceiling-check.sh --self-test fires on a 2.38 need against 2.36, passes 2.34 against 2.36 and 2.36 against 2.36 set -euo pipefail +class_ceiling() { case "$1" in hive|rig) echo 2.31 ;; seed|linux) echo 2.35 ;; native) echo native ;; *) echo "glibc-ceiling: unknown class '$1' (hive, rig, seed, linux, native)" >&2; return 2 ;; esac; } ceiling_of() { grep -oE 'GLIBC_[0-9]+\.[0-9]+' | sed 's/GLIBC_//' | sort -t. -k1,1n -k2,2n | tail -1; } le() { [ "$(printf '%s\n%s\n' "$1" "$2" | sort -t. -k1,1n -k2,2n | tail -1)" = "$2" ]; } # $1 <= $2 as glibc versions judge() { #