From 7089aa161fc73ff697eb61682e72895c03d1bb27 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 23:37:31 +0000 Subject: [PATCH] Build server: zig and cargo-zigbuild on the box; build-remote.sh --ship builds glibc 2.36 Linux artefacts for seeds and HiveOS; the glibc ceiling check Main's order of 7 October 2026 after a seed took 14 restarts and three minutes down on a glibc 2.39 binary. provision.sh: step_zig (zig 0.17.0 from ziglang.org, sha256 from the official download index) and cargo-zigbuild 0.23.4 in the cargo tools. tools/build-remote.sh --ship [--glibc 2.36]: cargo zigbuild --target x86_64-unknown-linux-gnu.2.36 with zig as the C/C++ toolchain (the Mac's infra/cross/build-linux.sh recipe), artefacts from the target-triple dir, each checked by tools/ci/glibc-ceiling-check.sh (need at most the ceiling; self-test fires on 2.38 against 2.36, passes 2.34 and 2.36; --symbols reads a saved objdump -T text so CI needs no ELF tools). tools/workers-remote.sh builds the two GPU workers with zig at 2.36 by default (GLIBC=native for clang). Proof on the box: fork 3bfe346f igneumd needs GLIBC_2.34 (47,023,120 B, sha256 345dfb95...), igneum-miner GLIBC_2.34 (9,248,808 B, d09dc27b...), 3 min 17 s cold through zig; the workers at 2.36. Rule: anything that ships to a seed or a HiveOS rig is built with --ship; a plain build (glibc 2.39) is for the box and Ubuntu 24.04 hosts only. Co-Authored-By: Claude Fable 5.1 --- docs/plans/build-server.md | 2 +- infra/build-server/provision.sh | 22 +++++++++++++++++- tools/build-remote.sh | 33 ++++++++++++++++++++++----- tools/ci/glibc-ceiling-check.sh | 40 +++++++++++++++++++++++++++++++++ tools/workers-remote.sh | 11 +++++---- 5 files changed, 96 insertions(+), 12 deletions(-) create mode 100755 tools/ci/glibc-ceiling-check.sh diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 572c2920..1f0ee9f9 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -124,7 +124,7 @@ Self-test: `tools/build-remote.sh --self-test-repro [--full]` from a fork worktr | Gap | Why it matters | Next step | |---|---|---| -| No zig / cargo-zigbuild | the devnet seed (Debian 12, glibc 2.36) takes the Mac's zig build; a native box build links glibc 2.39, which Debian 13 seeds accept and HiveOS (Ubuntu 18/20 base) does not | install zig 0.17 + cargo-zigbuild in provision.sh, add `--target x86_64-unknown-linux-gnu.2.36` mode to build-remote.sh | +| zig / cargo-zigbuild: DONE 7 Oct 2026 (main's order after a seed took 14 restarts and three minutes down on a glibc 2.39 binary) | provision.sh `step_zig` (zig 0.17.0 from ziglang.org, sha256 of the official index) and cargo-zigbuild 0.23.4 in `step_cargo_tools`; `tools/build-remote.sh --ship [--glibc 2.36]` runs `cargo zigbuild --target x86_64-unknown-linux-gnu.2.36`, fetches from the target-triple dir and runs `tools/ci/glibc-ceiling-check.sh` (need at most 2.36) on every artefact; `tools/workers-remote.sh` builds with `zig cc/c++ -target x86_64-linux-gnu.2.36` by default (`GLIBC=native` for clang). Rule: anything that ships to a seed or a HiveOS rig is built with `--ship`; a plain build is glibc 2.39 for the box and Ubuntu 24.04 hosts only. Proof (fork 3bfe346f, cold through zig, 3 min 17 s): igneumd 47,023,120 B sha256 345dfb95... needs GLIBC_2.34; igneum-miner 9,248,808 B sha256 d09dc27b... needs GLIBC_2.34; the check's self-test fires on 2.38 against 2.36 and passes 2.34 and 2.36 | the Mac's infra/cross/build-linux.sh is the same recipe and can retire once two seed releases shipped from the box | | No macOS target | agents who run nodes on the Mac still build there | out of scope (needs the macOS SDK on Linux); the fleet or the box's own Devnet 2 seed takes the test-network runs instead | | No CI runner | DONE 6 October 2026, 19:19Z (section 7): the runner `igneum-build-1` is online under user `runner`, never build; the workflow change is proposed in docs/plans/ci-self-hosted.md | main flips `IGNEUM_CI_RUNNER=box` after the shipper's cut | | Byte identity with the Mac's exes | different C/C++ toolchain (Homebrew mingw vs Ubuntu GCC 13) and embedded source paths | not a goal; the box is identical with itself build to build, cross-remote.sh reports sha256 and the DLL list per exe | diff --git a/infra/build-server/provision.sh b/infra/build-server/provision.sh index 945b128d..bfa06b91 100755 --- a/infra/build-server/provision.sh +++ b/infra/build-server/provision.sh @@ -332,6 +332,22 @@ step_node() { changed node "$(/usr/local/bin/node --version) from nodejs.org (sha256 checked)" } +# zig (main, 7 October 2026): the glibc 2.36 Linux artefacts for Debian 12 seeds and HiveOS rigs come from cargo-zigbuild with zig as +# the C/C++ toolchain, as infra/cross/build-linux.sh does on the Mac (tonight's seed took 14 restarts and three minutes down on a +# glibc 2.39 native build). The release the Mac uses (0.17.0), from ziglang.org with the sha256 of the official download index. +ZIG_VERSION="${ZIG_VERSION:-0.17.0}" +ZIG_SHA256="${ZIG_SHA256:-1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026}" # zig-x86_64-linux-0.17.0.tar.xz, index.json 7 Oct 2026 +step_zig() { + local have dir tar + have=$(/usr/local/bin/zig version 2>/dev/null || true) + if [ "$have" = "$ZIG_VERSION" ]; then ok zig "$have"; return; fi + dir=/usr/local/lib/zig; tar="zig-x86_64-linux-$ZIG_VERSION.tar.xz" + install -d "$dir" + ( cd "$dir" && curl -fsSLO "https://ziglang.org/download/$ZIG_VERSION/$tar" && echo "$ZIG_SHA256 $tar" | sha256sum -c --quiet - && tar -xJf "$tar" && rm -f "$tar" ) + ln -sfn "$dir/zig-x86_64-linux-$ZIG_VERSION/zig" /usr/local/bin/zig + changed zig "$(/usr/local/bin/zig version) from ziglang.org (sha256 checked) at /usr/local/bin/zig" +} + step_sshd() { local f=/etc/ssh/sshd_config.d/10-igneum-build.conf tmp tmp=$(mktemp) @@ -543,7 +559,9 @@ step_runner() { step_cargo_tools() { local cargo="$BUILD_HOME/.cargo/bin/cargo" any=0 if [ ! -x "$BUILD_HOME/.cargo/bin/cargo-audit" ]; then as_build "$cargo install cargo-audit --locked" >/dev/null 2>&1 || as_build "$cargo install cargo-audit" >/dev/null; any=1; fi - [ "$any" = 1 ] && changed cargo-tools "$(as_build "$BUILD_HOME/.cargo/bin/cargo-audit --version")" || ok cargo-tools "$(as_build "$BUILD_HOME/.cargo/bin/cargo-audit --version")" + # cargo-zigbuild (main, 7 October 2026): the glibc 2.36 Linux artefacts for Debian 12 seeds and HiveOS rigs (tools/build-remote.sh --ship) + if [ ! -x "$BUILD_HOME/.cargo/bin/cargo-zigbuild" ]; then as_build "$cargo install cargo-zigbuild --locked" >/dev/null 2>&1 || as_build "$cargo install cargo-zigbuild" >/dev/null; any=1; fi + [ "$any" = 1 ] && changed cargo-tools "$(as_build "$BUILD_HOME/.cargo/bin/cargo-audit --version"), $(as_build "$BUILD_HOME/.cargo/bin/cargo-zigbuild --version")" || ok cargo-tools "$(as_build "$BUILD_HOME/.cargo/bin/cargo-audit --version"), $(as_build "$BUILD_HOME/.cargo/bin/cargo-zigbuild --version")" } # the night battery (infra/build-server/night): the script and remote-run.sh into /srv/builds/_bin, the two units, the timer @@ -579,6 +597,7 @@ step_summary() { printf 'raid: %s\n' "$(grep -E '^md' /proc/mdstat 2>/dev/null | tr '\n' ';' || echo none)" printf 'rust: %s | %s | targets %s\n' "$(as_build "$BUILD_HOME/.cargo/bin/rustc --version")" "$(as_build "$BUILD_HOME/.cargo/bin/cargo --version")" "$(as_build "$BUILD_HOME/.cargo/bin/rustup target list --installed" | tr '\n' ' ')" printf 'sccache: %s, %s\n' "$(as_build "$BUILD_HOME/.cargo/bin/sccache --version")" "$(cat "$BUILD_HOME/.config/sccache/config" | tr '\n' ' ')" + printf 'zig: %s, cargo-zigbuild %s (glibc 2.36 Linux artefacts: tools/build-remote.sh --ship, tools/workers-remote.sh)\n' "$(/usr/local/bin/zig version 2>/dev/null || echo missing)" "$(as_build "$BUILD_HOME/.cargo/bin/cargo-zigbuild --version 2>/dev/null | awk '{ print \$NF }'" || echo missing)" printf 'mingw: %s\n' "$(x86_64-w64-mingw32-gcc-posix --version | head -1)" printf 'clang: %s | lld: %s\n' "$(clang --version | head -1)" "$(ld.lld --version | head -1)" printf 'node: %s | git: %s | tmux: %s\n' "$(/usr/local/bin/node --version)" "$(git --version)" "$(tmux -V)" @@ -616,6 +635,7 @@ do_provision() { step_ufw step_runner step_cargo_tools + step_zig step_night step_summary log "done" diff --git a/tools/build-remote.sh b/tools/build-remote.sh index 30cffa0a..8f22dd94 100755 --- a/tools/build-remote.sh +++ b/tools/build-remote.sh @@ -12,6 +12,15 @@ # tools/build-remote.sh --jobs 48 -- check # tools/build-remote.sh --target-dir target-exp -- build --release another persistent target dir on the box # tools/build-remote.sh --no-fetch -- clippy --all-targets nothing comes back (tests, check, clippy) +# tools/build-remote.sh --ship [--glibc 2.36] anything that SHIPS to a seed (Debian 12) or a HiveOS +# rig: cargo zigbuild for x86_64-unknown-linux-gnu. +# (zig as the C/C++ toolchain, as the Mac's +# infra/cross/build-linux.sh), artefacts from +# target/x86_64-unknown-linux-gnu/release, each checked by +# tools/ci/glibc-ceiling-check.sh (need at most ). +# A plain build is native glibc 2.39: the box, the fleet's +# Ubuntu 24.04 hosts, never a seed or a rig (7 Oct 2026: +# a seed took 14 restarts on a 2.39 binary). # tools/build-remote.sh --self-test-repro [--full] from a fork worktree: igneum-miner built twice a minute # apart without sccache into one target dir must give one # sha256, and a per-run target path must not (prost's @@ -48,7 +57,7 @@ BS_TOOL=build-remote . "$HERE/../infra/build-server/lib.sh" # JOBS empty = the box decides: 90 alone, 45 beside another slot holder (remote-run.sh, main's ruling 6 Oct 2026) -JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0 +JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0; SHIP=0; GLIBC="${GLIBC:-2.36}" while [ $# -gt 0 ]; do case "$1" in --jobs) JOBS="$2"; shift 2 ;; @@ -57,6 +66,8 @@ while [ $# -gt 0 ]; do --target-dir) TARGET_DIR="$2"; shift 2 ;; --no-fetch) FETCH=0; shift ;; --self-test-repro) SELFTEST=1; shift ;; + --ship) SHIP=1; shift ;; + --glibc) GLIBC="$2"; shift 2 ;; --full) FULL=1; shift ;; --) shift; CARGO_ARGS=("$@"); break ;; -h|--help) sed -n '2,32p' "$0"; exit 0 ;; @@ -103,21 +114,29 @@ fi exit 0 fi +# --ship: the zig path and the target triple dir for the artefacts +SHIP_TARGET=x86_64-unknown-linux-gnu +if [ "$SHIP" = 1 ]; then TARGET_SUB="$SHIP_TARGET/release"; else TARGET_SUB="release"; fi # defaults per crate case "$BS_KIND:$BS_CRATE_REL" in node:*) [ -n "${CARGO_ARGS[*]:-}" ] || CARGO_ARGS=(build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow) - [ -n "$ARTEFACTS" ] || ARTEFACTS="$TARGET_DIR/release/igneumd $TARGET_DIR/release/igneum-miner" ;; + [ -n "$ARTEFACTS" ] || ARTEFACTS="$TARGET_DIR/$TARGET_SUB/igneumd $TARGET_DIR/$TARGET_SUB/igneum-miner" ;; repo:app/igneum-app) [ -n "${CARGO_ARGS[*]:-}" ] || CARGO_ARGS=(build --release) - [ -n "$ARTEFACTS" ] || ARTEFACTS="$TARGET_DIR/release/igneum-app $TARGET_DIR/release/igneum-ota-sign $TARGET_DIR/release/igneum-prove-verify" ;; + [ -n "$ARTEFACTS" ] || ARTEFACTS="$TARGET_DIR/$TARGET_SUB/igneum-app $TARGET_DIR/$TARGET_SUB/igneum-ota-sign $TARGET_DIR/$TARGET_SUB/igneum-prove-verify" ;; repo:proving/igneum-prove) [ -n "${CARGO_ARGS[*]:-}" ] || CARGO_ARGS=(build --release) - [ -n "$ARTEFACTS" ] || ARTEFACTS="$TARGET_DIR/release/igneum-prove-host $TARGET_DIR/release/igneum-prove-export" ;; + [ -n "$ARTEFACTS" ] || ARTEFACTS="$TARGET_DIR/$TARGET_SUB/igneum-prove-host $TARGET_DIR/$TARGET_SUB/igneum-prove-export" ;; *) [ -n "${CARGO_ARGS[*]:-}" ] || CARGO_ARGS=(build --release) ;; esac case "${CARGO_ARGS[0]}" in build) ;; *) [ -n "${ARTEFACTS_SET:-}" ] || { FETCH=0; ARTEFACTS=""; } ;; esac # test, check, clippy: nothing to fetch +if [ "$SHIP" = 1 ]; then + [ "${CARGO_ARGS[0]}" = build ] || bs_die "--ship is for cargo build" + CARGO_ARGS=(zigbuild "${CARGO_ARGS[@]:1}" --target "$SHIP_TARGET.$GLIBC") + BR_TARGET_SHIP="$SHIP_TARGET.$GLIBC" +fi [ -n "$OUT" ] || OUT="$BS_CRATE/target-remote" bs_log "$BS_KIND crate $BS_WT/$BS_CRATE_REL at $BS_SHA ($BS_BRANCH) -> $BS_HOST:$BS_REMOTE_CRATE; cargo ${CARGO_ARGS[*]} -j ${JOBS:-auto}; target dir $TARGET_DIR" @@ -130,12 +149,12 @@ bs_log "sources in place after $(( $(date +%s) - t_sync0 )) s (every changed fil # rerun-if-changed and is never run again by cargo (release-0.3.11 plan: `cargo clean -p kaspa-build-info` first); so when # the commit the box builds differs from the last one built in this target dir, that one crate is cleaned (a relink, seconds) pre="" -if [ "$BS_KIND" = node ] && [ "${CARGO_ARGS[0]}" = build ]; then +if [ "$BS_KIND" = node ] && { [ "${CARGO_ARGS[0]}" = build ] || [ "${CARGO_ARGS[0]}" = zigbuild ]; }; then pre="[ \"\$(cat '.build-remote-sha-$TARGET_DIR' 2>/dev/null)\" = '$BS_SHA' ] || CARGO_TARGET_DIR='$TARGET_DIR' cargo clean -q --release -p kaspa-build-info 2>/dev/null; " fi cmd="$(bs_repro_env)${pre}CARGO_TARGET_DIR='$TARGET_DIR' cargo $(printf '%q ' "${CARGO_ARGS[@]}")${JOBS:+-j $JOBS} 2>&1 | tee -a '$BS_REMOTE_WT/.build-remote.log'; rc=\${PIPESTATUS[0]}; [ \$rc = 0 ] && echo '$BS_SHA' > '.build-remote-sha-$TARGET_DIR'; ( exit \$rc )" # a subshell exit: the runner reads \$? and still prints its RESULT line label="$BS_WT/$BS_CRATE_REL cargo ${CARGO_ARGS[*]}" -BR_KIND=$(bs_kind build-remote "${CARGO_ARGS[0]}"); BR_COMMAND="cargo ${CARGO_ARGS[*]}"; BR_TARGET=x86_64-unknown-linux-gnu +BR_KIND=$(bs_kind build-remote "$( [ "${CARGO_ARGS[0]}" = zigbuild ] && echo build || echo "${CARGO_ARGS[0]}")"); BR_COMMAND="cargo ${CARGO_ARGS[*]}"; BR_TARGET="${BR_TARGET_SHIP:-x86_64-unknown-linux-gnu}" for ((i = 0; i < ${#CARGO_ARGS[@]}; i++)); do [ "${CARGO_ARGS[$i]}" = --target ] && BR_TARGET="${CARGO_ARGS[$((i + 1))]:-}"; done BR_ARTEFACTS=""; [ "$FETCH" = 1 ] && BR_ARTEFACTS="$ARTEFACTS" export BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS @@ -162,6 +181,8 @@ if [ "$FETCH" = 1 ] && [ -n "$ARTEFACTS" ]; then bs_log "artefact $dest: $(bs_size "$dest") bytes, sha256 $(bs_sha256 "$dest"), $(file -b "$dest" | cut -c1-60)" # the commit-string gate (rule of 6 October 2026): a node binary without its commit in its strings fails the run case "$BS_KIND:$(basename "$dest")" in node:igneumd) "$HERE/ci/commit-string-check.sh" "$dest" "$BS_SHA" || bs_die "commit-string gate failed for $a" ;; esac # only kaspad depends on kaspa-build-info + # the glibc ceiling of anything that ships (main, 7 Oct 2026): a seed or a rig refuses a binary needing more than 2.36 + if [ "$SHIP" = 1 ]; then "$HERE/ci/glibc-ceiling-check.sh" "$dest" "$GLIBC" || bs_die "glibc ceiling gate failed for $a"; fi done fi bs_wt_unlock diff --git a/tools/ci/glibc-ceiling-check.sh b/tools/ci/glibc-ceiling-check.sh new file mode 100755 index 00000000..e163ef13 --- /dev/null +++ b/tools/ci/glibc-ceiling-check.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# The glibc ceiling of a shipped Linux binary (main, 7 October 2026: a seed took 14 restarts and three minutes down on a binary +# built natively on Ubuntu 24.04, glibc 2.39, while Debian 12 seeds have 2.36 and HiveOS rigs less). Rule: anything that ships to +# a seed or a rig needs at most GLIBC_2.36; tools/build-remote.sh --ship and tools/workers-remote.sh build with zig for that and run +# this on every artefact they fetch. Reads the versioned symbol needs (`objdump -T`, or `nm -D` where objdump is absent) and compares +# the highest GLIBC_x.y with the ceiling. +# +# tools/ci/glibc-ceiling-check.sh [ceiling, default 2.36] exit 0 when the need is at or under the ceiling +# tools/ci/glibc-ceiling-check.sh --symbols [ceiling] the same from a saved `objdump -T` text (the self-test, CI without ELF tools) +# tools/ci/glibc-ceiling-check.sh --self-test fires on a 2.38 need against 2.36, passes 2.34 against 2.36 and 2.36 against 2.36 +set -euo pipefail +ceiling_of() { grep -oE 'GLIBC_[0-9]+\.[0-9]+' | sed 's/GLIBC_//' | sort -t. -k1,1n -k2,2n | tail -1; } +le() { [ "$(printf '%s\n%s\n' "$1" "$2" | sort -t. -k1,1n -k2,2n | tail -1)" = "$2" ]; } # $1 <= $2 as glibc versions +judge() { #