From 2ea1128342227d95c014d27c3b5338a789cf147b Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:36:39 +0000 Subject: [PATCH] Build server: a path dependency inside a vendor repository is synced as a whole repository (the shipper's proving build) proving/igneum-prove depends on vendor/igneum-node-exec/igneum/evm-types, a member of the fork's workspace that inherits from the fork's root manifest; syncing that one directory left cargo without a workspace root on the box. lib.sh now groups path dependencies by git top level: a repository under vendor/ is pushed to its mirror (a fork worktree to /srv/igneum-node.git, a repository of its own to /srv/.git, created on first use), checked out whole at /srv/builds//vendor/ and overlaid whole; run-from-mac.sh wires every vendor repository the Cargo.toml files reach. libprotobuf-dev added (sp1-prover-types imports google/protobuf/empty.proto). build-remote.sh no longer fails on a default artefact the caller's own -p selection did not build. Proof on the box: igneum-prove-host 71,943,192 B, sha256 e9213e3a6c979512d7859f6d8e848105bab53f4355e99fb0d30fb4a72c2d5714, 1 min 05 s warm. Plan: gotcha rows for the case, the protoc miss and one lost ssh session (collector cleared by test). Co-Authored-By: Claude Fable 5.1 --- docs/plans/build-server.md | 2 + infra/build-server/lib.sh | 62 +++++++++++++++++++++++------- infra/build-server/provision.sh | 4 +- infra/build-server/run-from-mac.sh | 10 +++++ tools/build-remote.sh | 8 +++- 5 files changed, 71 insertions(+), 15 deletions(-) diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 1b1cc697d..5951f79a8 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -91,6 +91,8 @@ Also logged for context: the Mac's Linux cross-build with zig (`infra/cross/buil | An all-identical overlay listed only directories | the first run's touch pipeline got an empty file list and failed | files only are counted and re-stamped (lib.sh bs_overlay_dir) | | bash 3.2 on the Mac treats an empty array as unbound under `set -u` | run-from-mac.sh died on `PASS[*]` | a string instead of an array | | `grep -q` plus `pipefail` turned a strings hit into a miss | the commit-string gate failed a stamped igneumd on its first use (SIGPIPE on `strings`) | `grep -c` | +| A path dependency inside a vendor repo (the shipper's proving build, 6 Oct 2026) | proving/igneum-prove depends on vendor/igneum-node-exec/igneum/evm-types, a MEMBER of the fork's workspace (it inherits `thiserror` from the fork's root manifest); the first design synced that one directory, so cargo found no workspace root on the box ("failed to load manifest for workspace member"), and the shipper cross-built the Linux prove-host on the Mac with cargo-zigbuild meanwhile | lib.sh groups path dependencies by git top level: one under vendor/ is a whole repository, pushed to its mirror (a fork worktree such as igneum-node-exec goes to /srv/igneum-node.git, which already held its branch; a repository of its own gets /srv/.git, created on first use) and checked out whole at /srv/builds//vendor/; run-from-mac.sh wires every vendor repo the Cargo.toml files reach (today only igneum-node-exec). The detector's first version tested "under BS_TOP" before "own repository" and missed it, since vendor/ lies under the igneum top level on disk. Then `libprotobuf-dev` was missing (sp1-prover-types's build script imports google/protobuf/empty.proto); added to provision.sh. Proof: `cd proving/igneum-prove && tools/build-remote.sh -- build --release -p igneum-prove-host`: igneum-prove-host 71,943,192 B, sha256 e9213e3a6c979512d7859f6d8e848105bab53f4355e99fb0d30fb4a72c2d5714, ELF x86-64, 1 min 05 s warm (the cold run compiled 605 crates in 55 s before protoc stopped it). A Mac worktree has no vendor/ of its own, so a worktree that builds proving needs `git -C vendor/igneum-node worktree add /vendor/igneum-node-exec execution-layer` first, as the fork worktrees do | +| One remote build lost its ssh session after 75 s (18:30:50 UTC, the first full proving build) | the remote bash died with it (slot line left behind, no JSONL line); no OOM, no reboot, the retry a minute later passed | the dashboard collector's one-pass unit finished within a second of the drop, so it was tested: a 90 s remote session through the same ControlMaster path survived two collector passes triggered by hand; the collector only reads (/proc, lock files, `flock -n`, `sccache --show-stats`, `kill(pid, 0)`). One event, no cause in the journal, the retry passed. A build that must survive a dropped connection would need the remote command under setsid with the Mac reconnecting to wait; not done, open if it happens again | | Let's Encrypt saw NXDOMAIN for build.igneum.network | the deSEC record was minutes old; Ubuntu's Caddy then fell back to ZeroSSL and failed with HTTP 422 for ever | issuer pinned to Let's Encrypt; the retry got the certificate | ## 6. What the box does not do yet diff --git a/infra/build-server/lib.sh b/infra/build-server/lib.sh index b16fb76c0..ce61a96df 100755 --- a/infra/build-server/lib.sh +++ b/infra/build-server/lib.sh @@ -83,22 +83,49 @@ bs_context() { BS_REMOTE_CRATE="$BS_REMOTE_WT/$BS_CRATE_REL" # every local (path) package of the crate's dependency graph, as directories relative to the worktree root; the ones inside # BS_TOP are covered by the git checkout plus the overlay of BS_TOP itself (node kind) or synced one by one (repo kind) + # A path dependency that lives inside another git repository under vendor/ (6 October 2026, the shipper's proving build: + # proving/igneum-prove -> vendor/igneum-node-exec/igneum/evm-types, a MEMBER of the fork's workspace that inherits + # `thiserror` from the fork's root manifest) is not a directory to copy: it needs its whole repository on the box, checked + # out at the Mac's commit, as BS_TOP gets. Such repositories are listed in BS_VENDOR_REPOS (relative to the worktree root), + # synced whole by bs_sync_sources, and dropped from BS_LOCAL_DIRS. + BS_VENDOR_REPOS="" BS_LOCAL_DIRS=$(cd "$BS_CRATE" && "${CARGO_HOME:-$HOME/.cargo}/bin/cargo" metadata --format-version 1 2>/dev/null | python3 -c ' -import json, os, sys +import json, os, subprocess, sys d = json.load(sys.stdin); root = sys.argv[1]; top = sys.argv[2]; kind = sys.argv[3] -dirs = set() +def toplevel(m): + try: return subprocess.run(["git", "-C", m, "rev-parse", "--show-toplevel"], capture_output=True, text=True, check=True).stdout.strip() + except subprocess.CalledProcessError: return None +dirs, repos = set(), set() for p in d["packages"]: if p["source"] is not None: continue m = os.path.dirname(p["manifest_path"]) - if kind == "node" and (m == top or m.startswith(top + "/")): dirs.add(top); continue + # a repository under vendor/ first: on disk it also lies under the igneum top level (vendor/ is ignored, not a + # submodule), so the path test alone would take it for a directory of BS_TOP (the first run of this detector did) + t = toplevel(m) + if t and t != top and t.startswith(root + "/vendor/"): + repos.add(t); continue + if m == top or m.startswith(top + "/"): + dirs.add(top if kind == "node" else m); continue dirs.add(m) -out = [] -for m in sorted(dirs): +def rel(m): r = os.path.relpath(m, root) if r.startswith(".."): sys.exit("path dependency %s is outside the worktree root %s" % (m, root)) - out.append(r) -print("\n".join(out))' "$BS_WT_ROOT" "$BS_TOP" "$BS_KIND") || bs_die "cargo metadata failed in $BS_CRATE" - [ -n "$BS_LOCAL_DIRS" ] || bs_die "cargo metadata listed no local packages in $BS_CRATE" + return r +print("\n".join(rel(m) for m in sorted(dirs))) +print("VENDOR_REPOS " + " ".join(rel(t) for t in sorted(repos)))' "$BS_WT_ROOT" "$BS_TOP" "$BS_KIND") || bs_die "cargo metadata failed in $BS_CRATE" + BS_VENDOR_REPOS=$(printf '%s\n' "$BS_LOCAL_DIRS" | sed -n 's/^VENDOR_REPOS //p') + BS_LOCAL_DIRS=$(printf '%s\n' "$BS_LOCAL_DIRS" | grep -v '^VENDOR_REPOS ' || true) + [ -n "$BS_LOCAL_DIRS$BS_VENDOR_REPOS" ] || bs_die "cargo metadata listed no local packages in $BS_CRATE" +} + +# which bare mirror on the box holds a repository under vendor/: a worktree of the fork (its common git dir is +# vendor/igneum-node/.git) or the fork itself -> /srv/igneum-node.git; any other repository of its own -> /srv/.git, +# created on the box on first use (run-from-mac.sh wires and pushes the ones the Cargo.toml files reach) +bs_mirror_for() { + local dir="$1" common + common=$(cd "$dir" && git rev-parse --git-common-dir 2>/dev/null) || { echo ""; return; } + common=$(cd "$dir" && cd "$common" && pwd -P) + case "$common" in */vendor/igneum-node/.git) echo "$BS_MIRROR_NODE" ;; *) echo "/srv/$(basename "$(dirname "$common")").git" ;; esac } # push the crate repo's HEAD to its bare mirror on the box (fast after the first time), then check the remote tree out at that @@ -106,15 +133,23 @@ print("\n".join(out))' "$BS_WT_ROOT" "$BS_TOP" "$BS_KIND") || bs_die "cargo meta # directory AND HEAD is a symbolic ref to a loose branch file; a detached HEAD or a worktree's .git file gives an empty hash # (which is why the Mac's worktree builds print "igneumd 2.1.0" with no commit, 6 Oct 2026). The mirror doubles as the CI # runner's source later. +# bs_push_and_checkout the crate's own repository (BS_TOP) at BS_SHA on BS_BRANCH +# bs_push_and_checkout another repository under vendor/ (a path dependency's), at its own HEAD bs_push_and_checkout() { - local url="$BS_HOST:$BS_MIRROR" remote_top="$BS_REMOTE_WT/$BS_TOP_REL" - [ "$BS_TOP_REL" = . ] && remote_top="$BS_REMOTE_WT" - bs_log "push $BS_TOP HEAD $BS_SHA ($BS_BRANCH) -> $url" - GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$BS_TOP" push -q --force "$url" "HEAD:refs/heads/$BS_BRANCH" || bs_die "push to the mirror failed" + local top="${1:-$BS_TOP}" rel="${2:-$BS_TOP_REL}" mirror branch sha url remote_top + if [ -z "${1:-}" ]; then mirror="$BS_MIRROR"; branch="$BS_BRANCH"; sha="$BS_SHA"; else + mirror=$(bs_mirror_for "$top"); [ -n "$mirror" ] || bs_die "$top is not a git repository" + sha=$(git -C "$top" rev-parse HEAD); branch=$(git -C "$top" branch --show-current 2>/dev/null || true); [ -n "$branch" ] || branch="detached-$(git -C "$top" rev-parse --short HEAD)" + [ "$mirror" = "$BS_MIRROR_NODE" ] || [ "$mirror" = "$BS_MIRROR_REPO" ] || bs_ssh "[ -d '$mirror' ] || git init -q --bare -b master '$mirror'" || bs_die "cannot create the mirror $mirror on the box" + fi + url="$BS_HOST:$mirror"; remote_top="$BS_REMOTE_WT/$rel" + [ "$rel" = . ] && remote_top="$BS_REMOTE_WT" + bs_log "push $top HEAD $sha ($branch) -> $url" + GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$top" push -q --force "$url" "HEAD:refs/heads/$branch" || bs_die "push to the mirror failed" # the checkout runs as remote-run.sh's `checkout` mode: discard the previous overlay (tracked edits and untracked files, # target dirs kept), then the branch at the commit. 6 October 2026, PC 1 worker's first use: the overlay of an earlier # commit's uncommitted files stayed in the box's tree and `git checkout -B` refused with "local changes would be overwritten". - BR_MODE=checkout BR_CO_DIR="$remote_top" BR_CO_MIRROR="$BS_MIRROR" BR_CO_BRANCH="$BS_BRANCH" BR_CO_SHA="$BS_SHA" BR_CO_WT="$BS_REMOTE_WT" \ + BR_MODE=checkout BR_CO_DIR="$remote_top" BR_CO_MIRROR="$mirror" BR_CO_BRANCH="$branch" BR_CO_SHA="$sha" BR_CO_WT="$BS_REMOTE_WT" \ bash -c ' for v in BR_MODE BR_CO_DIR BR_CO_MIRROR BR_CO_BRANCH BR_CO_SHA BR_CO_WT; do printf "export %s=%q\n" "$v" "${!v}"; done cat "$0"' "$(dirname "${BASH_SOURCE[0]}")/remote-run.sh" | bs_ssh 'bash -s' || bs_die "remote checkout at $remote_top failed" @@ -148,6 +183,7 @@ bs_sync_sources() { local d bs_push_and_checkout for d in $BS_LOCAL_DIRS; do bs_overlay_dir "$d"; done + for d in $BS_VENDOR_REPOS; do bs_push_and_checkout "$BS_WT_ROOT/$d" "$d"; bs_overlay_dir "$d"; done } bs_sha256() { shasum -a 256 "$1" | awk '{ print $1 }'; } diff --git a/infra/build-server/provision.sh b/infra/build-server/provision.sh index 893ac37b9..8c4d35aa3 100755 --- a/infra/build-server/provision.sh +++ b/infra/build-server/provision.sh @@ -118,11 +118,13 @@ step_os_check() { ok os "$PRETTY_NAME, $(nproc) threads, $(awk '/MemTotal/ { printf "%d GB", $2 / 1024 / 1024 }' /proc/meminfo)" } +# libprotobuf-dev: the well-known .proto files (google/protobuf/empty.proto) that sp1-prover-types's build script imports +# (6 October 2026, the first proving build on the box: "protoc failed: google/protobuf/empty.proto: File not found"). # the proven Ubuntu 24.04 set: the PC build job's APT lists (app/igneum-app/src/jobbuild.rs: mingw-w64 posix threads so # libstdc++ has std::thread for rocksdb, clang for librocksdb-sys's bindgen, protoc for the node's proto crates) plus the # task's list (build-essential, clang, lld, pkg-config, libssl-dev, cmake, git, tmux) and what the scripts here call APT_PACKAGES=( - build-essential clang lld llvm libclang-dev pkg-config libssl-dev cmake protobuf-compiler + build-essential clang lld llvm libclang-dev pkg-config libssl-dev cmake protobuf-compiler libprotobuf-dev gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-x86-64-dev mingw-w64-tools git tmux curl ca-certificates xz-utils zstd unzip rsync jq python3 ufw htop file caddy ) diff --git a/infra/build-server/run-from-mac.sh b/infra/build-server/run-from-mac.sh index 47fbef574..f36fe94fc 100755 --- a/infra/build-server/run-from-mac.sh +++ b/infra/build-server/run-from-mac.sh @@ -57,6 +57,16 @@ wire_remote() { # repo-dir mirror label } wire_remote "$MAIN_REPO" "$BS_MIRROR_REPO" "igneum" wire_remote "$MAIN_REPO/vendor/igneum-node" "$BS_MIRROR_NODE" "igneum-node (the fork)" +# every repository under vendor/ that a Cargo.toml of the repo reaches by path (6 October 2026: proving/igneum-prove -> +# vendor/igneum-node-exec, a worktree of the fork, so already on /srv/igneum-node.git; a repository of its own gets its own +# mirror /srv/.git here, and lib.sh checks it out whole on the box) +grep -rhoE 'path *= *"\.\./[^"]*vendor/[^/"]+' --include=Cargo.toml "$MAIN_REPO/app" "$MAIN_REPO/proving" "$MAIN_REPO/igneum-pow" "$MAIN_REPO/igneum-census" "$MAIN_REPO/proto-vdf" 2>/dev/null \ + | sed -E 's|.*vendor/||' | sort -u | while read -r name; do + dir="$MAIN_REPO/vendor/$name"; [ -d "$dir" ] || { bs_log "vendor/$name is reached by a path dependency but missing on this Mac"; continue; } + mirror=$(bs_mirror_for "$dir") + if [ "$mirror" = "$BS_MIRROR_NODE" ]; then bs_log "vendor/$name: a worktree of the fork ($(git -C "$dir" branch --show-current)), on $BS_MIRROR_NODE already" + else bs_ssh "[ -d '$mirror' ] || git init -q --bare -b master '$mirror'"; wire_remote "$dir" "$mirror" "vendor/$name"; fi +done bs_log "ssh line: ssh -i $BS_KEY build@$IP" bs_log "next: cd && $REPO/tools/build-remote.sh (cross: tools/cross-remote.sh)" diff --git a/tools/build-remote.sh b/tools/build-remote.sh index e1dddffc9..7d792790d 100755 --- a/tools/build-remote.sh +++ b/tools/build-remote.sh @@ -50,6 +50,7 @@ while [ $# -gt 0 ]; do esac done [ "${CARGO_ARGS[0]:-}" = cargo ] && CARGO_ARGS=("${CARGO_ARGS[@]:1}") +CARGO_ARGS_GIVEN=""; [ "${#CARGO_ARGS[@]}" -gt 0 ] && CARGO_ARGS_GIVEN=1 bs_host bs_context @@ -104,7 +105,12 @@ if [ "$FETCH" = 1 ] && [ -n "$ARTEFACTS" ]; then mkdir -p "$OUT" for a in $ARTEFACTS; do rel="${a#"$TARGET_DIR"/}"; dest="$OUT/$rel"; mkdir -p "$(dirname "$dest")" - bs_rsync -p "$BS_HOST:$BS_REMOTE_CRATE/$a" "$dest" || bs_die "no $a on the box after the build" + if ! bs_rsync -p "$BS_HOST:$BS_REMOTE_CRATE/$a" "$dest" 2>/dev/null; then + # a default artefact the caller's own `-p` selection did not build is noted, not fatal (6 Oct 2026: `-p igneum-prove-host` + # alone left no igneum-prove-export); a missing artefact the caller NAMED with --artefacts is fatal + if [ -n "${ARTEFACTS_SET:-}" ] || [ -z "${CARGO_ARGS_GIVEN:-}" ]; then bs_die "no $a on the box after the build"; fi + bs_log "no $a on the box (not built by cargo ${CARGO_ARGS[*]}); skipped"; continue + fi bs_log "artefact $dest: $(bs_size "$dest") bytes, sha256 $(bs_sha256 "$dest"), $(file -b "$dest" | cut -c1-60)" # the commit-string gate (rule of 6 October 2026): a node binary without its commit in its strings fails the run case "$BS_KIND:$(basename "$dest")" in node:igneumd) "$HERE/ci/commit-string-check.sh" "$dest" "$BS_SHA" || bs_die "commit-string gate failed for $a" ;; esac # only kaspad depends on kaspa-build-info