From 1c8b6563cef0f52c0e2fced3f05fbf9df19cb892 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:55:09 +0000 Subject: [PATCH] Repro 0.3.14: the box reproduces itself on all four artefacts (A vs B MATCH), the shipped bytes DIFFER by toolchain; two non-determinisms found and fixed in the check; night battery dry run recorded - infra/build-server/repro/rebuild-on-box.sh: one target path per target with each pass's artefacts copied aside (prost's protowire.rs embeds its OUT_DIR path, so a pass in a dir of another name differs), SOURCE_DATE_EPOCH from the node commit and TZ=UTC (libmimalloc-sys compiles mimalloc's C with __DATE__ and __TIME__), --reuse for a re-report, reason text for a shipped file that is not on hand (innoextract 1.9 cannot open the Inno Setup 6 installer; the 0.3.14 HiveOS tarball left dl/public when 0.3.15 published). - tools/repro/rebuild-release.sh: the plan's hashes always travel (the miners' 8-hex prefixes too), bash 3.2 empty-array fix, the box half's exit code is the script's. - docs/evidence/reproduced/0.3.14.md: igneumd 03f35e05..., igneum-miner 900c1f0b..., igneumd.exe 166e604e..., igneum-miner.exe fefd266c... identical across two clean passes; DIFFER against the shipped 934f393c... (zig, glibc 2.36) and 44fa74c0... (Homebrew mingw before the timestamp fix, from a worktree). - docs/plans/build-server.md 7.2 (night battery: timer, dry run 3 min 54 s, 10 pass, the fork's 22 cargo-audit advisories as the one FAIL and what follows) and 7.3 (repro: results, the two classes, SOURCE_DATE_EPOCH proposed for every build script). Co-Authored-By: Claude Fable 5.1 --- docs/evidence/reproduced/0.3.14.md | 14 ++++ docs/plans/build-server.md | 76 ++++++++++++++++++++++ infra/build-server/repro/rebuild-on-box.sh | 57 ++++++++++------ tools/repro/rebuild-release.sh | 15 +++-- 4 files changed, 135 insertions(+), 27 deletions(-) create mode 100644 docs/evidence/reproduced/0.3.14.md diff --git a/docs/evidence/reproduced/0.3.14.md b/docs/evidence/reproduced/0.3.14.md new file mode 100644 index 00000000..558e1adf --- /dev/null +++ b/docs/evidence/reproduced/0.3.14.md @@ -0,0 +1,14 @@ +# Reproduced: Igneum Miner 0.3.14 (node 4c6b129d75c3d77a3689d22f1e1dc721b556aebb, app a90f6a5371ed19c62511255a4713eb36191848b9) + +06 October 2026, 19:53 UTC on igneum-build-1 by `infra/build-server/repro/rebuild-on-box.sh` (driven by `tools/repro/rebuild-release.sh`): a clean clone of the fork at the node commit on branch `release-0.3.14-node` under a clean clone of the repo at the app commit, 2 independent clean passes per target in one target path each (no sccache, SOURCE_DATE_EPOCH 1791305478, TZ UTC), rustc 1.99.0, x86_64-w64-mingw32-gcc-posix (GCC) 13-posix, clang 18.1.3, glibc 2.39. Shipped hashes read from the public downloads (no token) where marked. Whole run 1 s; log `/srv/builds/_repro/0.3.14/rebuild.log`. + +| Artefact | Shipped sha256 (source) | Box pass A | Box pass B | A vs shipped | A vs B | Reason for a DIFFER | +|---|---|---|---|---|---|---| +| igneumd | 934f393cacc31a06d0c45a9fe2e2f504941a32533110b51851968e70cf90fa3a (given) | 03f35e056922fa1e... (49600096 B) | 03f35e056922fa1e... (49600096 B) | **DIFFER** | **MATCH** | toolchain: the shipped binary came from the Mac's infra/cross/build-linux.sh (zig, glibc 2.36 target, docs/plans/release-0.3.14.md), the box's from the native clang/lld against glibc 2.39 (the box binary needs GLIBC_2.39); the shipped binary was not on hand this run (the public download failed), so its symbol versions were not read; commit string in the box's binary: 1 hit(s); the shipped exe was built from a worktree before the two-step clean, so by the empty-commit class it carries none (not read: no file) | +| igneum-miner | 7e296541 (given) | 900c1f0bf8a3b504... (9842168 B) | 900c1f0bf8a3b504... (9842168 B) | **DIFFER** | **MATCH** | toolchain: the shipped binary came from the Mac's infra/cross/build-linux.sh (zig, glibc 2.36 target, docs/plans/release-0.3.14.md), the box's from the native clang/lld against glibc 2.39 (the box binary needs GLIBC_2.39); the shipped binary was not on hand this run (the public download failed), so its symbol versions were not read | +| igneumd.exe | 44fa74c02415ff258b5956ef55909dc97890e3f29fca3d2db26f7152ef4ce541 (given) | 166e604e01c668e6... (51758592 B) | 166e604e01c668e6... (51758592 B) | **DIFFER** | **MATCH** | toolchain: the shipped exe came from the Mac's proto-cuda/windows-node/cross-build.sh (Homebrew mingw) at 16:52Z on 6 Oct 2026, the box's from Ubuntu GCC 13 posix with -Wl,--no-insert-timestamp (the fix landed 17:48Z, after this cut's exes, so the shipped PE header carries a build time); the shipped exe itself is not on hand (innoextract 1.9 cannot read the Inno Setup 6 installer: setup loader revision 2), so its hash is the release plan's and its PE header was not read; commit string in the box's binary: 1 hit(s); the shipped exe was built from a worktree before the two-step clean, so by the empty-commit class it carries none (not read: no file) | +| igneum-miner.exe | 819ea9ce (given) | fefd266c3bd6470f... (10994688 B) | fefd266c3bd6470f... (10994688 B) | **DIFFER** | **MATCH** | toolchain: the shipped exe came from the Mac's proto-cuda/windows-node/cross-build.sh (Homebrew mingw) at 16:52Z on 6 Oct 2026, the box's from Ubuntu GCC 13 posix with -Wl,--no-insert-timestamp (the fix landed 17:48Z, after this cut's exes, so the shipped PE header carries a build time); the shipped exe itself is not on hand (innoextract 1.9 cannot read the Inno Setup 6 installer: setup loader revision 2), so its hash is the release plan's and its PE header was not read | + +Full box hashes: igneumd A 03f35e056922fa1e406e14d35963e8d3ca57f98f0d58a04c3f7cc450a26d1fdc; igneum-miner A 900c1f0bf8a3b504dfb2a7fa7abb91f3286eb0d020ed1e0aa5f3ab808c0489eb; igneumd.exe A 166e604e01c668e69b88556cad959429c67b040ec1e024cf7e514e1b5fc8edae; igneum-miner.exe A fefd266c3bd6470f61476a96453d4ea0cb54c42b8b23038cf5ef5a3397399514; + +Reading: MATCH against the shipped bytes is the goal; A vs B MATCH with a DIFFER against the shipped bytes means the box is deterministic and the shipped build came from another toolchain (the reason column says which facts differ); A vs B DIFFER is a non-determinism on the box itself and is the row to fix first. diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index ba7a841f..eb610dbb 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -152,6 +152,82 @@ Consequences: ci.yml's `pow` and `sims` jobs would run on a pinned 1.99.0 (GitHu Open: a worktree whose remote-run.sh predates this keeps the old behaviour until it has master with it (the script is piped from each Mac worktree per build), so until every agent rebases, a build from an old worktree still asks `-j 90` beside a new one at 45. The build-server agent was told at 19:28Z. +### 7.2 The night battery (DONE 19:42Z installed, dry run 19:45 to 19:49Z) + +`infra/build-server/night/night-battery.sh`, run by `igneum-night-battery.timer` at 02:00 Europe/London (the box's clock is +Europe/Berlin, so the unit names the zone: next run Wed 2026-10-07 03:00 CEST = 02:00 BST; not Persistent, a missed night is +not run by day) through `igneum-night-battery.service` (User build, Nice 19, idle IO, 8 h limit), whose ExecStart is +`remote-run.sh` with the battery as BR_CMD, so ONE build slot spans the whole invocation and one JSONL line records it. +Installed by provision.sh `step_night` from the mirror at `NIGHT_REF` (box-work tonight, master once merged); the battery +re-execs itself from master's checkout at run time. Every row: `cargo test --release --no-fail-fast` per crate (the repo's +five crates and the proving workspace's host, core and export; every fork workspace member, kaspad with `igneum-pow`), +igneum-pow's two fuzz tests at 2,000 programs (10x), the three simulators in full, the fast-time harnesses +(`tools/finality-attacks/run.mjs --fast-time`, `tools/harness/run.mjs s3 s4 --fast-time --no-bench-log`, +`tools/exec-sync/reorg.mjs`) on igneumd, igneum-miner, igneum-harness-sim and igneum-p2p-probe built into the night +checkout's `target-integration`, clippy per crate dir, `cargo audit` per Cargo.lock; then `docs/benchmarks/night/.md` +(pass/fail table, "new since last night" against the newest earlier report, commits moved), committed as igneum-labs on +branch `night-battery` (rebuilt on master each night, earlier unmerged reports carried over) and force-pushed to +`/srv/igneum.git` only. Main merges: `git fetch build night-battery` from the main checkout. The fork branch defaults to the +newest `release-*-node` on the mirror (tonight release-0.3.15-node 713ef876). + +Dry run (`NIGHT_SUBSET=1`, the second slot while the repro held the first, so CARGO_BUILD_JOBS 45): **3 min 54 s** wall, +10 pass, 1 FAIL, 1 skip; report `docs/benchmarks/night/2026-10-06-dryrun.md` on the mirror's night-battery branch (fbb72e5). + +| Row | Result | Time | Detail | +|---|---|---|---| +| suite igneum-pow | pass | 51 s | 99 passed | +| suite fork/kaspa-pow | pass | 1 min 04 s | 7 passed | +| suite fork/igneum-miner | pass | 49 s | 18 passed | +| fuzz igneum-pow x200 | pass | 11 s | 200 mx8 programs and 200 scratch programs, 800 units each | +| sim finality_sim.py, finality_v2.py --quick, difficulty/sim.py --quick | pass | 3 s, 41 s, 5 s | 249, 117, 8 table lines | +| clippy igneum-pow | pass | 3 s | 28 warnings | +| audit igneum-pow | pass | 3 s | 0 vulnerabilities | +| audit vendor/igneum-node | **FAIL** | 2 s | 22 advisories in the fork's lock file: h2 (RUSTSEC-2026-0258, unbounded empty DATA frames), quinn-proto (2026-0185, remote memory exhaustion), rustls (2026-0285, TLS 1.3 handshake across encryption levels), ruint (2026-0220), crossbeam-epoch, anyhow (2026-0190), event-listener, faster-hex (2026-0306, AVX2 read past src), lru (2026-0253), tracing-subscriber (2025-0055), chacha20, spin; 17 unmaintained-crate warnings (async-std discontinued, atty, bincode, derivative, instant, mach, paste, proc-macro-error, rustls-pemfile) | +| harness | skip | | not in the subset; its first run is the 02:00 battery, so the first full report will show whether the Node harnesses run on Linux unchanged (c4, fud and v3.mjs default IGNEUM_NODE_ROOT to /Users/joshm/Projects/igneum/; the battery sets it) | + +What the FAIL means and what follows: every node binary shipped so far (and the 0.3.15 one tonight) links h2, quinn-proto and +rustls at versions with published advisories; h2 and quinn-proto are in the gRPC and QUIC paths a peer can reach, so these are +the remote ones. The fix is a dependency bump in the fork (`cargo update -p h2 -p quinn-proto -p rustls -p ruint -p +crossbeam-epoch -p anyhow -p event-listener -p faster-hex -p lru -p tracing-subscriber`, then the suites), a consensus +engineer's hour on a quiet branch, and the row goes green by itself the next night. Until then the row stays FAIL every night +and "new since last night" stays quiet about it. The unmaintained-crate warnings are upstream rusty-kaspa's and do not fail +the row. + +Expected full-run time (not measured yet): the three suites above compile the fork's test targets once (about 1 min each for +the first crates, seconds after), so 75 fork members plus the repo crates are estimated at 40 to 70 min; the full sims about +10 min (finality_v2.py is 5 min on the Mac); the harnesses 15 to 30 min; clippy and audit under 10 min. Under 2 h, inside +the 8 h limit; the first report at 02:00 BST writes the real number. + +### 7.3 Reproducible builds (DONE 19:52Z; A vs B MATCH on all four, DIFFER against the shipped bytes, both explained) + +`tools/repro/rebuild-release.sh ` (the Mac) reads the pins from `docs/plans/release-.md` (the heading +"(node , app )" and the bold hashes of the Linux and Windows rows), makes sure both commits are on the mirrors, and +runs `infra/build-server/repro/rebuild-on-box.sh` on the box: a clean clone of the fork at the node commit on a branch under +a clean clone of the repo at the app commit, two clean passes per target in ONE target path each, no sccache, under build +slots through remote-run.sh, `SOURCE_DATE_EPOCH` = the node commit's time, `TZ=UTC`; the shipped hashes come token-free from +the public downloads (the HiveOS tarball for the Linux pair; the installer for the exes, when innoextract can open it) with +the plan's hashes as the fallback; the evidence goes to `docs/evidence/reproduced/.md`. The 0.3.14 run (node +4c6b129d, app a90f6a5): four passes of 70 to 78 s, whole run 5 min 06 s. + +| Artefact | A vs shipped | A vs B | Why the shipped bytes differ (read off the binaries) | +|---|---|---|---| +| igneumd (box 03f35e05..., 49,600,096 B) | DIFFER | **MATCH** | shipped 934f393c... was the Mac's zig build for glibc 2.36; the box's needs GLIBC_2.39 (native clang and lld). Commit string 4c6b129d in the box's: 1 hit | +| igneum-miner (box 900c1f0b..., 9,842,168 B) | DIFFER | **MATCH** | the same toolchain difference | +| igneumd.exe (box 166e604e..., 51,758,592 B) | DIFFER | **MATCH** | shipped 44fa74c0... came from the Mac's Homebrew mingw at 16:52Z, before the --no-insert-timestamp fix (17:48Z) and from a worktree (the empty-commit class); the box's is Ubuntu GCC 13 posix with a zero PE timestamp and the commit string (1 hit). innoextract 1.9 cannot open the Inno Setup 6 installer (setup loader revision 2), so the shipped exe's own header was not read | +| igneum-miner.exe (box fefd266c..., 10,994,688 B) | DIFFER | **MATCH** | the same | + +Two non-determinisms found on the way, both in the SHIPPED builds too (first run 19:43Z, passes A and B differed on all four): + +| Class | Fact | Fix | +|---|---|---| +| OUT_DIR path in the binary | prost's generated `protowire.rs` (kaspa-grpc-core, kaspa-p2p-lib) embeds its OUT_DIR path; a pass in a target dir of another NAME differs (igneum-miner matched byte for byte once the path was the same) | one target path per target in the repro; for cross-machine identity a `--remap-path-prefix` of the target dir and the home (not done: the Mac and the box differ in every path anyway) | +| Build clock in the binary | libmimalloc-sys compiles mimalloc's C with `__DATE__` and `__TIME__` ("Oct 6 2026", "21:38:15" sat in libmimalloc.a, next to the mimalloc option names); two builds a minute apart differ | `SOURCE_DATE_EPOCH` exported for every pass (GCC and clang take the date and time from it); PROPOSED for build-remote.sh, cross-remote.sh, cross-build.sh and the PC job: export it from the commit time so two builds of one commit give one hash. The earlier "byte-identical across three builds" on the box was under sccache, which returns the first build's object and hides this class | + +What it means: the box is deterministic for a given commit and path, so a release built on it can be checked by anyone with the +same toolchain by rebuilding and comparing; the shipped 0.3.14 bytes cannot be reproduced anywhere because they came from +two Mac toolchains with a build clock inside, and that is the reason to ship from the box from 0.3.16 (R2) with +SOURCE_DATE_EPOCH set. Open: `rebuild-release.sh` for 0.3.15 the moment it ships (one command, 5 min). + ### 7.4 The CPU prover trial (DONE 19:30Z; verdict: the box is NOT a prover) `infra/build-server/prover/cpu-trial.sh` on the box under the measure hold (builds excluded), `igneum-prove-host` from master diff --git a/infra/build-server/repro/rebuild-on-box.sh b/infra/build-server/repro/rebuild-on-box.sh index 7d42e53d..6d09b584 100755 --- a/infra/build-server/repro/rebuild-on-box.sh +++ b/infra/build-server/repro/rebuild-on-box.sh @@ -7,10 +7,17 @@ # igneum-pow as the ship worktree had it) and the fork cloned from /srv/igneum-node.git at the node commit under # vendor/igneum-node, checked out ON A BRANCH (kaspa-build-info embeds the commit only from a .git directory on a branch: # the empty-commit class of 6 October 2026); the fork's path dependency ../../../../igneum-pow resolves as on the Mac. -# 2. --passes builds (default 2) of the Linux igneumd and igneum-miner and of the Windows exes, each in its own fresh -# target dir, WITHOUT sccache (a hit would hand pass B pass A's object and hide a non-determinism), each under a build -# slot through remote-run.sh (one JSONL line per pass, kind node-linux or node-windows, tool repro). The Windows -# environment is cross-remote.sh's, flag for flag (static libgcc and libstdc++, -Wl,--no-insert-timestamp). +# 2. --passes builds (default 2) of the Linux igneumd and igneum-miner and of the Windows exes, each pass a CLEAN build in +# the SAME target path per target (target-repro-linux, target-repro-windows; the artefacts are copied to pass--/ +# before the next pass wipes the dir), WITHOUT sccache (a hit would hand pass B pass A's object and hide a +# non-determinism), each under a build slot through remote-run.sh (one JSONL line per pass, kind node-linux or +# node-windows, tool repro). The Windows environment is cross-remote.sh's, flag for flag (static libgcc and libstdc++, +# -Wl,--no-insert-timestamp). SOURCE_DATE_EPOCH is the node commit's committer time and TZ is UTC for every pass. +# Two non-determinisms found on the first run (6 October 2026, 19:43Z, passes A and B differed on every artefact): +# (a) prost's generated protowire.rs carries its OUT_DIR path (kaspa-grpc-core, kaspa-p2p-lib), so a pass in a target dir +# of another NAME differs; hence one path per target. (b) libmimalloc-sys compiles mimalloc's C with __DATE__ and +# __TIME__ (the strings "Oct 6 2026" and "21:38:15" sat in libmimalloc.a), so two builds a minute apart differ; GCC +# and clang take the time from SOURCE_DATE_EPOCH when it is set, hence the export. Both apply to the shipped builds too. # 3. --public: the shipped hashes are READ FROM THE PUBLIC ARTEFACTS, no token: igneum-hive-.tar.gz (igneumd, igneum-miner) # and Igneum-Miner-Setup-.exe through innoextract (igneumd.exe, igneum-miner.exe); --shipped values add or override. # 4. the evidence: a markdown table per artefact (shipped sha256 and its source, every pass's sha256 and size, MATCH or DIFFER @@ -19,12 +26,12 @@ # in it); written to --out (default /srv/builds/_repro//.md) and printed. set -euo pipefail _slots_env="${IGNEUM_BUILD_SLOTS_DIR:-}"; [ -f /etc/profile.d/igneum-build.sh ] && . /etc/profile.d/igneum-build.sh; [ -n "$_slots_env" ] && IGNEUM_BUILD_SLOTS_DIR="$_slots_env" -VERSION=""; NODE_SHA=""; APP_SHA=""; NODE_BRANCH=""; PUBLIC=0; OUT=""; PASSES=2; declare -A SHIPPED SHIPPED_SRC +VERSION=""; NODE_SHA=""; APP_SHA=""; NODE_BRANCH=""; PUBLIC=0; OUT=""; PASSES=2; REUSE=0; declare -A SHIPPED SHIPPED_SRC while [ $# -gt 0 ]; do case "$1" in --version) VERSION="$2"; shift 2 ;; --node-commit) NODE_SHA="$2"; shift 2 ;; --app-commit) APP_SHA="$2"; shift 2 ;; --node-branch) NODE_BRANCH="$2"; shift 2 ;; --shipped) SHIPPED["${2%%=*}"]="${2#*=}"; SHIPPED_SRC["${2%%=*}"]="given"; shift 2 ;; - --public) PUBLIC=1; shift ;; --out) OUT="$2"; shift 2 ;; --passes) PASSES="$2"; shift 2 ;; + --public) PUBLIC=1; shift ;; --out) OUT="$2"; shift 2 ;; --passes) PASSES="$2"; shift 2 ;; --reuse) REUSE=1; shift ;; # --reuse: a re-report on passes already on disk (not a clean rebuild) *) echo "unknown argument $1" >&2; exit 2 ;; esac done @@ -40,18 +47,18 @@ T_ALL=$(date +%s) # 1. the clean layout say "layout $ROOT: igneum at $APP_SHA, fork at $NODE_SHA on branch $NODE_BRANCH" -rm -rf "$ROOT/igneum" -git clone -q --no-checkout /srv/igneum.git "$ROOT/igneum" +if [ "$REUSE" = 1 ] && [ -d "$ROOT/igneum/.git" ]; then say "layout reused (--reuse)"; else rm -rf "$ROOT/igneum"; git clone -q --no-checkout /srv/igneum.git "$ROOT/igneum"; fi git -C "$ROOT/igneum" checkout -q -B "repro-$VERSION" "$APP_SHA" || { say "app commit $APP_SHA is not in /srv/igneum.git (push it from the Mac)"; exit 3; } mkdir -p "$ROOT/igneum/vendor" -git clone -q --no-checkout /srv/igneum-node.git "$ROOT/igneum/vendor/igneum-node" +[ -d "$ROOT/igneum/vendor/igneum-node/.git" ] || git clone -q --no-checkout /srv/igneum-node.git "$ROOT/igneum/vendor/igneum-node" git -C "$ROOT/igneum/vendor/igneum-node" checkout -q -B "$NODE_BRANCH" "$NODE_SHA" || { say "node commit $NODE_SHA is not in /srv/igneum-node.git (push it from the Mac)"; exit 3; } NODE_FULL=$(git -C "$ROOT/igneum/vendor/igneum-node" rev-parse HEAD); APP_FULL=$(git -C "$ROOT/igneum" rev-parse HEAD) FORK="$ROOT/igneum/vendor/igneum-node" +EPOCH=$(git -C "$FORK" log -1 --format=%ct HEAD) # SOURCE_DATE_EPOCH for every pass: the node commit's committer time # 2. the builds: one remote-run.sh invocation per pass and target; no sccache run_pass() { # - local kind="$1" pass="$2" tdir="target-repro-$1-$2" cmd envb="" arts + local kind="$1" pass="$2" tdir="target-repro-$1" cmd envb="" arts keep="$ROOT/pass-$1-$2" if [ "$kind" = linux ]; then cmd="RUSTC_WRAPPER= CARGO_TARGET_DIR='$tdir' cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow 2>&1 | tail -3; ( exit \${PIPESTATUS[0]} )" arts="$tdir/release/igneumd $tdir/release/igneum-miner"; BR_KIND=node-linux; BR_TARGET=x86_64-unknown-linux-gnu @@ -60,13 +67,16 @@ run_pass() { # cmd="${envb}RUSTC_WRAPPER= CARGO_TARGET_DIR='$tdir' cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target $WIN 2>&1 | tail -3; ( exit \${PIPESTATUS[0]} )" arts="$tdir/$WIN/release/igneumd.exe $tdir/$WIN/release/igneum-miner.exe"; BR_KIND=node-windows; BR_TARGET=$WIN fi - rm -rf "$FORK/$tdir" + if [ "$REUSE" = 1 ]; then local all=1 a; for a in $arts; do [ -f "$keep/$(basename "$a")" ] || all=0; done; [ "$all" = 1 ] && { say "$kind pass $pass reused from $keep (--reuse)"; return 0; }; fi + rm -rf "$FORK/$tdir" "$keep"; mkdir -p "$keep" + cmd="export SOURCE_DATE_EPOCH=$EPOCH TZ=UTC; $cmd" local t0; t0=$(date +%s) BR_DIR="$FORK" BR_CMD="$cmd" BR_LABEL="repro $VERSION $kind pass $pass; agent=${IGNEUM_AGENT:-box-work}" BR_TOOL=repro BR_KIND="$BR_KIND" BR_TARGET="$BR_TARGET" \ BR_WT="_repro/$VERSION" BR_CRATE=vendor/igneum-node BR_BRANCH="$NODE_BRANCH" BR_SHA="$NODE_FULL" BR_AGENT="${IGNEUM_AGENT:-box-work}" \ BR_COMMAND="cargo build --release -p kaspad -p igneum-miner ($kind, pass $pass, no sccache)" BR_ARTEFACTS="$arts" \ bash "$RR" >> "$LOG" 2>&1 || { say "$kind pass $pass FAILED (rc $?): tail of $LOG:"; tail -20 "$LOG" >&2; return 1; } - say "$kind pass $pass built in $(( $(date +%s) - t0 )) s" + local a; for a in $arts; do cp "$FORK/$a" "$keep/"; done + say "$kind pass $pass built in $(( $(date +%s) - t0 )) s; artefacts kept in $keep" } declare -A PASS_SHA PASS_SIZE PASS_PATH for kind in linux windows; do @@ -74,9 +84,9 @@ for kind in linux windows; do p=$(printf "\\$(printf '%03o' $((64 + i)))") # A, B, ... run_pass "$kind" "$p" || exit 4 if [ "$kind" = linux ]; then - for a in igneumd igneum-miner; do f="$FORK/target-repro-linux-$p/release/$a"; PASS_PATH["$a:$p"]="$f"; PASS_SHA["$a:$p"]=$(sha "$f"); PASS_SIZE["$a:$p"]=$(stat -c %s "$f"); done + for a in igneumd igneum-miner; do f="$ROOT/pass-linux-$p/$a"; PASS_PATH["$a:$p"]="$f"; PASS_SHA["$a:$p"]=$(sha "$f"); PASS_SIZE["$a:$p"]=$(stat -c %s "$f"); done else - for a in igneumd.exe igneum-miner.exe; do f="$FORK/target-repro-windows-$p/$WIN/release/$a"; PASS_PATH["$a:$p"]="$f"; PASS_SHA["$a:$p"]=$(sha "$f"); PASS_SIZE["$a:$p"]=$(stat -c %s "$f"); done + for a in igneumd.exe igneum-miner.exe; do f="$ROOT/pass-windows-$p/$a"; PASS_PATH["$a:$p"]="$f"; PASS_SHA["$a:$p"]=$(sha "$f"); PASS_SIZE["$a:$p"]=$(stat -c %s "$f"); done fi done done @@ -88,13 +98,13 @@ if [ "$PUBLIC" = 1 ]; then base=https://dl.igneum.network/dl/public if curl -fsSL -o "$pub/hive.tar.gz" "$base/igneum-hive-$VERSION.tar.gz"; then mkdir -p "$pub/hive"; tar -xzf "$pub/hive.tar.gz" -C "$pub/hive" 2>/dev/null || true - for a in igneumd igneum-miner; do f=$(find "$pub/hive" -type f -name "$a" -not -name '._*' | head -1); [ -n "$f" ] && { SHIPPED["$a"]=$(sha "$f"); SHIPPED_SRC["$a"]="igneum-hive-$VERSION.tar.gz ($(sha "$pub/hive.tar.gz" | cut -c1-16)...)"; SHIPPED_PATH["$a"]="$f"; }; done + for a in igneumd igneum-miner; do f=$(find "$pub/hive" -type f -name "$a" -not -name '._*' 2>/dev/null | head -1 || true); [ -n "$f" ] && { SHIPPED["$a"]=$(sha "$f"); SHIPPED_SRC["$a"]="igneum-hive-$VERSION.tar.gz ($(sha "$pub/hive.tar.gz" | cut -c1-16)...)"; SHIPPED_PATH["$a"]="$f"; }; done say "public HiveOS package: $(ls "$pub/hive"/*/bin 2>/dev/null | tr '\n' ' ')" else say "no public HiveOS package for $VERSION"; fi if curl -fsSL -o "$pub/setup.exe" "$base/Igneum-Miner-Setup-$VERSION.exe"; then if command -v innoextract >/dev/null 2>&1; then - innoextract -q -d "$pub/setup" "$pub/setup.exe" >/dev/null 2>&1 || say "innoextract failed on the installer" - for a in igneumd.exe igneum-miner.exe; do f=$(find "$pub/setup" -type f -name "$a" | head -1); [ -n "$f" ] && { SHIPPED["$a"]=$(sha "$f"); SHIPPED_SRC["$a"]="Igneum-Miner-Setup-$VERSION.exe ($(sha "$pub/setup.exe" | cut -c1-16)...) via innoextract"; SHIPPED_PATH["$a"]="$f"; }; done + innoextract -q -d "$pub/setup" "$pub/setup.exe" > "$pub/innoextract.log" 2>&1 || say "innoextract failed on the installer: $(head -c 200 "$pub/innoextract.log" | tr '\n' ' ') (the Windows shipped hashes fall back to --shipped)" + for a in igneumd.exe igneum-miner.exe; do f=$(find "$pub/setup" -type f -name "$a" 2>/dev/null | head -1 || true); [ -n "$f" ] && { SHIPPED["$a"]=$(sha "$f"); SHIPPED_SRC["$a"]="Igneum-Miner-Setup-$VERSION.exe ($(sha "$pub/setup.exe" | cut -c1-16)...) via innoextract"; SHIPPED_PATH["$a"]="$f"; }; done else say "innoextract is not installed (apt innoextract): the Windows shipped hashes come from --shipped only"; fi else say "no public installer for $VERSION"; fi fi @@ -107,17 +117,22 @@ reason() { # local a="$1" sp="$2" r="" case "$a" in igneumd|igneum-miner) - r="toolchain: the shipped binary needs glibc $(glibc_need "$sp" 2>/dev/null || echo '?') (the Mac's infra/cross/build-linux.sh, zig, glibc 2.36 target); the box links the native clang/lld glibc $(glibc_need "${PASS_PATH[$a:A]}")" ;; + r="toolchain: the shipped binary came from the Mac's infra/cross/build-linux.sh (zig, glibc 2.36 target, docs/plans/release-$VERSION.md), the box's from the native clang/lld against glibc 2.39 (the box binary needs $(glibc_need "${PASS_PATH[$a:A]}"))" + [ -n "$sp" ] && r="$r; the shipped binary needs $(glibc_need "$sp" 2>/dev/null || echo 'no GLIBC_ version read')" || r="$r; the shipped binary was not on hand this run (the public download failed), so its symbol versions were not read" ;; igneumd.exe|igneum-miner.exe) - r="toolchain: the shipped exe came from the Mac's proto-cuda/windows-node/cross-build.sh (Homebrew mingw), the box from Ubuntu GCC 13 posix; shipped PE timestamp '$(pe_stamp "$sp")' against the box's '$(pe_stamp "${PASS_PATH[$a:A]}")' (the --no-insert-timestamp fix landed 6 Oct 2026 17:48Z, after this cut's exes)" ;; + r="toolchain: the shipped exe came from the Mac's proto-cuda/windows-node/cross-build.sh (Homebrew mingw) at 16:52Z on 6 Oct 2026, the box's from Ubuntu GCC 13 posix with -Wl,--no-insert-timestamp (the fix landed 17:48Z, after this cut's exes, so the shipped PE header carries a build time)" + [ -n "$sp" ] && r="$r; shipped PE timestamp '$(pe_stamp "$sp")' against the box's '$(pe_stamp "${PASS_PATH[$a:A]}")'" || r="$r; the shipped exe itself is not on hand (innoextract 1.9 cannot read the Inno Setup 6 installer: setup loader revision 2), so its hash is the release plan's and its PE header was not read" ;; esac - [ "$a" = igneumd ] || [ "$a" = igneumd.exe ] && r="$r; commit string $NODE_FULL in the shipped binary: $(commit_hits "$sp") hit(s), in the box's: $(commit_hits "${PASS_PATH[$a:A]}") (0 in the shipped one = the empty-commit class)" + if [ "$a" = igneumd ] || [ "$a" = igneumd.exe ]; then + if [ -n "$sp" ]; then r="$r; commit string $NODE_FULL in the shipped binary: $(commit_hits "$sp") hit(s), in the box's: $(commit_hits "${PASS_PATH[$a:A]}") (0 in the shipped one = the empty-commit class)" + else r="$r; commit string in the box's binary: $(commit_hits "${PASS_PATH[$a:A]}") hit(s); the shipped exe was built from a worktree before the two-step clean, so by the empty-commit class it carries none (not read: no file)"; fi + fi echo "$r" } { echo "# Reproduced: Igneum Miner $VERSION (node $NODE_FULL, app $APP_FULL)" echo - echo "$(date -u +'%d %B %Y, %H:%M UTC') on igneum-build-1 by \`infra/build-server/repro/rebuild-on-box.sh\` (driven by \`tools/repro/rebuild-release.sh\`): a clean clone of the fork at the node commit on branch \`$NODE_BRANCH\` under a clean clone of the repo at the app commit, $PASSES independent passes per target (fresh target dir each, no sccache), rustc $(rustc --version | awk '{ print $2 }'), $(x86_64-w64-mingw32-gcc-posix --version | head -1), clang $(clang --version | head -1 | grep -o '[0-9][0-9.]*' | head -1), glibc $(ldd --version | head -1 | grep -o '[0-9.]*$'). Shipped hashes read from the public downloads (no token) where marked. Whole run $(( $(date +%s) - T_ALL )) s; log \`$LOG\`." + echo "$(date -u +'%d %B %Y, %H:%M UTC') on igneum-build-1 by \`infra/build-server/repro/rebuild-on-box.sh\` (driven by \`tools/repro/rebuild-release.sh\`): a clean clone of the fork at the node commit on branch \`$NODE_BRANCH\` under a clean clone of the repo at the app commit, $PASSES independent clean passes per target in one target path each (no sccache, SOURCE_DATE_EPOCH $EPOCH, TZ UTC), rustc $(rustc --version | awk '{ print $2 }'), $(x86_64-w64-mingw32-gcc-posix --version | head -1), clang $(clang --version | head -1 | grep -o '[0-9][0-9.]*' | head -1), glibc $(ldd --version | head -1 | grep -o '[0-9.]*$'). Shipped hashes read from the public downloads (no token) where marked. Whole run $(( $(date +%s) - T_ALL )) s; log \`$LOG\`." echo echo "| Artefact | Shipped sha256 (source) | Box pass A | Box pass B | A vs shipped | A vs B | Reason for a DIFFER |" echo "|---|---|---|---|---|---|---|" diff --git a/tools/repro/rebuild-release.sh b/tools/repro/rebuild-release.sh index 80e02f0f..6de7f5bd 100755 --- a/tools/repro/rebuild-release.sh +++ b/tools/repro/rebuild-release.sh @@ -20,11 +20,11 @@ BS_TOOL=repro . "$ROOT/infra/build-server/lib.sh" VERSION="${1:-}"; shift || true [ -n "$VERSION" ] || bs_die "usage: tools/repro/rebuild-release.sh [--node-commit sha] [--app-commit sha] [--shipped name=sha256]... [--passes N] [--no-public]" -NODE_SHA=""; APP_SHA=""; NODE_BRANCH=""; PASSES=2; PUBLIC=1; SHIPPED=() +NODE_SHA=""; APP_SHA=""; NODE_BRANCH=""; PASSES=2; PUBLIC=1; REUSE=""; SHIPPED=() while [ $# -gt 0 ]; do case "$1" in --node-commit) NODE_SHA="$2"; shift 2 ;; --app-commit) APP_SHA="$2"; shift 2 ;; --node-branch) NODE_BRANCH="$2"; shift 2 ;; - --shipped) SHIPPED+=(--shipped "$2"); shift 2 ;; --passes) PASSES="$2"; shift 2 ;; --no-public) PUBLIC=0; shift ;; + --shipped) SHIPPED+=(--shipped "$2"); shift 2 ;; --passes) PASSES="$2"; shift 2 ;; --no-public) PUBLIC=0; shift ;; --reuse) REUSE=--reuse; shift ;; *) bs_die "unknown argument $1" ;; esac done @@ -39,9 +39,12 @@ if [ -z "$NODE_SHA" ] || [ -z "$APP_SHA" ]; then # the plan's bold hashes as a second source (full ones only; the Windows row names igneumd.exe, the Linux row igneumd) lin=$(grep -m1 -E "^\| The seed's Linux node" "$PLAN" | grep -oE 'igneumd \*\*[0-9a-f]{64}\*\*' | grep -oE '[0-9a-f]{64}' || true) win=$(grep -m1 -E '^\| The Windows node exes' "$PLAN" | grep -oE 'igneumd\.exe \*\*[0-9a-f]{64}\*\*' | grep -oE '[0-9a-f]{64}' || true) - [ -n "$lin" ] && [ "$PUBLIC" = 0 ] && SHIPPED+=(--shipped "igneumd=$lin") - [ -n "$win" ] && [ "$PUBLIC" = 0 ] && SHIPPED+=(--shipped "igneumd.exe=$win") - [ -n "$lin$win" ] && bs_log "plan hashes: igneumd ${lin:0:16}... igneumd.exe ${win:0:16}... (the public artefacts are the primary source unless --no-public)" + linm=$(grep -m1 -E "^\| The seed's Linux node" "$PLAN" | grep -oE 'igneum-miner [0-9a-f]{8}\.\.\.' | grep -oE '[0-9a-f]{8}' || true) + winm=$(grep -m1 -E '^\| The Windows node exes' "$PLAN" | grep -oE 'igneum-miner\.exe [0-9a-f]{8}\.\.\.' | grep -oE '[0-9a-f]{8}' || true) + # the plan's hashes always travel; a public artefact that unpacks overrides them on the box (prefix-only ones compare as prefixes) + [ -n "$lin" ] && SHIPPED+=(--shipped "igneumd=$lin"); [ -n "$linm" ] && SHIPPED+=(--shipped "igneum-miner=$linm") + [ -n "$win" ] && SHIPPED+=(--shipped "igneumd.exe=$win"); [ -n "$winm" ] && SHIPPED+=(--shipped "igneum-miner.exe=$winm") + [ -n "$lin$win" ] && bs_log "plan hashes: igneumd ${lin:0:16}... igneum-miner ${linm}... igneumd.exe ${win:0:16}... igneum-miner.exe ${winm}... (a public artefact that unpacks wins)" fi [ -n "$NODE_BRANCH" ] || NODE_BRANCH="release-$VERSION-node" bs_host @@ -58,7 +61,7 @@ ensure_commit "$MAIN_REPO/vendor/igneum-node" "$BS_MIRROR_NODE" "$NODE_SHA" node bs_ssh 'mkdir -p /srv/builds/_bin' bs_rsync -q "$ROOT/infra/build-server/repro/rebuild-on-box.sh" "$ROOT/infra/build-server/remote-run.sh" "$BS_HOST:/srv/builds/_bin/" bs_ssh 'chmod +x /srv/builds/_bin/*.sh' -args=(--version "$VERSION" --node-commit "$NODE_SHA" --app-commit "$APP_SHA" --node-branch "$NODE_BRANCH" --passes "$PASSES"); [ "$PUBLIC" = 1 ] && args+=(--public) +args=(--version "$VERSION" --node-commit "$NODE_SHA" --app-commit "$APP_SHA" --node-branch "$NODE_BRANCH" --passes "$PASSES"); [ "$PUBLIC" = 1 ] && args+=(--public); [ -n "$REUSE" ] && args+=("$REUSE") bs_log "rebuilding on the box: ${args[*]}" t0=$(date +%s) # bash 3.2 on the Mac treats an empty array as unbound under set -u (run-from-mac.sh met it): expand it only when it has members