From 4b3c66ce28e4968822344e13cb175fb090c3cfd0 Mon Sep 17 00:00:00 2001 From: igneum-josh <337424239+igneum-josh@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:13:43 +0100 Subject: [PATCH 1/2] Repro check: sccache really off (RUSTC_WRAPPER=/usr/bin/env, an empty value falls back to the box's config), the author-time epoch of lib.sh bs_sde, and the re-stamp line the copied-sources check reads The build-server agent's two findings on rebuild-on-box.sh (6 Oct 2026, 20:1xZ): tools/ci/copied-sources-check.sh named it (a tar on a code line plus cargo build with no touch), and `RUSTC_WRAPPER=` did not switch sccache off, so some passes of the first runs took hits. Both 0.3.14 and 0.3.15 are re-run with this version before their evidence files are trusted. Co-Authored-By: Claude Fable 5.1 --- infra/build-server/repro/rebuild-on-box.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/infra/build-server/repro/rebuild-on-box.sh b/infra/build-server/repro/rebuild-on-box.sh index ff23c9eea..eb00d88da 100755 --- a/infra/build-server/repro/rebuild-on-box.sh +++ b/infra/build-server/repro/rebuild-on-box.sh @@ -12,7 +12,10 @@ # before the next pass wipes the dir), WITHOUT sccache (a hit would hand pass B pass A's object and hide a # non-determinism), each under a build slot through remote-run.sh (one JSONL line per pass, kind node-linux or # node-windows, tool repro). The Windows environment is cross-remote.sh's, flag for flag (static libgcc and libstdc++, -# -Wl,--no-insert-timestamp). SOURCE_DATE_EPOCH is the node commit's committer time and TZ is UTC for every pass. +# -Wl,--no-insert-timestamp). SOURCE_DATE_EPOCH is the node commit's author time (lib.sh bs_sde) and TZ is UTC for every +# pass, and RUSTC_WRAPPER=/usr/bin/env is a true pass-through: an EMPTY RUSTC_WRAPPER is treated by cargo as unset and falls +# back to the box's cargo config, which names sccache (the build-server agent, 6 October 2026, 20:1xZ: the first runs of +# this script had `RUSTC_WRAPPER=` and some passes took hits, so their MATCH rows were re-run with the pass-through). # Two non-determinisms found on the first run (6 October 2026, 19:43Z, passes A and B differed on every artefact): # (a) prost's generated protowire.rs carries its OUT_DIR path (kaspa-grpc-core, kaspa-p2p-lib), so a pass in a target dir # of another NAME differs; hence one path per target. (b) libmimalloc-sys compiles mimalloc's C with __DATE__ and @@ -54,17 +57,21 @@ mkdir -p "$ROOT/igneum/vendor" git -C "$ROOT/igneum/vendor/igneum-node" checkout -q -B "$NODE_BRANCH" "$NODE_SHA" || { say "node commit $NODE_SHA is not in /srv/igneum-node.git (push it from the Mac)"; exit 3; } NODE_FULL=$(git -C "$ROOT/igneum/vendor/igneum-node" rev-parse HEAD); APP_FULL=$(git -C "$ROOT/igneum" rev-parse HEAD) FORK="$ROOT/igneum/vendor/igneum-node" -EPOCH=$(git -C "$FORK" log -1 --format=%ct HEAD) # SOURCE_DATE_EPOCH for every pass: the node commit's committer time +EPOCH=$(git -C "$FORK" log -1 --format=%at HEAD) # SOURCE_DATE_EPOCH for every pass: the node commit's AUTHOR time (lib.sh bs_sde's rule, master 03ac8fd) +# the copied-sources rule: a tree that reaches the box by copy is re-stamped before cargo sees it. These are fresh git checkouts, +# not copies, but the check reads the tar of the public artefacts below and the cargo lines together; the re-stamp is cheap and +# makes the rule visible here (find ... touch over both clean clones, .git and target dirs excluded) +find "$ROOT/igneum" -type f -not -path '*/.git/*' -not -path '*/target*' -exec touch {} + # 2. the builds: one remote-run.sh invocation per pass and target; no sccache run_pass() { # local kind="$1" pass="$2" tdir="target-repro-$1" cmd envb="" arts keep="$ROOT/pass-$1-$2" if [ "$kind" = linux ]; then - cmd="RUSTC_WRAPPER= CARGO_TARGET_DIR='$tdir' cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow 2>&1 | tail -3; ( exit \${PIPESTATUS[0]} )" + cmd="RUSTC_WRAPPER=/usr/bin/env CARGO_TARGET_DIR='$tdir' cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow 2>&1 | tail -3; ( exit \${PIPESTATUS[0]} )" arts="$tdir/release/igneumd $tdir/release/igneum-miner"; BR_KIND=node-linux; BR_TARGET=x86_64-unknown-linux-gnu else envb='LLVM_LIB=$(ls -d /usr/lib/llvm-*/lib 2>/dev/null | sort -V | tail -1); export CC_x86_64_pc_windows_gnu=x86_64-w64-mingw32-gcc-posix CXX_x86_64_pc_windows_gnu=x86_64-w64-mingw32-g++-posix AR_x86_64_pc_windows_gnu=x86_64-w64-mingw32-ar CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER=x86_64-w64-mingw32-gcc-posix; export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS="-C link-arg=-static -C link-arg=-static-libgcc -C link-arg=-static-libstdc++ -C link-arg=-Wl,--no-insert-timestamp"; export IGNEUM_WINDRES=x86_64-w64-mingw32-windres LIBCLANG_PATH="$LLVM_LIB" BINDGEN_EXTRA_CLANG_ARGS_x86_64_pc_windows_gnu="--target=x86_64-w64-mingw32 --sysroot=/usr/x86_64-w64-mingw32 -I/usr/x86_64-w64-mingw32/include"; ' - cmd="${envb}RUSTC_WRAPPER= CARGO_TARGET_DIR='$tdir' cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target $WIN 2>&1 | tail -3; ( exit \${PIPESTATUS[0]} )" + cmd="${envb}RUSTC_WRAPPER=/usr/bin/env CARGO_TARGET_DIR='$tdir' cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target $WIN 2>&1 | tail -3; ( exit \${PIPESTATUS[0]} )" arts="$tdir/$WIN/release/igneumd.exe $tdir/$WIN/release/igneum-miner.exe"; BR_KIND=node-windows; BR_TARGET=$WIN fi if [ "$REUSE" = 1 ]; then local all=1 a; for a in $arts; do [ -f "$keep/$(basename "$a")" ] || all=0; done; [ "$all" = 1 ] && { say "$kind pass $pass reused from $keep (--reuse)"; return 0; }; fi From 2bd3becebc6685bac3b860cd52f3c0770c2031a4 Mon Sep 17 00:00:00 2001 From: igneum-josh <337424239+igneum-josh@users.noreply.github.com> Date: Tue, 6 Oct 2026 21:22:06 +0100 Subject: [PATCH 2/2] Repro 0.3.14 and 0.3.15 re-run with sccache off (RUSTC_WRAPPER pass-through): A vs B MATCH on all eight artefacts, hashes unchanged; evidence files regenerated Co-Authored-By: Claude Fable 5.1 --- docs/evidence/reproduced/0.3.14.md | 2 +- docs/evidence/reproduced/0.3.15.md | 2 +- docs/plans/build-server.md | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/evidence/reproduced/0.3.14.md b/docs/evidence/reproduced/0.3.14.md index 32a4d037b..31eded204 100644 --- a/docs/evidence/reproduced/0.3.14.md +++ b/docs/evidence/reproduced/0.3.14.md @@ -1,6 +1,6 @@ # Reproduced: Igneum Miner 0.3.14 (node 4c6b129d75c3d77a3689d22f1e1dc721b556aebb, app a90f6a5371ed19c62511255a4713eb36191848b9) -06 October 2026, 20:03 UTC on igneum-build-1 by `infra/build-server/repro/rebuild-on-box.sh` (driven by `tools/repro/rebuild-release.sh`): a clean clone of the fork at the node commit on branch `release-0.3.14-node` under a clean clone of the repo at the app commit, 2 independent clean passes per target in one target path each (no sccache, SOURCE_DATE_EPOCH 1791305478, TZ UTC), rustc 1.99.0, x86_64-w64-mingw32-gcc-posix (GCC) 13-posix, clang 18.1.3, glibc 2.39. Shipped hashes read from the public downloads (no token) where marked. Whole run 3 s; log `/srv/builds/_repro/0.3.14/rebuild.log`. +06 October 2026, 20:20 UTC on igneum-build-1 by `infra/build-server/repro/rebuild-on-box.sh` (driven by `tools/repro/rebuild-release.sh`): a clean clone of the fork at the node commit on branch `release-0.3.14-node` under a clean clone of the repo at the app commit, 2 independent clean passes per target in one target path each (no sccache, SOURCE_DATE_EPOCH 1791305478, TZ UTC), rustc 1.99.0, x86_64-w64-mingw32-gcc-posix (GCC) 13-posix, clang 18.1.3, glibc 2.39. Shipped hashes read from the public downloads (no token) where marked. Whole run 382 s; log `/srv/builds/_repro/0.3.14/rebuild.log`. | Artefact | Shipped sha256 (source) | Box pass A | Box pass B | A vs shipped | A vs B | Reason for a DIFFER | |---|---|---|---|---|---|---| diff --git a/docs/evidence/reproduced/0.3.15.md b/docs/evidence/reproduced/0.3.15.md index 6f28b6534..79435e195 100644 --- a/docs/evidence/reproduced/0.3.15.md +++ b/docs/evidence/reproduced/0.3.15.md @@ -1,6 +1,6 @@ # Reproduced: Igneum Miner 0.3.15 (node 713ef876073d3661e9b48d2ead9a515afd1b2156, app 563485b769868ee34a530f1c40f7419109cc493f) -06 October 2026, 20:03 UTC on igneum-build-1 by `infra/build-server/repro/rebuild-on-box.sh` (driven by `tools/repro/rebuild-release.sh`): a clean clone of the fork at the node commit on branch `release-0.3.15-node` under a clean clone of the repo at the app commit, 2 independent clean passes per target in one target path each (no sccache, SOURCE_DATE_EPOCH 1791312828, TZ UTC), rustc 1.99.0, x86_64-w64-mingw32-gcc-posix (GCC) 13-posix, clang 18.1.3, glibc 2.39. Shipped hashes read from the public downloads (no token) where marked. Whole run 2 s; log `/srv/builds/_repro/0.3.15/rebuild.log`. +06 October 2026, 20:21 UTC on igneum-build-1 by `infra/build-server/repro/rebuild-on-box.sh` (driven by `tools/repro/rebuild-release.sh`): a clean clone of the fork at the node commit on branch `release-0.3.15-node` under a clean clone of the repo at the app commit, 2 independent clean passes per target in one target path each (no sccache, SOURCE_DATE_EPOCH 1791312828, TZ UTC), rustc 1.99.0, x86_64-w64-mingw32-gcc-posix (GCC) 13-posix, clang 18.1.3, glibc 2.39. Shipped hashes read from the public downloads (no token) where marked. Whole run 480 s; log `/srv/builds/_repro/0.3.15/rebuild.log`. | Artefact | Shipped sha256 (source) | Box pass A | Box pass B | A vs shipped | A vs B | Reason for a DIFFER | |---|---|---|---|---|---|---| diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 95df11ba5..9cdd3d385 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -241,6 +241,12 @@ igneum-miner c5b48910...): DIFFER, and the binaries say why: the shipped pair ne box's 06211d55... of 19:00Z (the box build needs GLIBC_2.39, which HiveOS cannot run; the zig lane is right for that package). The Windows exes have no shipped hash yet (the public installer is still 0.3.14). `docs/evidence/reproduced/0.3.15.md`. +Re-run 20:13 to 20:22Z with sccache REALLY off (the build-server agent's finding: an empty `RUSTC_WRAPPER=` is read by cargo as +unset and falls back to the box's config, so the first runs' passes could take hits; now `RUSTC_WRAPPER=/usr/bin/env`, a +pass-through, and `SOURCE_DATE_EPOCH` = the author time of lib.sh `bs_sde`): both versions, all four artefacts, A vs B +**MATCH** with the same hashes as above (0.3.14: 03f35e05, 900c1f0b, 166e604e, fefd266c; 0.3.15: 1f1b6eee, a34e0a56, +9b377455, 65b30edd); passes of 73 to 177 s with the two repros side by side on the two slots. The MATCH rows stand on their own. + What it means: the box is deterministic for a given commit and path, so a release built on it can be checked by anyone with the same toolchain by rebuilding and comparing; the shipped 0.3.14 and 0.3.15 Linux bytes came from the Mac's zig lane with a build clock inside and cannot be reproduced anywhere, and the Windows 0.3.14 exes carried a PE timestamp. The reason to ship