From 67ae1e4c6d4edc63b3809eb03e01626d9dd5a1d3 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:09:22 +0000 Subject: [PATCH] Reproducible builds: SOURCE_DATE_EPOCH from the commit's author time, TZ=UTC and one fixed target path in every build path; self-test Main's rule of 6 October 2026 from the 0.3.14 repro (docs/evidence/reproduced/0.3.14.md): prost's protowire.rs embeds OUT_DIR, libmimalloc-sys embeds __DATE__/__TIME__, sccache hid both. lib.sh bs_repro_env exports SOURCE_DATE_EPOCH= TZ=UTC in front of every remote command (build-remote.sh, cross-remote.sh, workers-remote.sh); remote-run.sh exports BR_SDE too and logs it as source_date_epoch; proto-cuda/windows-node/cross-build.sh exports the same; the PC job carries node.commit_time in the manifest (push-build-inputs.sh) and exports it before every cargo build of a stage (jobbuild.rs, unit test asserts it; 4 of 4 pass on the box). Target dirs stay one fixed path per target. tools/build-remote.sh --self-test-repro [--full] from a fork worktree, run on the box: igneum-miner twice a minute apart without sccache (RUSTC_WRAPPER=/usr/bin/env, an empty value is unset to cargo) MATCH 91e130f5..., a per-run target path differs (OUT_DIR shown); --full: kaspad with libmimalloc-sys recompiled a minute later MATCH 70219bc2..., without the epoch differs (__DATE__ shown). Co-Authored-By: Claude Fable 5.1 --- app/igneum-app/src/jobbuild.rs | 14 ++++++-- docs/plans/build-server.md | 10 ++++++ infra/build-server/lib.sh | 13 +++++-- infra/build-server/remote-run.sh | 4 +++ packaging/windows/push-build-inputs.sh | 9 ++--- proto-cuda/windows-node/cross-build.sh | 3 ++ tools/build-remote.sh | 50 ++++++++++++++++++++++++-- tools/cross-remote.sh | 2 +- tools/workers-remote.sh | 2 +- 9 files changed, 95 insertions(+), 12 deletions(-) diff --git a/app/igneum-app/src/jobbuild.rs b/app/igneum-app/src/jobbuild.rs index 204dadd4a..12d0795d9 100644 --- a/app/igneum-app/src/jobbuild.rs +++ b/app/igneum-app/src/jobbuild.rs @@ -135,6 +135,9 @@ pub struct Manifest { pub node_commit: String, /// the 40-hex commit (manifest node.commit_full, packer of 6 October 2026); empty in older manifests pub node_commit_full: String, + /// the commit's author time (manifest node.commit_time, unix seconds): SOURCE_DATE_EPOCH of every build stage, so mimalloc's + /// __DATE__/__TIME__ and anything else that reads the clock give one answer per commit (6 October 2026, the 0.3.14 repro) + pub node_commit_time: u64, pub node_dirty: bool, pub app_version: String, pub builds: Vec, @@ -161,7 +164,7 @@ pub fn parse_manifest(text: &str) -> Result { let s = |k: &str| v.get(k).and_then(|x| x.as_str()).unwrap_or("").trim().to_string(); let node = v.get("node").cloned().unwrap_or(Value::Null); let ns = |k: &str| node.get(k).and_then(|x| x.as_str()).unwrap_or("").trim().to_string(); - let mut m = Manifest { created_at: s("created_at"), node_branch: ns("branch"), node_commit: ns("commit"), node_commit_full: ns("commit_full"), node_dirty: node.get("dirty").and_then(|x| x.as_bool()).unwrap_or(false), app_version: s("app_version"), ..Default::default() }; + let mut m = Manifest { created_at: s("created_at"), node_branch: ns("branch"), node_commit: ns("commit"), node_commit_full: ns("commit_full"), node_commit_time: node.get("commit_time").and_then(|x| x.as_u64()).unwrap_or(0), node_dirty: node.get("dirty").and_then(|x| x.as_bool()).unwrap_or(false), app_version: s("app_version"), ..Default::default() }; let builds = v.get("builds").and_then(|b| b.as_array()).ok_or("manifest.json has no \"builds\" list")?; for (i, b) in builds.iter().enumerate() { let dir = b.get("dir").and_then(|x| x.as_str()).unwrap_or("").trim().to_string(); @@ -329,6 +332,11 @@ pub fn build_script(p: &BuildParams, job_id: &str, m: &Manifest, target: &str) - s.push_str(&windows_env()); } s.push_str(&format!("echo \"STAGE {target} start $(now)\"\n")); + if m.node_commit_time > 0 { + // reproducible builds: the commit's author time and UTC for every compiler of this stage; the target dir is the one + // persistent $CARGO_TARGET_DIR (never a per-run name: prost's generated code embeds OUT_DIR) + s.push_str(&format!("export SOURCE_DATE_EPOCH={} TZ=UTC; echo \"SOURCE_DATE_EPOCH=$SOURCE_DATE_EPOCH TZ=UTC\"\n", m.node_commit_time)); + } s.push_str(&format!("mkdir -p \"$OUT/{target}\"\n")); s.push_str("rc_all=0\n"); // the node's full commit from the manifest (each stage is its own script; the extract stage read it too) @@ -498,7 +506,7 @@ mod tests { use super::*; use serde_json::json; - const MANIFEST: &str = r#"{"created_at":"2026-10-04T20:00:00Z","node":{"branch":"devnet-v4","commit":"3bfe346f","dirty":false,"source":"vendor/igneum-node-v4"},"repo":{"commit":"0f44edd","branch":"build-job","dirty":true},"app_version":"0.3.4", + const MANIFEST: &str = r#"{"created_at":"2026-10-04T20:00:00Z","node":{"branch":"devnet-v4","commit":"3bfe346f","commit_time":1791300000,"dirty":false,"source":"vendor/igneum-node-v4"},"repo":{"commit":"0f44edd","branch":"build-job","dirty":true},"app_version":"0.3.4", "builds":[{"dir":"node","packages":["kaspad","igneum-miner"],"features":["kaspad/igneum-pow"],"bins":["igneumd","igneum-miner"],"targets":["linux","windows"]}, {"dir":"app/igneum-app","packages":["igneum-app"],"bins":["igneum-app"],"optional_on":["linux"]}], "tests":[{"dir":"app/igneum-app","packages":["igneum-app"]},{"dir":"node","packages":["igneum-miner"]},{"dir":"node","packages":[]}]}"#; @@ -577,6 +585,8 @@ mod tests { let lin = build_script(&p, id, &m, "linux"); assert!(lin.contains("cargo build --release $JOBS -p kaspad -p igneum-miner --features kaspad/igneum-pow 2>&1")); assert!(lin.contains("cd \"$SRC/app/igneum-app\"")); + // reproducible builds (6 October 2026): the commit's author time and UTC exported before any cargo build of the stage + assert!(lin.contains("export SOURCE_DATE_EPOCH=1791300000 TZ=UTC") && lin.find("SOURCE_DATE_EPOCH=1791300000").unwrap() < lin.find("cargo build").unwrap(), "{lin}"); assert!(lin.contains("optional on linux: not fatal")); assert!(!lin.contains("--target x86_64-pc-windows-gnu")); // the windows stage ships the runtime DLLs of its own toolchain next to the exes; the linux stage does not diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 803840060..0e6230d46 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -99,6 +99,16 @@ Also logged for context: the Mac's Linux cross-build with zig (`infra/cross/buil | The 0.3.15 prover pair for the PCs needs `--features igneum-prove-host/cuda` (the PCs run SP1_PROVER=cuda) | my first proving build named no feature | built on the box from master e1b5bc9: igneum-prove-host 73,161,528 B sha256 71bc2438856bb141f6cad3d18489f708568144fad5a06a002fbadefb9ce256f9, igneum-prove-export 3,609,360 B sha256 263bf4cef70af4a13a45b2e79b8dbab373282ea4c571f624d02ddb5791935361 (52 s warm, no CUDA needed at build time); handed to the shipper | | Let's Encrypt saw NXDOMAIN for build.igneum.network | the deSEC record was minutes old; Ubuntu's Caddy then fell back to ZeroSSL and failed with HTTP 422 for ever | issuer pinned to Let's Encrypt; the retry got the certificate | +## 5b. Reproducible builds (main's rule, 6 October 2026, from the 0.3.14 repro docs/evidence/reproduced/0.3.14.md) + +| Class | What differed | Standard fix, in every build path | +|---|---|---| +| prost's generated `protowire.rs` embeds `OUT_DIR` | two builds in differently named target dirs give different bytes | ONE fixed target path per target: `target` (or the name `--target-dir` gives) on the box, `$CARGO_TARGET_DIR` on the Mac's cross-build.sh, the persistent dir of the PC job; never a per-run name | +| libmimalloc-sys compiles mimalloc's C with `__DATE__`/`__TIME__` | two builds a minute apart differ when mimalloc recompiles | `SOURCE_DATE_EPOCH` = the node commit's author time and `TZ=UTC`, exported in lib.sh `bs_repro_env` (build-remote.sh, cross-remote.sh, workers-remote.sh), remote-run.sh (`BR_SDE`, logged in the JSONL line as `source_date_epoch`), proto-cuda/windows-node/cross-build.sh, and the PC job (push-build-inputs.sh writes `node.commit_time`, jobbuild.rs exports it before every cargo build of a stage; unit test asserts it) | +| sccache hid both | a cache hit returns the first build's object | the self-test runs with `RUSTC_WRAPPER=/usr/bin/env` (a pass-through; an EMPTY value is "unset" to cargo and would fall back to the configured sccache) | + +Self-test: `tools/build-remote.sh --self-test-repro [--full]` from a fork worktree. Run 6 Oct 20:02 UTC on the box (igneum-node-bs at 3bfe346f, epoch 1791120573): igneum-miner twice a minute apart, kaspa-grpc-core cleaned in between: MATCH 91e130f52438edf012466d3f1d3d634ab9263cd7858e67d2dd8d590b152f8a22; the same build into a per-run target path: 548671e7... (differs, the OUT_DIR class shown firing). `--full` (kaspad with libmimalloc-sys recompiled a minute later, with and without the epoch): run 6 Oct 20:04 to 20:08 UTC (247 s): kaspad with libmimalloc-sys recompiled a minute later, epoch set: MATCH 70219bc29cc98ac75da00702b9966f9f5d73cbf10efaca1c8841c00bb5aae7bf; without the epoch, a minute later: 45169e88... (differs, the __DATE__ class shown firing); the miner line again MATCH 91e130f5..., the per-run path 310383f5... (differs). The box is deterministic with the rule and shown non-deterministic without it + ## 5a. The GPU workers (added 6 October 2026, 19:10 UTC, for the class v4 rehearsal) | What | Fact | diff --git a/infra/build-server/lib.sh b/infra/build-server/lib.sh index fbf2550d4..a2c758876 100755 --- a/infra/build-server/lib.sh +++ b/infra/build-server/lib.sh @@ -205,6 +205,15 @@ bs_sync_sources() { for d in $BS_VENDOR_REPOS; do bs_push_and_checkout "$BS_WT_ROOT/$d" "$d"; bs_overlay_dir "$d"; done } +# Reproducible builds (main, 6 October 2026, from the 0.3.14 repro docs/evidence/reproduced/0.3.14.md): two classes made two +# builds of one tree differ and sccache hid both. (1) prost's generated protowire.rs embeds OUT_DIR, so the TARGET PATH must be +# the same between builds: every tool here builds into one fixed directory per target (`target`, or the name --target-dir gives, +# never a per-run name). (2) libmimalloc-sys compiles mimalloc's C with __DATE__/__TIME__, so SOURCE_DATE_EPOCH is set to the +# commit's author time and TZ to UTC; the same two exports go into the Mac's cross-build.sh and the PC job (jobbuild.rs). +# bs_repro_env prints the export line a remote command starts with; BR_SDE carries the value into remote-run.sh's JSONL line. +bs_sde() { git -C "${1:-$BS_TOP}" log -1 --format=%at HEAD; } +bs_repro_env() { local sde; sde=$(bs_sde "${1:-$BS_TOP}"); BR_SDE="$sde"; export BR_SDE; printf 'export SOURCE_DATE_EPOCH=%s TZ=UTC; ' "$sde"; } + bs_sha256() { shasum -a 256 "$1" | awk '{ print $1 }'; } bs_size() { stat -f %z "$1" 2>/dev/null || stat -c %s "$1"; } bs_fmt_secs() { local s=$1; printf '%d min %02d s' $((s / 60)) $((s % 60)); } @@ -230,9 +239,9 @@ bs_remote_run() { label="$label; agent=$agent" BR_DIR="$dir" BR_LABEL="$label" BR_CMD="$cmd" BR_TOOL="${BS_TOOL:-build-remote}" BR_WT="$BS_WT" BR_CRATE="$BS_CRATE_REL" \ BR_BRANCH="$BS_BRANCH" BR_SHA="$BS_SHA" BR_AGENT="$agent" BR_KIND="${BR_KIND:-other}" BR_COMMAND="${BR_COMMAND:-}" \ - BR_TARGET="${BR_TARGET:-}" BR_ARTEFACTS="${BR_ARTEFACTS:-}" \ + BR_TARGET="${BR_TARGET:-}" BR_ARTEFACTS="${BR_ARTEFACTS:-}" BR_SDE="${BR_SDE:-}" \ bash -c ' - for v in BR_DIR BR_LABEL BR_CMD BR_TOOL BR_WT BR_CRATE BR_BRANCH BR_SHA BR_AGENT BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS; do + for v in BR_DIR BR_LABEL BR_CMD BR_TOOL BR_WT BR_CRATE BR_BRANCH BR_SHA BR_AGENT BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS BR_SDE; do printf "export %s=%q\n" "$v" "${!v}" done cat "$0"' "$(dirname "${BASH_SOURCE[0]}")/remote-run.sh" | bs_ssh 'bash -s' diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh index f04b87baa..c09f3c5b4 100755 --- a/infra/build-server/remote-run.sh +++ b/infra/build-server/remote-run.sh @@ -161,6 +161,7 @@ d = { "agent": e.get('BR_AGENT'), "slot": num(e['BR_SLOT']), "wait_s": num(e['BR_WAIT']), "queued_at": iso(e['BR_T0']), "start": iso(e['BR_START']), "end": iso(e['BR_END']), "secs": num(e['BR_SECS']), "exit": num(e['BR_EXIT']), "compiles": num(e['BR_COMPILES']), "jobs": num(e.get('BR_JOBS')), "measure": (e.get('BR_MEASURE') == '1') or None, + "source_date_epoch": num(e.get('BR_SDE')), } sc = {k: num(e[v]) for k, v in (("hits", "BR_HITS"), ("misses", "BR_MISSES"), ("hits_total", "BR_HITS_T"), ("misses_total", "BR_MISSES_T"))} sc = {k: v for k, v in sc.items() if v is not None} @@ -255,6 +256,9 @@ else fi cd "$BR_DIR" || { jsonlog 2 "$got" "$waited" "" "$(date +%s)" "" "" "" "" "" ""; exit 2; } +# reproducible builds (main, 6 October 2026, the 0.3.14 repro): the commit's author time as SOURCE_DATE_EPOCH (mimalloc's +# __DATE__/__TIME__), UTC; the target dir is fixed per target by the caller (prost's generated code embeds OUT_DIR) +if [ -n "${BR_SDE:-}" ]; then export SOURCE_DATE_EPOCH="$BR_SDE" TZ=UTC; echo "build-remote: SOURCE_DATE_EPOCH=$BR_SDE TZ=UTC" >&2; fi sccache --start-server >/dev/null 2>&1 || true stat_field() { sccache --show-stats 2>/dev/null | awk -v key="$1" 'index($0, key) == 1 { print $NF; exit }'; } exec_before=$(stat_field "Compile requests executed"); hits_before=$(stat_field "Cache hits "); misses_before=$(stat_field "Cache misses ") diff --git a/packaging/windows/push-build-inputs.sh b/packaging/windows/push-build-inputs.sh index 1cfb2c381..39775bf73 100755 --- a/packaging/windows/push-build-inputs.sh +++ b/packaging/windows/push-build-inputs.sh @@ -80,7 +80,8 @@ fi NODE_BRANCH="$(git -C "$NODE_SRC" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)" NODE_COMMIT="$(git -C "$NODE_SRC" rev-parse --short HEAD 2>/dev/null || echo unknown)" -NODE_COMMIT_FULL="$(git -C "$NODE_SRC" rev-parse HEAD 2>/dev/null || echo unknown)" # the PC job writes it into a .git for kaspa-build-info (6 Oct 2026) +NODE_COMMIT_FULL="$(git -C "$NODE_SRC" rev-parse HEAD 2>/dev/null || echo unknown)" +NODE_COMMIT_TIME="$(git -C "$NODE_SRC" log -1 --format=%at HEAD 2>/dev/null || echo 0)" # SOURCE_DATE_EPOCH on the PC (reproducible builds, 6 Oct 2026) # the PC job writes it into a .git for kaspa-build-info (6 Oct 2026) NODE_DIRTY=false; [ -z "$(git -C "$NODE_SRC" status --porcelain 2>/dev/null)" ] || NODE_DIRTY=true REPO_BRANCH="$(git -C "$ROOT" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)" REPO_COMMIT="$(git -C "$ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)" @@ -109,9 +110,9 @@ rsync -a --exclude 'target' --exclude 'target-*' --exclude '.DS_Store' "$ROOT/ig echo "packing proto-cuda (without nvrtc/redist)" rsync -a --exclude 'nvrtc/redist' --exclude '.DS_Store' --exclude '*.exe' --exclude '*.dll' "$ROOT/proto-cuda/" "$STAGE/proto-cuda/" -python3 - "$STAGE/manifest.json" "$NODE_BRANCH" "$NODE_COMMIT" "$NODE_DIRTY" "${NODE_SRC#"$ROOT"/}" "$REPO_BRANCH" "$REPO_COMMIT" "$REPO_DIRTY" "$APP_VERSION" "$WITH_APP" "$NODE_TESTS" "$APP_TESTS" "${WITH_NODE:-1}" "$NODE_COMMIT_FULL" <<'PY' +python3 - "$STAGE/manifest.json" "$NODE_BRANCH" "$NODE_COMMIT" "$NODE_DIRTY" "${NODE_SRC#"$ROOT"/}" "$REPO_BRANCH" "$REPO_COMMIT" "$REPO_DIRTY" "$APP_VERSION" "$WITH_APP" "$NODE_TESTS" "$APP_TESTS" "${WITH_NODE:-1}" "$NODE_COMMIT_FULL" "$NODE_COMMIT_TIME" <<'PY' import json, sys, datetime -out, nb, nc, nd, ns, rb, rc, rd, av, with_app, node_tests, app_tests, with_node, ncf = sys.argv[1:15] +out, nb, nc, nd, ns, rb, rc, rd, av, with_app, node_tests, app_tests, with_node, ncf, nct = sys.argv[1:16] builds = [{"dir": "node", "packages": ["kaspad", "igneum-miner"], "features": ["kaspad/igneum-pow"], "bins": ["igneumd", "igneum-miner"], "targets": ["linux", "windows"]}] if with_node == "1" else [] tests = [] if node_tests.strip() and with_node == "1": @@ -122,7 +123,7 @@ if with_app == "1": tests.append({"dir": "app/igneum-app", "packages": app_tests.split()}) m = { "created_at": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "node": {"branch": nb, "commit": nc, "commit_full": ncf, "dirty": nd == "true", "source": ns}, + "node": {"branch": nb, "commit": nc, "commit_full": ncf, "commit_time": int(nct) if nct.isdigit() else 0, "dirty": nd == "true", "source": ns}, "repo": {"branch": rb, "commit": rc, "dirty": rd == "true"}, "app_version": av if with_app == "1" else "", "builds": builds, diff --git a/proto-cuda/windows-node/cross-build.sh b/proto-cuda/windows-node/cross-build.sh index 209fbe426..fc3efc586 100755 --- a/proto-cuda/windows-node/cross-build.sh +++ b/proto-cuda/windows-node/cross-build.sh @@ -40,6 +40,9 @@ export BINDGEN_EXTRA_CLANG_ARGS_x86_64_pc_windows_gnu="--target=x86_64-w64-mingw # byte-level difference between two builds of one tree on igneum-build-1); the box (tools/cross-remote.sh) and the PC job carry it too export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS="-C link-arg=-static -C link-arg=-static-libgcc -C link-arg=-static-libstdc++ -C link-arg=-Wl,--no-insert-timestamp" cd "$NODE" +# reproducible (main, 6 October 2026, the 0.3.14 repro): the commit's author time for mimalloc's __DATE__/__TIME__, UTC, and ONE +# target dir per target (CARGO_TARGET_DIR above is fixed, never a per-run name; prost embeds OUT_DIR) +SOURCE_DATE_EPOCH=$(git log -1 --format=%at HEAD); export SOURCE_DATE_EPOCH TZ=UTC # The commit hash (6 October 2026, found on igneum-build-1): kaspa-build-info's build.rs embeds the commit only when .git is # a DIRECTORY and HEAD is a symbolic ref to a branch file, and once it has found nothing it emits no rerun-if-changed, so # cargo never runs it again in this target dir. Two steps: clean that one crate when the commit changed since the last build diff --git a/tools/build-remote.sh b/tools/build-remote.sh index 73deaf326..f63618bf3 100755 --- a/tools/build-remote.sh +++ b/tools/build-remote.sh @@ -12,6 +12,15 @@ # tools/build-remote.sh --jobs 48 -- check # tools/build-remote.sh --target-dir target-exp -- build --release another persistent target dir on the box # tools/build-remote.sh --no-fetch -- clippy --all-targets nothing comes back (tests, check, clippy) +# tools/build-remote.sh --self-test-repro [--full] from a fork worktree: igneum-miner built twice a minute +# apart without sccache into one target dir must give one +# sha256, and a per-run target path must not (prost's +# OUT_DIR); --full adds kaspad with libmimalloc-sys +# recompiled between the builds (mimalloc's __DATE__), with +# and without SOURCE_DATE_EPOCH. Takes 2 to 6 min on the box. +# +# Reproducible: every command runs with SOURCE_DATE_EPOCH = the commit's author time and TZ=UTC, into ONE fixed target directory per +# target (lib.sh bs_repro_env; main's rule of 6 October 2026 from the 0.3.14 repro: prost embeds OUT_DIR, mimalloc embeds the date). # # Defaults by crate: a fork worktree builds `-p kaspad -p igneum-miner --features kaspad/igneum-pow` in release and fetches # target/release/{igneumd,igneum-miner} (what packaging/README-ship.md and infra/cross expect); app/igneum-app builds release @@ -36,7 +45,8 @@ BS_TOOL=build-remote # shellcheck source=../infra/build-server/lib.sh . "$HERE/../infra/build-server/lib.sh" -JOBS="${JOBS:-}"; # empty = the box decides: 90 alone, 45 beside another slot holder (remote-run.sh, main's ruling 6 Oct 2026) OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=() +# JOBS empty = the box decides: 90 alone, 45 beside another slot holder (remote-run.sh, main's ruling 6 Oct 2026) +JOBS="${JOBS:-}"; OUT=""; ARTEFACTS=""; TARGET_DIR="target"; FETCH=1; CARGO_ARGS=(); SELFTEST=0; FULL=0 while [ $# -gt 0 ]; do case "$1" in --jobs) JOBS="$2"; shift 2 ;; @@ -44,6 +54,8 @@ while [ $# -gt 0 ]; do --artefacts) ARTEFACTS="$2"; ARTEFACTS_SET=1; shift 2 ;; --target-dir) TARGET_DIR="$2"; shift 2 ;; --no-fetch) FETCH=0; shift ;; + --self-test-repro) SELFTEST=1; shift ;; + --full) FULL=1; shift ;; --) shift; CARGO_ARGS=("$@"); break ;; -h|--help) sed -n '2,32p' "$0"; exit 0 ;; *) CARGO_ARGS=("$@"); break ;; @@ -55,6 +67,40 @@ CARGO_ARGS_GIVEN=""; [ "${#CARGO_ARGS[@]}" -gt 0 ] && CARGO_ARGS_GIVEN=1 bs_host bs_context +if [ "$SELFTEST" = 1 ]; then + [ "$BS_KIND" = node ] || bs_die "--self-test-repro runs from a fork worktree (igneum-miner and kaspad live there)" + bs_toolchain_check; bs_sync_sources + # the remote script: no sccache (RUSTC_WRAPPER empty overrides the box's cargo config), the env from bs_repro_env, one fixed target + # dir `target-repro`; between the two builds the generated-code crate (prost, OUT_DIR) is cleaned so it is regenerated, and the + # clock is let past the next minute boundary so a __DATE__/__TIME__ stamp would differ + # RUSTC_WRAPPER=/usr/bin/env is a true pass-through: cargo treats an EMPTY value as unset and would fall back to the box's + # configured sccache, which is what hid both classes in the 0.3.14 repro + cmd="$(bs_repro_env)export RUSTC_WRAPPER=/usr/bin/env; set -u +sha() { sha256sum \"\$1\" | cut -c1-64; } +wait_minute() { local m0; m0=\$(date +%M); while [ \"\$(date +%M)\" = \"\$m0\" ]; do sleep 2; done; } +build() { local f=\"\"; [ \"\$2\" = kaspad ] && f=\"--features kaspad/igneum-pow\"; CARGO_TARGET_DIR=\"\$1\" cargo build --release -p \"\$2\" \$f > /tmp/repro-build.log 2>&1 || { echo \"self-test: cargo build -p \$2 into \$1 FAILED:\"; tail -5 /tmp/repro-build.log; exit 1; }; } +rc=0 +echo \"self-test: SOURCE_DATE_EPOCH=\$SOURCE_DATE_EPOCH TZ=\$TZ, RUSTC_WRAPPER=\$RUSTC_WRAPPER (sccache off), \$(rustc --version)\" +build target-repro igneum-miner; a=\$(sha target-repro/release/igneum-miner) +wait_minute; CARGO_TARGET_DIR=target-repro cargo clean -q --release -p kaspa-grpc-core -p igneum-miner; build target-repro igneum-miner; b=\$(sha target-repro/release/igneum-miner) +if [ \"\$a\" = \"\$b\" ]; then echo \"self-test: igneum-miner twice a minute apart, one target dir: MATCH \$a\"; else echo \"self-test: igneum-miner DIFFERS across a minute: \$a vs \$b\"; rc=1; fi +build target-repro-\$\$ igneum-miner; c=\$(sha target-repro-\$\$/release/igneum-miner); rm -rf target-repro-\$\$ +if [ \"\$a\" != \"\$c\" ]; then echo \"self-test: a per-run target path gives a different igneum-miner (prost OUT_DIR), as expected: \$c\"; else echo \"self-test: a per-run target path gave the SAME bytes; the OUT_DIR class no longer reproduces (the rule still stands)\"; fi +if [ $FULL = 1 ]; then + build target-repro kaspad; d=\$(sha target-repro/release/igneumd) + wait_minute; CARGO_TARGET_DIR=target-repro cargo clean -q --release -p libmimalloc-sys -p kaspad; build target-repro kaspad; e=\$(sha target-repro/release/igneumd) + if [ \"\$d\" = \"\$e\" ]; then echo \"self-test: kaspad with mimalloc recompiled a minute later: MATCH \$d\"; else echo \"self-test: kaspad DIFFERS with mimalloc recompiled: \$d vs \$e\"; rc=1; fi + unset SOURCE_DATE_EPOCH; wait_minute; CARGO_TARGET_DIR=target-repro cargo clean -q --release -p libmimalloc-sys -p kaspad; build target-repro kaspad; f=\$(sha target-repro/release/igneumd) + if [ \"\$d\" != \"\$f\" ]; then echo \"self-test: without SOURCE_DATE_EPOCH kaspad differs a minute later (mimalloc __DATE__), as expected: \$f\"; else echo \"self-test: without SOURCE_DATE_EPOCH kaspad was still identical (mimalloc's stamp did not move this time)\"; fi +fi +( exit \$rc )" + BR_KIND=check BR_COMMAND="self-test-repro" BR_TARGET=x86_64-unknown-linux-gnu BR_ARTEFACTS=""; export BR_KIND BR_COMMAND BR_TARGET BR_ARTEFACTS + set +e; bs_remote_run "$BS_REMOTE_CRATE" "$BS_WT/$BS_CRATE_REL self-test-repro" "$cmd" 2>&1 | grep -E '^self-test:|RESULT' | sed 's/^/ /' >&2; rc=${PIPESTATUS[0]}; set -e + bs_wt_unlock + [ "$rc" = 0 ] && bs_log "self-test-repro passed" || bs_die "self-test-repro FAILED (rc $rc)" + exit 0 +fi + # defaults per crate case "$BS_KIND:$BS_CRATE_REL" in node:*) @@ -85,7 +131,7 @@ pre="" if [ "$BS_KIND" = node ] && [ "${CARGO_ARGS[0]}" = build ]; then pre="[ \"\$(cat '.build-remote-sha-$TARGET_DIR' 2>/dev/null)\" = '$BS_SHA' ] || CARGO_TARGET_DIR='$TARGET_DIR' cargo clean -q --release -p kaspa-build-info 2>/dev/null; " fi -cmd="${pre}CARGO_TARGET_DIR='$TARGET_DIR' cargo $(printf '%q ' "${CARGO_ARGS[@]}")${JOBS:+-j $JOBS} 2>&1 | tee -a '$BS_REMOTE_WT/.build-remote.log'; rc=\${PIPESTATUS[0]}; [ \$rc = 0 ] && echo '$BS_SHA' > '.build-remote-sha-$TARGET_DIR'; ( exit \$rc )" # a subshell exit: the runner reads \$? and still prints its RESULT line +cmd="$(bs_repro_env)${pre}CARGO_TARGET_DIR='$TARGET_DIR' cargo $(printf '%q ' "${CARGO_ARGS[@]}")${JOBS:+-j $JOBS} 2>&1 | tee -a '$BS_REMOTE_WT/.build-remote.log'; rc=\${PIPESTATUS[0]}; [ \$rc = 0 ] && echo '$BS_SHA' > '.build-remote-sha-$TARGET_DIR'; ( exit \$rc )" # a subshell exit: the runner reads \$? and still prints its RESULT line label="$BS_WT/$BS_CRATE_REL cargo ${CARGO_ARGS[*]}" BR_KIND=$(bs_kind build-remote "${CARGO_ARGS[0]}"); BR_COMMAND="cargo ${CARGO_ARGS[*]}"; BR_TARGET=x86_64-unknown-linux-gnu for ((i = 0; i < ${#CARGO_ARGS[@]}; i++)); do [ "${CARGO_ARGS[$i]}" = --target ] && BR_TARGET="${CARGO_ARGS[$((i + 1))]:-}"; done diff --git a/tools/cross-remote.sh b/tools/cross-remote.sh index 258f1dc6a..cf5509601 100755 --- a/tools/cross-remote.sh +++ b/tools/cross-remote.sh @@ -81,7 +81,7 @@ export BINDGEN_EXTRA_CLANG_ARGS_x86_64_pc_windows_gnu="--target=x86_64-w64-mingw echo "cross-remote: $(x86_64-w64-mingw32-gcc-posix --version | head -1); libclang $LLVM_LIB"' pre="" # the same kaspa-build-info clean on a new commit as build-remote.sh (the Windows target has its own fingerprint) [ "$BS_KIND" = node ] && pre="[ \"\$(cat '.cross-remote-sha-$TARGET_DIR' 2>/dev/null)\" = '$BS_SHA' ] || CARGO_TARGET_DIR='$TARGET_DIR' cargo clean -q --release -p kaspa-build-info --target $TARGET 2>/dev/null; " -cmd="$env_block +cmd="$(bs_repro_env)$env_block ${pre}CARGO_TARGET_DIR='$TARGET_DIR' cargo $(printf '%q ' "${CARGO_ARGS[@]}")${JOBS:+-j $JOBS} 2>&1 | tee -a '$BS_REMOTE_WT/.cross-remote.log'; rc=\${PIPESTATUS[0]}; [ \$rc = 0 ] && echo '$BS_SHA' > '.cross-remote-sha-$TARGET_DIR' for exe in $EXES; do f='$TARGET_DIR/$TARGET/release/'\$exe; [ -f \"\$f\" ] && echo \"cross-remote: DLLS \$exe: \$(x86_64-w64-mingw32-objdump -p \"\$f\" | awk '/DLL Name/ { print \$3 }' | sort -u | tr '\n' ' ')\"; done ( exit \$rc )" # a subshell exit: the runner reads \$? and still prints its RESULT line diff --git a/tools/workers-remote.sh b/tools/workers-remote.sh index b97bdc052..3cee0acd6 100755 --- a/tools/workers-remote.sh +++ b/tools/workers-remote.sh @@ -28,7 +28,7 @@ BS_LOCAL_DIRS="proto-cuda proto-opencl"; BS_VENDOR_REPOS="" bs_log "workers from $BS_WT at $BS_SHA ($BS_BRANCH) -> $BS_HOST:$BS_REMOTE_WT (proto-cuda, proto-opencl)" bs_sync_sources PLACEHOLDER=proto-cuda/packs/igneum-devnet-v4-epoch0 # the OpenCL worker's compile-time pack, as build-workers-linux.sh -cmd='. /etc/profile.d/igneum-build.sh +cmd="$(bs_repro_env)"'. /etc/profile.d/igneum-build.sh CUDA=$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1); [ -n "$CUDA" ] || { echo "no CUDA headers under /usr/local/cuda-12.*: provision.sh step_cuda"; exit 2; } [ -f /usr/include/CL/cl.h ] || { echo "no /usr/include/CL/cl.h: apt opencl-c-headers"; exit 2; } mkdir -p out-workers-box