Box: the night battery (02:00 London timer, one slot, every suite, fuzz, sims, harnesses, clippy, audit, report on branch night-battery) and the reproducible-build check
- infra/build-server/night/night-battery.sh: checkout of master and the newest release-*-node fork branch under
/srv/builds/_night, cargo test --release --no-fail-fast per crate (repo and fork), igneum-pow fuzz at 10x, the three
simulators in full, the fast-time harnesses (finality-attacks, harness s3 s4, exec-sync reorg) on binaries built into
target-integration, clippy per crate dir, cargo audit per Cargo.lock; docs/benchmarks/night/<date>.md with a pass/fail
table and "new since last night"; committed as igneum-labs on night-battery and pushed to the mirror, never master.
NIGHT_SUBSET=1 is the dry-run subset. The unit runs it through remote-run.sh so the slot spans the invocation.
- igneum-night-battery.{service,timer}: 02:00 Europe/London, User build, Nice 19, idle IO, 8 h limit, not Persistent.
- provision.sh: step_cargo_tools (cargo-audit), step_night (files from the mirror at NIGHT_REF, timer enabled), innoextract.
- tools/repro/rebuild-release.sh + infra/build-server/repro/rebuild-on-box.sh: pins from docs/plans/release-<v>.md
("(node <sha>, app <sha>)"), shipped hashes from the public downloads (the HiveOS tarball, the installer via
innoextract) or --shipped, a clean clone of repo and fork on the box, two passes per target without sccache under build
slots, MATCH or DIFFER per artefact against the shipped bytes and against the other pass, with the reason read off the
binaries (glibc version needed, PE timestamp, commit string); evidence into docs/evidence/reproduced/<version>.md.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
parent
9c553e0725
commit
3a6231ff2b
6 changed files with 504 additions and 0 deletions
34
infra/build-server/night/igneum-night-battery.service
Normal file
34
infra/build-server/night/igneum-night-battery.service
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
# igneum-build-1: the night battery (infra/build-server/night/night-battery.sh) under one build slot through remote-run.sh.
|
||||
# Installed by infra/build-server/provision.sh step_night; started by igneum-night-battery.timer at 02:00 Europe/London.
|
||||
[Unit]
|
||||
Description=Igneum night battery (every test suite, fuzz, simulators, harnesses, clippy, audit; report on branch night-battery)
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=build
|
||||
Group=build
|
||||
Nice=19
|
||||
IOSchedulingClass=idle
|
||||
WorkingDirectory=/srv/builds/_night
|
||||
Environment=BR_DIR=/srv/builds/_night
|
||||
Environment=BR_CMD=/srv/builds/_bin/night-battery.sh
|
||||
Environment="BR_LABEL=night battery; agent=night"
|
||||
Environment=BR_TOOL=night-battery
|
||||
Environment=BR_KIND=other
|
||||
Environment=BR_WT=_night
|
||||
Environment=BR_CRATE=.
|
||||
Environment=BR_BRANCH=master
|
||||
Environment=BR_SHA=
|
||||
Environment=BR_AGENT=night
|
||||
Environment="BR_COMMAND=night-battery.sh (suites, fuzz, sims, harness, clippy, audit)"
|
||||
Environment=BR_TARGET=x86_64-unknown-linux-gnu
|
||||
Environment=IGNEUM_AGENT=night
|
||||
ExecStart=/bin/bash /srv/builds/_bin/remote-run.sh
|
||||
TimeoutStartSec=8h
|
||||
StandardOutput=append:/srv/builds/_log/night-battery.log
|
||||
StandardError=append:/srv/builds/_log/night-battery.log
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
12
infra/build-server/night/igneum-night-battery.timer
Normal file
12
infra/build-server/night/igneum-night-battery.timer
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# igneum-build-1: 02:00 London every night (the box's clock is Europe/Berlin; the time zone is named here, so BST and GMT both
|
||||
# land at 02:00 UK). Not Persistent: a missed night is not run during the day.
|
||||
[Unit]
|
||||
Description=Igneum night battery at 02:00 Europe/London
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 02:00:00 Europe/London
|
||||
Persistent=false
|
||||
AccuracySec=1min
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
207
infra/build-server/night/night-battery.sh
Executable file
207
infra/build-server/night/night-battery.sh
Executable file
|
|
@ -0,0 +1,207 @@
|
|||
#!/usr/bin/env bash
|
||||
# The night battery on igneum-build-1 (6 October 2026): one invocation, one build slot, every test the repo and the fork have.
|
||||
# Runs ON the box as user build at 02:00 Europe/London (igneum-night-battery.timer) through remote-run.sh, which holds the
|
||||
# slot for the whole run, sets CARGO_BUILD_JOBS (90 alone, 45 beside another build) and writes the JSONL line. By hand:
|
||||
# /srv/builds/_bin/night-battery.sh the full battery (hours)
|
||||
# NIGHT_SUBSET=1 /srv/builds/_bin/night-battery.sh the dry-run subset (igneum-pow suite and fuzz, two fork crates, quick sims, clippy and audit on igneum-pow)
|
||||
# NIGHT_NODE_BRANCH=release-0.3.15-node ... the fork branch (default: the newest release-*-node on the mirror)
|
||||
# NIGHT_SKIP="harness sims" ... skip stages by name
|
||||
# Stages, each a row (pass, FAIL, skip) with seconds and a detail:
|
||||
# checkout /srv/builds/_night/igneum from /srv/igneum.git master (the battery re-execs itself from that checkout, so the
|
||||
# script that runs is master's), vendor/igneum-node from /srv/igneum-node.git at the fork branch
|
||||
# suites cargo test --release --no-fail-fast per crate: the repo's crates (igneum-pow, igneum-census, pool, proto-vdf,
|
||||
# app/igneum-app, every member of proving/igneum-prove) and every workspace member of the fork (kaspad with
|
||||
# --features igneum-pow); the pass and fail counts are read from the `test result:` lines
|
||||
# fuzz igneum-pow's two fuzz tests at 10x their default (IGNEUM_MIXER_FUZZ and IGNEUM_SCRATCH_FUZZ 2000)
|
||||
# sims sim/finality_sim.py, sim/finality_v2.py and sim/difficulty/sim.py in full (the subset runs --quick)
|
||||
# harness the fork's igneumd, igneum-miner, igneum-harness-sim and igneum-p2p-probe built into target-integration, then
|
||||
# tools/finality-attacks/run.mjs --fast-time, tools/harness/run.mjs s3 s4 --fast-time --no-bench-log and
|
||||
# tools/exec-sync/reorg.mjs (loopback ports 27200+, 27800+, 29870+; nothing of the live devnet)
|
||||
# clippy cargo clippy --release --all-targets per crate dir (warnings counted; a row fails on an error)
|
||||
# audit cargo audit in every directory with a Cargo.lock
|
||||
# report docs/benchmarks/night/<date>.md (a pass/fail table and "new since last night" against the newest earlier
|
||||
# report), committed as igneum-labs on branch night-battery (based on master, earlier reports carried over) and
|
||||
# pushed to /srv/igneum.git night-battery; main merges (`git fetch build night-battery` on the Mac). Never master.
|
||||
set -uo pipefail
|
||||
_slots_env="${IGNEUM_BUILD_SLOTS_DIR:-}"; [ -f /etc/profile.d/igneum-build.sh ] && . /etc/profile.d/igneum-build.sh; [ -n "$_slots_env" ] && IGNEUM_BUILD_SLOTS_DIR="$_slots_env"
|
||||
NIGHT_ROOT="${NIGHT_ROOT:-/srv/builds/_night}"; REPO_MIRROR=/srv/igneum.git; NODE_MIRROR=/srv/igneum-node.git
|
||||
SUBSET="${NIGHT_SUBSET:-0}"; SKIP=" ${NIGHT_SKIP:-} "
|
||||
DATE=$(date -u +%Y-%m-%d); STAMP=$(date -u +%Y%m%dT%H%M%SZ); T_ALL=$(date +%s)
|
||||
REPORT_NAME="$DATE$( [ "$SUBSET" = 1 ] && echo -dryrun ).md"
|
||||
LOGDIR="$NIGHT_ROOT/logs/$STAMP"; mkdir -p "$LOGDIR"
|
||||
ROWS="$LOGDIR/rows.tsv"; : > "$ROWS"
|
||||
say() { printf '%s night: %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; }
|
||||
row() { printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "$4" "$5" >> "$ROWS"; say "$1 | $2 | $3 | ${4}s | $5"; } # stage name status secs detail
|
||||
skip() { case "$SKIP" in *" $1 "*) return 0 ;; esac; return 1; }
|
||||
cargo_jobs="${CARGO_BUILD_JOBS:-90}"
|
||||
|
||||
# checkout
|
||||
IG="$NIGHT_ROOT/igneum"; FORK="$IG/vendor/igneum-node"
|
||||
t0=$(date +%s)
|
||||
if [ ! -d "$IG/.git" ]; then git clone -q "$REPO_MIRROR" "$IG" || { row checkout repo FAIL 0 "clone failed"; exit 1; }; fi
|
||||
git -C "$IG" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' || { row checkout repo FAIL 0 "fetch failed"; exit 1; }
|
||||
git -C "$IG" checkout -q -- . 2>/dev/null; git -C "$IG" clean -qfd -e target -e 'target-*' -e vendor
|
||||
git -C "$IG" checkout -q -B night-battery origin/master
|
||||
# earlier reports not yet merged into master ride along
|
||||
git -C "$IG" checkout -q origin/night-battery -- docs/benchmarks/night 2>/dev/null || true
|
||||
REPO_SHA=$(git -C "$IG" rev-parse --short HEAD)
|
||||
if [ "${NIGHT_REEXEC:-0}" != 1 ] && [ -f "$IG/infra/build-server/night/night-battery.sh" ] && ! cmp -s "$0" "$IG/infra/build-server/night/night-battery.sh"; then
|
||||
say "re-exec from master's copy ($REPO_SHA)"; NIGHT_REEXEC=1 exec bash "$IG/infra/build-server/night/night-battery.sh"
|
||||
fi
|
||||
NODE_BRANCH="${NIGHT_NODE_BRANCH:-$(git -C "$NODE_MIRROR" branch --list 'release-*-node' | tr -d ' *' | sort -V | tail -1)}"
|
||||
[ -n "$NODE_BRANCH" ] || NODE_BRANCH=master
|
||||
mkdir -p "$IG/vendor"
|
||||
if [ ! -d "$FORK/.git" ]; then git clone -q --no-checkout "$NODE_MIRROR" "$FORK" || { row checkout fork FAIL 0 "clone failed"; exit 1; }; fi
|
||||
git -C "$FORK" fetch -q origin '+refs/heads/*:refs/remotes/origin/*'
|
||||
git -C "$FORK" checkout -q -- . 2>/dev/null; git -C "$FORK" clean -qfd -e target -e 'target-*'
|
||||
git -C "$FORK" checkout -q -B "$NODE_BRANCH" "origin/$NODE_BRANCH" || { row checkout fork FAIL 0 "no branch $NODE_BRANCH"; exit 1; }
|
||||
NODE_SHA=$(git -C "$FORK" rev-parse --short HEAD)
|
||||
row checkout "repo master $REPO_SHA, fork $NODE_BRANCH $NODE_SHA" pass $(( $(date +%s) - t0 )) "$IG; jobs $cargo_jobs; subset $SUBSET"
|
||||
|
||||
# helpers
|
||||
run_to() { # <log> <timeout secs> <cmd...>: runs in the current dir, returns the exit code, 124 on timeout
|
||||
local log="$1" to="$2"; shift 2
|
||||
timeout --signal=TERM --kill-after=60 "$to" "$@" > "$log" 2>&1; echo $?
|
||||
}
|
||||
counts() { # pass/fail totals from cargo test output
|
||||
awk '/^test result:/ { for (i = 1; i <= NF; i++) { if ($(i+1) == "passed;") p += $i; if ($(i+1) == "failed;") f += $i } } END { printf "%d passed, %d failed", p, f }' "$1"
|
||||
}
|
||||
suite() { # <dir> <label> <cargo test args...>
|
||||
local dir="$1" label="$2"; shift 2
|
||||
local log="$LOGDIR/suite-$(echo "$label" | tr '/ ' '__').log" t0 rc c
|
||||
t0=$(date +%s)
|
||||
rc=$(cd "$dir" && run_to "$log" 2400 cargo test --release --no-fail-fast "$@")
|
||||
c=$(counts "$log")
|
||||
if [ "$rc" = 0 ]; then row suite "$label" pass $(( $(date +%s) - t0 )) "$c"
|
||||
elif [ "$rc" = 124 ]; then row suite "$label" FAIL $(( $(date +%s) - t0 )) "TIMEOUT 40 min; $c"
|
||||
elif grep -q '^error\(\[E[0-9]*\]\)\?:' "$log" && ! grep -q '^test result:' "$log"; then row suite "$label" FAIL $(( $(date +%s) - t0 )) "did not compile: $(grep -m1 '^error' "$log" | cut -c1-120)"
|
||||
else row suite "$label" FAIL $(( $(date +%s) - t0 )) "$c; failed: $(grep -E '^ [a-z_:]+' "$log" | grep -v '^ Finished\|^ Running' | head -3 | tr -d ' ' | tr '\n' ' ' | cut -c1-160)"; fi
|
||||
}
|
||||
|
||||
# suites
|
||||
if ! skip suites; then
|
||||
if [ "$SUBSET" = 1 ]; then
|
||||
suite "$IG/igneum-pow" igneum-pow
|
||||
for c in kaspa-pow igneum-miner; do suite "$FORK" "fork/$c" -p "$c"; done
|
||||
else
|
||||
for d in igneum-pow igneum-census pool proto-vdf app/igneum-app; do [ -f "$IG/$d/Cargo.toml" ] && suite "$IG/$d" "$d"; done
|
||||
if [ -f "$IG/proving/igneum-prove/Cargo.toml" ]; then
|
||||
for m in $(cd "$IG/proving/igneum-prove" && cargo metadata --no-deps --format-version 1 2>/dev/null | python3 -c 'import json,sys; print(" ".join(p["name"] for p in json.load(sys.stdin)["packages"]))'); do
|
||||
case "$m" in *program*|*aggregator*) continue ;; esac # the guests are pinned ELFs, built only by pin-guests.sh
|
||||
suite "$IG/proving/igneum-prove" "proving/$m" -p "$m"
|
||||
done
|
||||
fi
|
||||
for m in $(cd "$FORK" && cargo metadata --no-deps --format-version 1 2>/dev/null | python3 -c 'import json,sys; print(" ".join(sorted(p["name"] for p in json.load(sys.stdin)["packages"])))'); do
|
||||
case "$m" in *wasm*) continue ;; esac # browser targets, no host test suite
|
||||
if [ "$m" = kaspad ]; then suite "$FORK" "fork/kaspad" -p kaspad --features igneum-pow; else suite "$FORK" "fork/$m" -p "$m"; fi
|
||||
done
|
||||
fi
|
||||
else row suite all skip 0 "NIGHT_SKIP"; fi
|
||||
|
||||
# fuzz
|
||||
if ! skip fuzz; then
|
||||
n=2000; [ "$SUBSET" = 1 ] && n=200
|
||||
t0=$(date +%s); log="$LOGDIR/fuzz.log"
|
||||
rc=$(cd "$IG/igneum-pow" && IGNEUM_MIXER_FUZZ=$n IGNEUM_SCRATCH_FUZZ=$n run_to "$log" 7200 cargo test --release --test mixer --test scratch -- fuzz --nocapture)
|
||||
[ "$rc" = 0 ] && row fuzz "igneum-pow mixer+scratch x$n" pass $(( $(date +%s) - t0 )) "$(grep -h '^fuzz:' "$log" | tr '\n' ';' | cut -c1-200)" || row fuzz "igneum-pow mixer+scratch x$n" FAIL $(( $(date +%s) - t0 )) "rc $rc: $(grep -m1 -E 'panicked|error' "$log" | cut -c1-160)"
|
||||
else row fuzz igneum-pow skip 0 "NIGHT_SKIP"; fi
|
||||
|
||||
# sims
|
||||
if ! skip sims; then
|
||||
q=""; [ "$SUBSET" = 1 ] && q="--quick"
|
||||
t0=$(date +%s); rc=$(cd "$IG/sim" && run_to "$LOGDIR/sim-finality.log" 600 python3 finality_sim.py); [ "$rc" = 0 ] && row sim finality_sim.py pass $(( $(date +%s) - t0 )) "$(grep -c '^|' "$LOGDIR/sim-finality.log") table lines" || row sim finality_sim.py FAIL $(( $(date +%s) - t0 )) "rc $rc"
|
||||
t0=$(date +%s); rc=$(cd "$IG/sim" && run_to "$LOGDIR/sim-finality-v2.log" 3600 python3 finality_v2.py $q); [ "$rc" = 0 ] && row sim "finality_v2.py $q" pass $(( $(date +%s) - t0 )) "$(grep -c '^|' "$LOGDIR/sim-finality-v2.log") table lines" || row sim "finality_v2.py $q" FAIL $(( $(date +%s) - t0 )) "rc $rc"
|
||||
t0=$(date +%s); rc=$(cd "$IG/sim/difficulty" && run_to "$LOGDIR/sim-difficulty.log" 5400 python3 sim.py $q); [ "$rc" = 0 ] && row sim "difficulty/sim.py $q" pass $(( $(date +%s) - t0 )) "$(grep -c '^|' "$LOGDIR/sim-difficulty.log") table lines" || row sim "difficulty/sim.py $q" FAIL $(( $(date +%s) - t0 )) "rc $rc"
|
||||
else row sim all skip 0 "NIGHT_SKIP"; fi
|
||||
|
||||
# harness (fast time)
|
||||
if ! skip harness && [ "$SUBSET" != 1 ]; then
|
||||
t0=$(date +%s); log="$LOGDIR/harness-build.log"
|
||||
rc=$(cd "$FORK" && CARGO_TARGET_DIR=target-integration run_to "$log" 3600 cargo build --release -p kaspad -p igneum-miner -p igneum-harness-sim -p igneum-p2p-probe --features kaspad/igneum-pow)
|
||||
if [ "$rc" = 0 ]; then
|
||||
row harness build pass $(( $(date +%s) - t0 )) "igneumd igneum-miner igneum-harness-sim igneum-p2p-probe at $NODE_SHA"
|
||||
REL="$FORK/target-integration/release"
|
||||
t0=$(date +%s); rc=$(cd "$IG" && IGNEUMD="$REL/igneumd" IGNEUM_MINER="$REL/igneum-miner" IGNEUM_NODE_ROOT="$IG/" run_to "$LOGDIR/harness-finality.log" 3600 node tools/finality-attacks/run.mjs --fast-time)
|
||||
[ "$rc" = 0 ] && row harness "finality-attacks --fast-time" pass $(( $(date +%s) - t0 )) "$(grep -c -i 'PASS' "$LOGDIR/harness-finality.log") PASS lines" || row harness "finality-attacks --fast-time" FAIL $(( $(date +%s) - t0 )) "rc $rc: $(grep -m1 -i -E 'FAIL|error' "$LOGDIR/harness-finality.log" | cut -c1-160)"
|
||||
t0=$(date +%s); rc=$(cd "$IG" && IGNEUM_HARNESS_TARGET="$REL" run_to "$LOGDIR/harness-s3s4.log" 3600 node tools/harness/run.mjs s3 s4 --fast-time --no-bench-log)
|
||||
[ "$rc" = 0 ] && row harness "harness s3 s4 --fast-time" pass $(( $(date +%s) - t0 )) "$(grep -c -i 'PASS' "$LOGDIR/harness-s3s4.log") PASS lines" || row harness "harness s3 s4 --fast-time" FAIL $(( $(date +%s) - t0 )) "rc $rc: $(grep -m1 -i -E 'FAIL|error' "$LOGDIR/harness-s3s4.log" | cut -c1-160)"
|
||||
t0=$(date +%s); rc=$(cd "$IG" && IGNEUM_EXEC_BIN="$REL" run_to "$LOGDIR/harness-exec-reorg.log" 3600 node tools/exec-sync/reorg.mjs)
|
||||
[ "$rc" = 0 ] && row harness "exec-sync reorg" pass $(( $(date +%s) - t0 )) "$(grep -c -i 'PASS\|ok' "$LOGDIR/harness-exec-reorg.log") ok lines" || row harness "exec-sync reorg" FAIL $(( $(date +%s) - t0 )) "rc $rc: $(grep -m1 -i -E 'FAIL|error|missing' "$LOGDIR/harness-exec-reorg.log" | cut -c1-160)"
|
||||
else row harness build FAIL $(( $(date +%s) - t0 )) "$(grep -m1 '^error' "$log" | cut -c1-160)"; fi
|
||||
else row harness all skip 0 "$( [ "$SUBSET" = 1 ] && echo subset || echo NIGHT_SKIP)"; fi
|
||||
|
||||
# clippy
|
||||
if ! skip clippy; then
|
||||
dirs="igneum-pow"; [ "$SUBSET" = 1 ] || dirs="igneum-pow igneum-census pool proto-vdf app/igneum-app proving/igneum-prove vendor/igneum-node"
|
||||
for d in $dirs; do
|
||||
[ -f "$IG/$d/Cargo.toml" ] || continue
|
||||
t0=$(date +%s); log="$LOGDIR/clippy-$(echo "$d" | tr '/' '_').log"
|
||||
feat=""; [ "$d" = vendor/igneum-node ] && feat="--features kaspad/igneum-pow"
|
||||
rc=$(cd "$IG/$d" && run_to "$log" 3600 cargo clippy --release --all-targets $feat)
|
||||
w=$(grep -c '^warning: ' "$log"); e=$(grep -c '^error' "$log")
|
||||
[ "$rc" = 0 ] && row clippy "$d" pass $(( $(date +%s) - t0 )) "$w warnings" || row clippy "$d" FAIL $(( $(date +%s) - t0 )) "rc $rc, $e errors, $w warnings: $(grep -m1 '^error' "$log" | cut -c1-120)"
|
||||
done
|
||||
else row clippy all skip 0 "NIGHT_SKIP"; fi
|
||||
|
||||
# audit
|
||||
if ! skip audit; then
|
||||
locks="igneum-pow/Cargo.lock vendor/igneum-node/Cargo.lock"; [ "$SUBSET" = 1 ] || locks=$(cd "$IG" && find . -name Cargo.lock -not -path '*/target*' -not -path './vendor/igneum-node/*' | sed 's|^\./||'; echo vendor/igneum-node/Cargo.lock)
|
||||
for l in $locks; do
|
||||
d=$(dirname "$l"); [ -f "$IG/$l" ] || continue
|
||||
t0=$(date +%s); log="$LOGDIR/audit-$(echo "$d" | tr '/' '_').log"
|
||||
rc=$(cd "$IG/$d" && run_to "$log" 600 cargo audit --color never)
|
||||
v=$(grep -c '^Crate:' "$log"); wn=$(grep -c -i '^warning:' "$log")
|
||||
[ "$rc" = 0 ] && row audit "$d" pass $(( $(date +%s) - t0 )) "$v vulnerabilities, $wn warnings" || row audit "$d" FAIL $(( $(date +%s) - t0 )) "rc $rc: $v vulnerabilities ($(grep -A1 '^Crate:' "$log" | grep -v '^--' | awk '{ print $2 }' | paste -sd, - | cut -c1-120)), $wn warnings"
|
||||
done
|
||||
else row audit all skip 0 "NIGHT_SKIP"; fi
|
||||
|
||||
# report
|
||||
OUTDIR="$IG/docs/benchmarks/night"; mkdir -p "$OUTDIR"; OUT="$OUTDIR/$REPORT_NAME"
|
||||
PREV=$(ls "$OUTDIR"/*.md 2>/dev/null | grep -v "/$REPORT_NAME$" | grep -v -- '-dryrun' | sort | tail -1)
|
||||
[ "$SUBSET" = 1 ] && PREV=$(ls "$OUTDIR"/*-dryrun.md 2>/dev/null | grep -v "/$REPORT_NAME$" | sort | tail -1)
|
||||
python3 - "$ROWS" "$OUT" "${PREV:-}" "$DATE" "$REPO_SHA" "$NODE_BRANCH" "$NODE_SHA" "$(( $(date +%s) - T_ALL ))" "$SUBSET" "$cargo_jobs" "$LOGDIR" <<'PY'
|
||||
import re, sys, os
|
||||
rows_f, out, prev, date, repo_sha, node_branch, node_sha, secs, subset, jobs, logdir = sys.argv[1:]
|
||||
rows = [l.rstrip("\n").split("\t") for l in open(rows_f) if l.strip()]
|
||||
def fmt(s):
|
||||
s = int(s); return f"{s // 60} min {s % 60:02d} s" if s >= 60 else f"{s} s"
|
||||
n_pass = sum(1 for r in rows if r[2] == "pass"); n_fail = sum(1 for r in rows if r[2] == "FAIL"); n_skip = sum(1 for r in rows if r[2] == "skip")
|
||||
lines = [f"# Night battery {date}{' (dry run, subset)' if subset == '1' else ''}", "",
|
||||
f"igneum-build-1, {fmt(secs)} wall, one build slot (CARGO_BUILD_JOBS {jobs}), repo master {repo_sha}, fork {node_branch} {node_sha}. "
|
||||
f"{n_pass} pass, {n_fail} FAIL, {n_skip} skip. Logs on the box: `{logdir}`. Written by `infra/build-server/night/night-battery.sh`.", "",
|
||||
"| Stage | What | Result | Time | Detail |", "|---|---|---|---|---|"]
|
||||
for st, what, res, t, det in rows:
|
||||
res_md = "**FAIL**" if res == "FAIL" else res
|
||||
lines.append(f"| {st} | {what} | {res_md} | {fmt(t)} | {det.replace('|', '/')} |")
|
||||
lines += ["", "## New since last night", ""]
|
||||
if prev and os.path.isfile(prev):
|
||||
old = {}
|
||||
for l in open(prev):
|
||||
m = re.match(r"\| (\w+) \| (.*?) \| (\*\*FAIL\*\*|pass|skip) \| (.*?) \| (.*) \|$", l.rstrip("\n"))
|
||||
if m: old[(m.group(1), m.group(2))] = m.group(3).strip("*")
|
||||
new = {(r[0], r[1]): r[2] for r in rows}
|
||||
changes = []
|
||||
for k, v in new.items():
|
||||
if k not in old: changes.append(f"- new row: {k[0]} {k[1]}: {v}")
|
||||
elif old[k] != v: changes.append(f"- {k[0]} {k[1]}: {old[k]} -> **{v}**")
|
||||
for k, v in old.items():
|
||||
if k not in new: changes.append(f"- gone: {k[0]} {k[1]} (was {v})")
|
||||
hdr = open(prev).read().split("\n")[2] if len(open(prev).read().split("\n")) > 2 else ""
|
||||
m = re.search(r"repo master (\w+), fork (\S+) (\w+)", hdr)
|
||||
if m and (m.group(1) != repo_sha or m.group(3) != node_sha):
|
||||
changes.insert(0, f"- commits moved: repo {m.group(1)} -> {repo_sha}, fork {m.group(2)} {m.group(3)} -> {node_branch} {node_sha}")
|
||||
lines.append(f"Against `{os.path.basename(prev)}`:")
|
||||
lines += changes if changes else ["- nothing: every row has the result it had"]
|
||||
else:
|
||||
lines.append("No earlier report to compare with: this is the first night.")
|
||||
open(out, "w").write("\n".join(lines) + "\n")
|
||||
print(f"{n_pass} pass, {n_fail} FAIL, {n_skip} skip; report {out}")
|
||||
PY
|
||||
# commit on night-battery, push to the mirror (never master)
|
||||
cd "$IG" && git add docs/benchmarks/night && \
|
||||
git -c user.name=igneum-labs -c user.email=337424239+[removed] commit -q -m "Night battery $DATE$( [ "$SUBSET" = 1 ] && echo ' (dry run)'): $(awk -F'\t' '$3 == "pass" { p++ } $3 == "FAIL" { f++ } END { printf "%d pass, %d FAIL", p, f }' "$ROWS") on master $REPO_SHA, fork $NODE_BRANCH $NODE_SHA
|
||||
|
||||
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>" && \
|
||||
git push -q --force origin night-battery:refs/heads/night-battery && say "pushed night-battery to $REPO_MIRROR ($(git rev-parse --short HEAD)); on the Mac: git fetch build night-battery" || say "commit or push FAILED"
|
||||
cat "$OUT"
|
||||
[ "$(awk -F'\t' '$3 == "FAIL"' "$ROWS" | wc -l)" = 0 ]
|
||||
|
|
@ -143,6 +143,8 @@ APT_PACKAGES=(
|
|||
# the GitHub Actions runner's .NET runtime needs libicu (bin/installdependencies.sh would install it); the two Python
|
||||
# simulators (sim/finality_v2.py, sim/difficulty/sim.py) need numpy, which ci.yml pip-installs on GitHub's runners
|
||||
libicu74 python3-numpy
|
||||
# innoextract: tools/repro reads the shipped igneumd.exe and igneum-miner.exe out of the public Inno Setup installer
|
||||
innoextract
|
||||
)
|
||||
step_apt() {
|
||||
local need=() p
|
||||
|
|
@ -528,6 +530,39 @@ step_runner() {
|
|||
|| ok runner "$svc active, runner $RUNNER_VERSION, $(as_runner "$cargo --version")"
|
||||
}
|
||||
|
||||
# cargo tools the night battery needs (infra/build-server/night): cargo-audit for the advisory check of every Cargo.lock
|
||||
step_cargo_tools() {
|
||||
local cargo="$BUILD_HOME/.cargo/bin/cargo" any=0
|
||||
if [ ! -x "$BUILD_HOME/.cargo/bin/cargo-audit" ]; then as_build "$cargo install cargo-audit --locked" >/dev/null 2>&1 || as_build "$cargo install cargo-audit" >/dev/null; any=1; fi
|
||||
[ "$any" = 1 ] && changed cargo-tools "$(as_build "$BUILD_HOME/.cargo/bin/cargo-audit --version")" || ok cargo-tools "$(as_build "$BUILD_HOME/.cargo/bin/cargo-audit --version")"
|
||||
}
|
||||
|
||||
# the night battery (infra/build-server/night): the script and remote-run.sh into /srv/builds/_bin, the two units, the timer
|
||||
# enabled. The files come out of the bare mirror /srv/igneum.git at NIGHT_REF (master; a branch while the work is unmerged),
|
||||
# so provision.sh stays one piped file and the box runs what the repository holds. The battery re-execs itself from master's
|
||||
# checkout at run time, so the copy here only has to be good enough to check out.
|
||||
NIGHT_REF="${NIGHT_REF:-master}"
|
||||
step_night() {
|
||||
local any=0 f tmp u
|
||||
install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" /srv/builds/_bin /srv/builds/_night /srv/builds/_log
|
||||
as_build "git -C /srv/igneum.git cat-file -e '$NIGHT_REF:infra/build-server/night/night-battery.sh'" 2>/dev/null || { log "night: $NIGHT_REF on /srv/igneum.git has no infra/build-server/night/night-battery.sh (push the branch, or NIGHT_REF=<branch>)"; return; }
|
||||
for f in infra/build-server/night/night-battery.sh infra/build-server/remote-run.sh; do
|
||||
tmp=$(mktemp); as_build "git -C /srv/igneum.git show '$NIGHT_REF:$f'" > "$tmp"
|
||||
if ! cmp -s "$tmp" "/srv/builds/_bin/$(basename "$f")"; then install -m 755 -o "$BUILD_USER" -g "$BUILD_USER" "$tmp" "/srv/builds/_bin/$(basename "$f")"; any=1; fi
|
||||
rm -f "$tmp"
|
||||
done
|
||||
for u in igneum-night-battery.service igneum-night-battery.timer; do
|
||||
tmp=$(mktemp); as_build "git -C /srv/igneum.git show '$NIGHT_REF:infra/build-server/night/$u'" > "$tmp"
|
||||
if ! cmp -s "$tmp" "/etc/systemd/system/$u"; then install -m 644 "$tmp" "/etc/systemd/system/$u"; any=1; fi
|
||||
rm -f "$tmp"
|
||||
done
|
||||
[ "$any" = 1 ] && systemctl daemon-reload
|
||||
systemctl is-enabled --quiet igneum-night-battery.timer 2>/dev/null || { systemctl enable --now --quiet igneum-night-battery.timer; any=1; }
|
||||
systemctl is-active --quiet igneum-night-battery.timer || systemctl start igneum-night-battery.timer
|
||||
[ "$any" = 1 ] && changed night "timer $(systemctl list-timers igneum-night-battery.timer --no-pager --no-legend | awk '{ print $1, $2, $3, $4 }'), files from $NIGHT_REF" \
|
||||
|| ok night "next $(systemctl list-timers igneum-night-battery.timer --no-pager --no-legend | awk '{ print $1, $2, $3, $4 }')"
|
||||
}
|
||||
|
||||
step_summary() {
|
||||
log "summary:"
|
||||
{
|
||||
|
|
@ -545,6 +580,7 @@ step_summary() {
|
|||
printf 'caddy: %s, %s\n' "$(caddy version 2>/dev/null | cut -d' ' -f1)" "$(systemctl is-active caddy 2>/dev/null) at https://$WORKERS_HOST/headline.json"
|
||||
printf 'cuda headers: %s\n' "$(ls -d /usr/local/cuda-*/include 2>/dev/null | tr '\n' ' ')$( [ -f /usr/include/CL/cl.h ] && echo '+ CL/cl.h' )"
|
||||
printf 'runner: %s\n' "$( [ -f "$RUNNER_DIR/.runner" ] && printf '%s, %s, user %s' "$(systemctl list-units --type=service --no-legend 'actions.runner.*' | awk '{ print $1 ": " $4 }' | head -1)" "v$(tr -d '"' < "$RUNNER_DIR/.runner_version" 2>/dev/null)" "$RUNNER_USER" || echo "installed, NOT registered (infra/build-server/runner/register.sh)" )"
|
||||
printf 'night battery: %s\n' "$(systemctl list-timers igneum-night-battery.timer --no-pager --no-legend 2>/dev/null | awk '{ print "next " $1, $2, $3, $4 }')"
|
||||
printf 'ssh line: ssh -i ~/.ssh/igneum_ed25519 build@%s\n' "$(hostname -I 2>/dev/null | awk '{ print $1 }')"
|
||||
} | sed 's/^/ /'
|
||||
}
|
||||
|
|
@ -570,6 +606,8 @@ do_provision() {
|
|||
step_cuda
|
||||
step_ufw
|
||||
step_runner
|
||||
step_cargo_tools
|
||||
step_night
|
||||
step_summary
|
||||
log "done"
|
||||
}
|
||||
|
|
|
|||
139
infra/build-server/repro/rebuild-on-box.sh
Executable file
139
infra/build-server/repro/rebuild-on-box.sh
Executable file
|
|
@ -0,0 +1,139 @@
|
|||
#!/usr/bin/env bash
|
||||
# Reproducible-build check, the box half (runs ON igneum-build-1 as user build; tools/repro/rebuild-release.sh drives it).
|
||||
# rebuild-on-box.sh --version 0.3.14 --node-commit <sha> --app-commit <sha> [--node-branch release-0.3.14-node]
|
||||
# [--shipped igneumd=<sha256> --shipped igneumd.exe=<sha256> ...] [--public] [--out <md>] [--passes 2]
|
||||
# What it does:
|
||||
# 1. a CLEAN layout at /srv/builds/_repro/<version>/: the igneum repo cloned from /srv/igneum.git at the app commit (this gives
|
||||
# igneum-pow as the ship worktree had it) and the fork cloned from /srv/igneum-node.git at the node commit under
|
||||
# vendor/igneum-node, checked out ON A BRANCH (kaspa-build-info embeds the commit only from a .git directory on a branch:
|
||||
# the empty-commit class of 6 October 2026); the fork's path dependency ../../../../igneum-pow resolves as on the Mac.
|
||||
# 2. --passes builds (default 2) of the Linux igneumd and igneum-miner and of the Windows exes, each in its own fresh
|
||||
# target dir, WITHOUT sccache (a hit would hand pass B pass A's object and hide a non-determinism), each under a build
|
||||
# slot through remote-run.sh (one JSONL line per pass, kind node-linux or node-windows, tool repro). The Windows
|
||||
# environment is cross-remote.sh's, flag for flag (static libgcc and libstdc++, -Wl,--no-insert-timestamp).
|
||||
# 3. --public: the shipped hashes are READ FROM THE PUBLIC ARTEFACTS, no token: igneum-hive-<v>.tar.gz (igneumd, igneum-miner)
|
||||
# and Igneum-Miner-Setup-<v>.exe through innoextract (igneumd.exe, igneum-miner.exe); --shipped values add or override.
|
||||
# 4. the evidence: a markdown table per artefact (shipped sha256 and its source, every pass's sha256 and size, MATCH or DIFFER
|
||||
# against the shipped bytes, pass A against pass B, and the reason for a DIFFER from facts read off the binaries: the
|
||||
# glibc symbol version the shipped Linux binary needs, the PE timestamp of the shipped exe, whether the commit string is
|
||||
# in it); written to --out (default /srv/builds/_repro/<version>/<version>.md) and printed.
|
||||
set -euo pipefail
|
||||
_slots_env="${IGNEUM_BUILD_SLOTS_DIR:-}"; [ -f /etc/profile.d/igneum-build.sh ] && . /etc/profile.d/igneum-build.sh; [ -n "$_slots_env" ] && IGNEUM_BUILD_SLOTS_DIR="$_slots_env"
|
||||
VERSION=""; NODE_SHA=""; APP_SHA=""; NODE_BRANCH=""; PUBLIC=0; OUT=""; PASSES=2; declare -A SHIPPED SHIPPED_SRC
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--version) VERSION="$2"; shift 2 ;; --node-commit) NODE_SHA="$2"; shift 2 ;; --app-commit) APP_SHA="$2"; shift 2 ;;
|
||||
--node-branch) NODE_BRANCH="$2"; shift 2 ;; --shipped) SHIPPED["${2%%=*}"]="${2#*=}"; SHIPPED_SRC["${2%%=*}"]="given"; shift 2 ;;
|
||||
--public) PUBLIC=1; shift ;; --out) OUT="$2"; shift 2 ;; --passes) PASSES="$2"; shift 2 ;;
|
||||
*) echo "unknown argument $1" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
[ -n "$VERSION" ] && [ -n "$NODE_SHA" ] && [ -n "$APP_SHA" ] || { echo "need --version, --node-commit and --app-commit" >&2; exit 2; }
|
||||
[ -n "$NODE_BRANCH" ] || NODE_BRANCH="release-$VERSION-node"
|
||||
RR="${IGNEUM_REMOTE_RUN:-/srv/builds/_bin/remote-run.sh}"; [ -f "$RR" ] || { echo "no $RR" >&2; exit 2; }
|
||||
ROOT=/srv/builds/_repro/$VERSION; mkdir -p "$ROOT"; [ -n "$OUT" ] || OUT="$ROOT/$VERSION.md"
|
||||
LOG="$ROOT/rebuild.log"; : > "$LOG"
|
||||
say() { printf '%s repro: %s\n' "$(date -u +%H:%M:%S)" "$*" | tee -a "$LOG" >&2; }
|
||||
sha() { sha256sum "$1" | cut -d' ' -f1; }
|
||||
WIN=x86_64-pc-windows-gnu
|
||||
T_ALL=$(date +%s)
|
||||
|
||||
# 1. the clean layout
|
||||
say "layout $ROOT: igneum at $APP_SHA, fork at $NODE_SHA on branch $NODE_BRANCH"
|
||||
rm -rf "$ROOT/igneum"
|
||||
git clone -q --no-checkout /srv/igneum.git "$ROOT/igneum"
|
||||
git -C "$ROOT/igneum" checkout -q -B "repro-$VERSION" "$APP_SHA" || { say "app commit $APP_SHA is not in /srv/igneum.git (push it from the Mac)"; exit 3; }
|
||||
mkdir -p "$ROOT/igneum/vendor"
|
||||
git clone -q --no-checkout /srv/igneum-node.git "$ROOT/igneum/vendor/igneum-node"
|
||||
git -C "$ROOT/igneum/vendor/igneum-node" checkout -q -B "$NODE_BRANCH" "$NODE_SHA" || { say "node commit $NODE_SHA is not in /srv/igneum-node.git (push it from the Mac)"; exit 3; }
|
||||
NODE_FULL=$(git -C "$ROOT/igneum/vendor/igneum-node" rev-parse HEAD); APP_FULL=$(git -C "$ROOT/igneum" rev-parse HEAD)
|
||||
FORK="$ROOT/igneum/vendor/igneum-node"
|
||||
|
||||
# 2. the builds: one remote-run.sh invocation per pass and target; no sccache
|
||||
run_pass() { # <kind: linux|windows> <pass letter>
|
||||
local kind="$1" pass="$2" tdir="target-repro-$1-$2" cmd envb="" arts
|
||||
if [ "$kind" = linux ]; then
|
||||
cmd="RUSTC_WRAPPER= CARGO_TARGET_DIR='$tdir' cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow 2>&1 | tail -3; ( exit \${PIPESTATUS[0]} )"
|
||||
arts="$tdir/release/igneumd $tdir/release/igneum-miner"; BR_KIND=node-linux; BR_TARGET=x86_64-unknown-linux-gnu
|
||||
else
|
||||
envb='LLVM_LIB=$(ls -d /usr/lib/llvm-*/lib 2>/dev/null | sort -V | tail -1); export CC_x86_64_pc_windows_gnu=x86_64-w64-mingw32-gcc-posix CXX_x86_64_pc_windows_gnu=x86_64-w64-mingw32-g++-posix AR_x86_64_pc_windows_gnu=x86_64-w64-mingw32-ar CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER=x86_64-w64-mingw32-gcc-posix; export CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUSTFLAGS="-C link-arg=-static -C link-arg=-static-libgcc -C link-arg=-static-libstdc++ -C link-arg=-Wl,--no-insert-timestamp"; export IGNEUM_WINDRES=x86_64-w64-mingw32-windres LIBCLANG_PATH="$LLVM_LIB" BINDGEN_EXTRA_CLANG_ARGS_x86_64_pc_windows_gnu="--target=x86_64-w64-mingw32 --sysroot=/usr/x86_64-w64-mingw32 -I/usr/x86_64-w64-mingw32/include"; '
|
||||
cmd="${envb}RUSTC_WRAPPER= CARGO_TARGET_DIR='$tdir' cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target $WIN 2>&1 | tail -3; ( exit \${PIPESTATUS[0]} )"
|
||||
arts="$tdir/$WIN/release/igneumd.exe $tdir/$WIN/release/igneum-miner.exe"; BR_KIND=node-windows; BR_TARGET=$WIN
|
||||
fi
|
||||
rm -rf "$FORK/$tdir"
|
||||
local t0; t0=$(date +%s)
|
||||
BR_DIR="$FORK" BR_CMD="$cmd" BR_LABEL="repro $VERSION $kind pass $pass; agent=${IGNEUM_AGENT:-box-work}" BR_TOOL=repro BR_KIND="$BR_KIND" BR_TARGET="$BR_TARGET" \
|
||||
BR_WT="_repro/$VERSION" BR_CRATE=vendor/igneum-node BR_BRANCH="$NODE_BRANCH" BR_SHA="$NODE_FULL" BR_AGENT="${IGNEUM_AGENT:-box-work}" \
|
||||
BR_COMMAND="cargo build --release -p kaspad -p igneum-miner ($kind, pass $pass, no sccache)" BR_ARTEFACTS="$arts" \
|
||||
bash "$RR" >> "$LOG" 2>&1 || { say "$kind pass $pass FAILED (rc $?): tail of $LOG:"; tail -20 "$LOG" >&2; return 1; }
|
||||
say "$kind pass $pass built in $(( $(date +%s) - t0 )) s"
|
||||
}
|
||||
declare -A PASS_SHA PASS_SIZE PASS_PATH
|
||||
for kind in linux windows; do
|
||||
for i in $(seq 1 "$PASSES"); do
|
||||
p=$(printf "\\$(printf '%03o' $((64 + i)))") # A, B, ...
|
||||
run_pass "$kind" "$p" || exit 4
|
||||
if [ "$kind" = linux ]; then
|
||||
for a in igneumd igneum-miner; do f="$FORK/target-repro-linux-$p/release/$a"; PASS_PATH["$a:$p"]="$f"; PASS_SHA["$a:$p"]=$(sha "$f"); PASS_SIZE["$a:$p"]=$(stat -c %s "$f"); done
|
||||
else
|
||||
for a in igneumd.exe igneum-miner.exe; do f="$FORK/target-repro-windows-$p/$WIN/release/$a"; PASS_PATH["$a:$p"]="$f"; PASS_SHA["$a:$p"]=$(sha "$f"); PASS_SIZE["$a:$p"]=$(stat -c %s "$f"); done
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# 3. the shipped bytes from the public artefacts
|
||||
declare -A SHIPPED_PATH
|
||||
if [ "$PUBLIC" = 1 ]; then
|
||||
pub="$ROOT/public"; rm -rf "$pub"; mkdir -p "$pub"
|
||||
base=https://dl.igneum.network/dl/public
|
||||
if curl -fsSL -o "$pub/hive.tar.gz" "$base/igneum-hive-$VERSION.tar.gz"; then
|
||||
mkdir -p "$pub/hive"; tar -xzf "$pub/hive.tar.gz" -C "$pub/hive" 2>/dev/null || true
|
||||
for a in igneumd igneum-miner; do f=$(find "$pub/hive" -type f -name "$a" -not -name '._*' | head -1); [ -n "$f" ] && { SHIPPED["$a"]=$(sha "$f"); SHIPPED_SRC["$a"]="igneum-hive-$VERSION.tar.gz ($(sha "$pub/hive.tar.gz" | cut -c1-16)...)"; SHIPPED_PATH["$a"]="$f"; }; done
|
||||
say "public HiveOS package: $(ls "$pub/hive"/*/bin 2>/dev/null | tr '\n' ' ')"
|
||||
else say "no public HiveOS package for $VERSION"; fi
|
||||
if curl -fsSL -o "$pub/setup.exe" "$base/Igneum-Miner-Setup-$VERSION.exe"; then
|
||||
if command -v innoextract >/dev/null 2>&1; then
|
||||
innoextract -q -d "$pub/setup" "$pub/setup.exe" >/dev/null 2>&1 || say "innoextract failed on the installer"
|
||||
for a in igneumd.exe igneum-miner.exe; do f=$(find "$pub/setup" -type f -name "$a" | head -1); [ -n "$f" ] && { SHIPPED["$a"]=$(sha "$f"); SHIPPED_SRC["$a"]="Igneum-Miner-Setup-$VERSION.exe ($(sha "$pub/setup.exe" | cut -c1-16)...) via innoextract"; SHIPPED_PATH["$a"]="$f"; }; done
|
||||
else say "innoextract is not installed (apt innoextract): the Windows shipped hashes come from --shipped only"; fi
|
||||
else say "no public installer for $VERSION"; fi
|
||||
fi
|
||||
|
||||
# 4. the evidence
|
||||
glibc_need() { objdump -T "$1" 2>/dev/null | grep -o 'GLIBC_[0-9.]*' | sort -uV | tail -1; }
|
||||
pe_stamp() { x86_64-w64-mingw32-objdump -p "$1" 2>/dev/null | awk '/Time\/Date/ { $1=""; $2=""; print; exit }' | sed 's/^ *//'; }
|
||||
commit_hits() { strings -n 7 "$1" 2>/dev/null | grep -c "$NODE_FULL" || true; }
|
||||
reason() { # <artefact> <shipped path or empty>
|
||||
local a="$1" sp="$2" r=""
|
||||
case "$a" in
|
||||
igneumd|igneum-miner)
|
||||
r="toolchain: the shipped binary needs glibc $(glibc_need "$sp" 2>/dev/null || echo '?') (the Mac's infra/cross/build-linux.sh, zig, glibc 2.36 target); the box links the native clang/lld glibc $(glibc_need "${PASS_PATH[$a:A]}")" ;;
|
||||
igneumd.exe|igneum-miner.exe)
|
||||
r="toolchain: the shipped exe came from the Mac's proto-cuda/windows-node/cross-build.sh (Homebrew mingw), the box from Ubuntu GCC 13 posix; shipped PE timestamp '$(pe_stamp "$sp")' against the box's '$(pe_stamp "${PASS_PATH[$a:A]}")' (the --no-insert-timestamp fix landed 6 Oct 2026 17:48Z, after this cut's exes)" ;;
|
||||
esac
|
||||
[ "$a" = igneumd ] || [ "$a" = igneumd.exe ] && r="$r; commit string $NODE_FULL in the shipped binary: $(commit_hits "$sp") hit(s), in the box's: $(commit_hits "${PASS_PATH[$a:A]}") (0 in the shipped one = the empty-commit class)"
|
||||
echo "$r"
|
||||
}
|
||||
{
|
||||
echo "# Reproduced: Igneum Miner $VERSION (node $NODE_FULL, app $APP_FULL)"
|
||||
echo
|
||||
echo "$(date -u +'%d %B %Y, %H:%M UTC') on igneum-build-1 by \`infra/build-server/repro/rebuild-on-box.sh\` (driven by \`tools/repro/rebuild-release.sh\`): a clean clone of the fork at the node commit on branch \`$NODE_BRANCH\` under a clean clone of the repo at the app commit, $PASSES independent passes per target (fresh target dir each, no sccache), rustc $(rustc --version | awk '{ print $2 }'), $(x86_64-w64-mingw32-gcc-posix --version | head -1), clang $(clang --version | head -1 | grep -o '[0-9][0-9.]*' | head -1), glibc $(ldd --version | head -1 | grep -o '[0-9.]*$'). Shipped hashes read from the public downloads (no token) where marked. Whole run $(( $(date +%s) - T_ALL )) s; log \`$LOG\`."
|
||||
echo
|
||||
echo "| Artefact | Shipped sha256 (source) | Box pass A | Box pass B | A vs shipped | A vs B | Reason for a DIFFER |"
|
||||
echo "|---|---|---|---|---|---|---|"
|
||||
for a in igneumd igneum-miner igneumd.exe igneum-miner.exe; do
|
||||
s="${SHIPPED[$a]:-}"; src="${SHIPPED_SRC[$a]:-}"
|
||||
A="${PASS_SHA[$a:A]}"; B="${PASS_SHA[$a:B]:-}"
|
||||
if [ -z "$s" ]; then vs="NO SHIPPED HASH"; elif [ "$A" = "$s" ]; then vs="**MATCH**"; elif [ "${s#${A:0:${#s}}}" = "" ] && [ ${#s} -lt 64 ]; then vs="**MATCH** (prefix)"; else vs="**DIFFER**"; fi
|
||||
if [ -z "$B" ]; then ab="one pass"; elif [ "$A" = "$B" ]; then ab="**MATCH**"; else ab="**DIFFER**"; fi
|
||||
why=""; [ "$vs" = "**DIFFER**" ] && why=$(reason "$a" "${SHIPPED_PATH[$a]:-}")
|
||||
[ "$ab" = "**DIFFER**" ] && why="${why:+$why; }the box does not reproduce ITSELF for this artefact: a non-determinism in the build, to be found"
|
||||
printf '| %s | %s%s | %s (%s B) | %s%s | %s | %s | %s |\n' "$a" "${s:-none}" "${src:+ ($src)}" "${A:0:16}..." "${PASS_SIZE[$a:A]}" "${B:+${B:0:16}...}" "${B:+ (${PASS_SIZE[$a:B]} B)}" "$vs" "$ab" "${why:-}"
|
||||
done
|
||||
echo
|
||||
echo "Full box hashes: $(for a in igneumd igneum-miner igneumd.exe igneum-miner.exe; do printf '%s A %s; ' "$a" "${PASS_SHA[$a:A]}"; done)"
|
||||
echo
|
||||
echo "Reading: MATCH against the shipped bytes is the goal; A vs B MATCH with a DIFFER against the shipped bytes means the box is deterministic and the shipped build came from another toolchain (the reason column says which facts differ); A vs B DIFFER is a non-determinism on the box itself and is the row to fix first."
|
||||
} > "$OUT"
|
||||
cat "$OUT"
|
||||
say "evidence at $OUT"
|
||||
74
tools/repro/rebuild-release.sh
Executable file
74
tools/repro/rebuild-release.sh
Executable file
|
|
@ -0,0 +1,74 @@
|
|||
#!/usr/bin/env bash
|
||||
# Rebuild a shipped release on igneum-build-1 from a clean checkout and compare the bytes with what shipped (6 October 2026).
|
||||
# tools/repro/rebuild-release.sh 0.3.14 pins from docs/plans/release-0.3.14.md, shipped hashes from the public downloads
|
||||
# tools/repro/rebuild-release.sh 0.3.14 --node-commit <sha> --app-commit <sha> explicit pins (the plan is not read)
|
||||
# tools/repro/rebuild-release.sh 0.3.14 --shipped igneumd.exe=<sha256> add or override a shipped hash (the plan's bold hashes are also read)
|
||||
# --passes N (default 2), --no-public (shipped hashes from the plan and --shipped only), --node-branch <name>
|
||||
# Where the pins live (docs/plans/release-0.3.14.md): the section heading "## 3. Builds and artefacts (node <sha>, app <sha>)";
|
||||
# the shipped hashes are the bold sha256 values in that table (the Linux row "The seed's Linux node", the Windows row "The
|
||||
# Windows node exes") and, token-free, inside the public downloads (packaging/README-ship.md "The public downloads path":
|
||||
# igneum-hive-<v>.tar.gz carries igneumd and igneum-miner, Igneum-Miner-Setup-<v>.exe carries the two exes).
|
||||
# The work runs on the box (infra/build-server/repro/rebuild-on-box.sh, under build slots through remote-run.sh); this script
|
||||
# makes sure the mirrors hold both commits (pushes them from this Mac's checkouts when they do not), ships the two scripts to
|
||||
# /srv/builds/_bin, runs the rebuild, and fetches the evidence into docs/evidence/reproduced/<version>.md of THIS worktree.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../.." && pwd)"
|
||||
MAIN_REPO="${IGNEUM_MAIN_REPO:-/Users/joshm/Projects/igneum}"
|
||||
# shellcheck disable=SC2034
|
||||
BS_TOOL=repro
|
||||
# shellcheck source=../../infra/build-server/lib.sh
|
||||
. "$ROOT/infra/build-server/lib.sh"
|
||||
VERSION="${1:-}"; shift || true
|
||||
[ -n "$VERSION" ] || bs_die "usage: tools/repro/rebuild-release.sh <version> [--node-commit sha] [--app-commit sha] [--shipped name=sha256]... [--passes N] [--no-public]"
|
||||
NODE_SHA=""; APP_SHA=""; NODE_BRANCH=""; PASSES=2; PUBLIC=1; SHIPPED=()
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--node-commit) NODE_SHA="$2"; shift 2 ;; --app-commit) APP_SHA="$2"; shift 2 ;; --node-branch) NODE_BRANCH="$2"; shift 2 ;;
|
||||
--shipped) SHIPPED+=(--shipped "$2"); shift 2 ;; --passes) PASSES="$2"; shift 2 ;; --no-public) PUBLIC=0; shift ;;
|
||||
*) bs_die "unknown argument $1" ;;
|
||||
esac
|
||||
done
|
||||
PLAN="$ROOT/docs/plans/release-$VERSION.md"
|
||||
if [ -z "$NODE_SHA" ] || [ -z "$APP_SHA" ]; then
|
||||
[ -f "$PLAN" ] || bs_die "no $PLAN and no --node-commit/--app-commit"
|
||||
pins=$(grep -m1 -oE '\(node [0-9a-f]{7,40}, app [0-9a-f]{7,40}\)' "$PLAN" || true)
|
||||
[ -n "$pins" ] || bs_die "no '(node <sha>, app <sha>)' heading in $PLAN; pass --node-commit and --app-commit"
|
||||
[ -n "$NODE_SHA" ] || NODE_SHA=$(sed -E 's/.*node ([0-9a-f]+),.*/\1/' <<<"$pins")
|
||||
[ -n "$APP_SHA" ] || APP_SHA=$(sed -E 's/.*app ([0-9a-f]+)\).*/\1/' <<<"$pins")
|
||||
bs_log "pins from $PLAN: node $NODE_SHA, app $APP_SHA"
|
||||
# the plan's bold hashes as a second source (full ones only; the Windows row names igneumd.exe, the Linux row igneumd)
|
||||
lin=$(grep -m1 -E "^\| The seed's Linux node" "$PLAN" | grep -oE 'igneumd \*\*[0-9a-f]{64}\*\*' | grep -oE '[0-9a-f]{64}' || true)
|
||||
win=$(grep -m1 -E '^\| The Windows node exes' "$PLAN" | grep -oE 'igneumd\.exe \*\*[0-9a-f]{64}\*\*' | grep -oE '[0-9a-f]{64}' || true)
|
||||
[ -n "$lin" ] && [ "$PUBLIC" = 0 ] && SHIPPED+=(--shipped "igneumd=$lin")
|
||||
[ -n "$win" ] && [ "$PUBLIC" = 0 ] && SHIPPED+=(--shipped "igneumd.exe=$win")
|
||||
[ -n "$lin$win" ] && bs_log "plan hashes: igneumd ${lin:0:16}... igneumd.exe ${win:0:16}... (the public artefacts are the primary source unless --no-public)"
|
||||
fi
|
||||
[ -n "$NODE_BRANCH" ] || NODE_BRANCH="release-$VERSION-node"
|
||||
bs_host
|
||||
# the mirrors must hold both commits; push them from the Mac's checkouts when they do not (a ref per repro, never master)
|
||||
ensure_commit() { # <local repo> <mirror> <sha> <label>
|
||||
local repo="$1" mirror="$2" sha="$3" label="$4" full
|
||||
if bs_ssh "git -C '$mirror' cat-file -e '$sha^{commit}' 2>/dev/null"; then bs_log "$label: $sha is on $mirror"; return; fi
|
||||
full=$(git -C "$repo" rev-parse --verify "$sha^{commit}" 2>/dev/null) || bs_die "$label: $sha is neither on the box's $mirror nor in $repo"
|
||||
GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$repo" push -q "$BS_HOST:$mirror" "$full:refs/heads/repro-$VERSION-$label" || bs_die "$label: push of $sha to $mirror failed"
|
||||
bs_log "$label: pushed $full to $mirror as repro-$VERSION-$label"
|
||||
}
|
||||
ensure_commit "$MAIN_REPO" "$BS_MIRROR_REPO" "$APP_SHA" app
|
||||
ensure_commit "$MAIN_REPO/vendor/igneum-node" "$BS_MIRROR_NODE" "$NODE_SHA" node
|
||||
bs_ssh 'mkdir -p /srv/builds/_bin'
|
||||
bs_rsync -q "$ROOT/infra/build-server/repro/rebuild-on-box.sh" "$ROOT/infra/build-server/remote-run.sh" "$BS_HOST:/srv/builds/_bin/"
|
||||
bs_ssh 'chmod +x /srv/builds/_bin/*.sh'
|
||||
args=(--version "$VERSION" --node-commit "$NODE_SHA" --app-commit "$APP_SHA" --node-branch "$NODE_BRANCH" --passes "$PASSES"); [ "$PUBLIC" = 1 ] && args+=(--public)
|
||||
bs_log "rebuilding on the box: ${args[*]}"
|
||||
t0=$(date +%s)
|
||||
# bash 3.2 on the Mac treats an empty array as unbound under set -u (run-from-mac.sh met it): expand it only when it has members
|
||||
[ "${#SHIPPED[@]}" -gt 0 ] && args+=("${SHIPPED[@]}")
|
||||
set +e
|
||||
bs_ssh "IGNEUM_AGENT=${IGNEUM_AGENT:-repro} /srv/builds/_bin/rebuild-on-box.sh $(printf '%q ' "${args[@]}")" 2>&1 | grep -v '^#\|^|\|^$\|^Full box\|^Reading'
|
||||
rc=${PIPESTATUS[0]}
|
||||
set -e
|
||||
[ "$rc" = 0 ] || bs_die "the rebuild on the box failed (rc $rc); its log: ssh build@box cat /srv/builds/_repro/$VERSION/rebuild.log"
|
||||
mkdir -p "$ROOT/docs/evidence/reproduced"
|
||||
bs_rsync -q "$BS_HOST:/srv/builds/_repro/$VERSION/$VERSION.md" "$ROOT/docs/evidence/reproduced/$VERSION.md"
|
||||
bs_log "done in $(bs_fmt_secs $(( $(date +%s) - t0 ))): docs/evidence/reproduced/$VERSION.md"
|
||||
grep -E '^\| (igneumd|igneum-miner)' "$ROOT/docs/evidence/reproduced/$VERSION.md" | cut -c1-200
|
||||
Loading…
Reference in a new issue