igneum/infra/build-server/night/night-battery.sh
igneum-labs 09c2634d2c Pre-public scrub, second pass (7 October 2026, 20:0x UK, main's rulings 2 and 4): the public tree names igneum-labs only; the public ledger generated from the full ledger
Ruling 2: every tracked text mention of the login's pre-rename spelling becomes igneum-labs (29 files); the commit identity in the scripts becomes igneum-labs <337424239+igneum-labs@users.noreply.github.com> (the same noreply id); the three scripts that hand the stored gh keyring name to gh (register.sh, fetch-ci-artifacts.sh, ship-app.mjs) read it from ~/.config/igneum/gh-user (never in the repository), default igneum-labs; fresh-repo.sh reads the login to rewrite from the encoded list's row 10; the old spelling joins the founder-strings list, so no tracked file may spell it; CLAUDE.md's GitHub paragraph rewritten. The red watcher's self-test fixtures use a neutral author.

Ruling 4: docs/ledger-public.md, one row per ledger item (id, the claim in one line, status, what was done in one line, the evidence link or the evidence in words), generated by tools/ledger/export-public.mjs from docs/fud-ledger.md: no round or status-update sections, no commit ids, no time of day, no lane, agent or branch names. 190 items. The gate runs its self-test (a fixture with a commit id, a time, a lane and a branch name comes out clean; --check fails on drift) and its --check on every merge; the founder check reads the output like every tracked file.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-07 18:56:08 +00:00

207 lines
17 KiB
Bash
Executable file

#!/usr/bin/env bash
# The night battery on igneum-build-1 (6 October 2026): one invocation, one build slot, every test the repo and the fork have.
# Runs ON the box as user build at 02:00 Europe/London (igneum-night-battery.timer) through remote-run.sh, which holds the
# slot for the whole run, sets CARGO_BUILD_JOBS (90 alone, 45 beside another build) and writes the JSONL line. By hand:
# /srv/builds/_bin/night-battery.sh the full battery (hours)
# NIGHT_SUBSET=1 /srv/builds/_bin/night-battery.sh the dry-run subset (igneum-pow suite and fuzz, two fork crates, quick sims, clippy and audit on igneum-pow)
# NIGHT_NODE_BRANCH=release-0.3.15-node ... the fork branch (default: the newest release-*-node on the mirror)
# NIGHT_SKIP="harness sims" ... skip stages by name
# Stages, each a row (pass, FAIL, skip) with seconds and a detail:
# checkout /srv/builds/_night/igneum from /srv/igneum.git master (the battery re-execs itself from that checkout, so the
# script that runs is master's), vendor/igneum-node from /srv/igneum-node.git at the fork branch
# suites cargo test --release --no-fail-fast per crate: the repo's crates (igneum-pow, igneum-census, pool, proto-vdf,
# app/igneum-app, every member of proving/igneum-prove) and every workspace member of the fork (kaspad with
# --features igneum-pow); the pass and fail counts are read from the `test result:` lines
# fuzz igneum-pow's two fuzz tests at 10x their default (IGNEUM_MIXER_FUZZ and IGNEUM_SCRATCH_FUZZ 2000)
# sims sim/finality_sim.py, sim/finality_v2.py and sim/difficulty/sim.py in full (the subset runs --quick)
# harness the fork's igneumd, igneum-miner, igneum-harness-sim and igneum-p2p-probe built into target-integration, then
# tools/finality-attacks/run.mjs --fast-time, tools/harness/run.mjs s3 s4 --fast-time --no-bench-log and
# tools/exec-sync/reorg.mjs (loopback ports 27200+, 27800+, 29870+; nothing of the live devnet)
# clippy cargo clippy --release --all-targets per crate dir (warnings counted; a row fails on an error)
# audit cargo audit in every directory with a Cargo.lock
# report docs/benchmarks/night/<date>.md (a pass/fail table and "new since last night" against the newest earlier
# report), committed as igneum-labs on branch night-battery (based on master, earlier reports carried over) and
# pushed to /srv/igneum.git night-battery; main merges (`git fetch build night-battery` on the Mac). Never master.
set -uo pipefail
_slots_env="${IGNEUM_BUILD_SLOTS_DIR:-}"; [ -f /etc/profile.d/igneum-build.sh ] && . /etc/profile.d/igneum-build.sh; [ -n "$_slots_env" ] && IGNEUM_BUILD_SLOTS_DIR="$_slots_env"
NIGHT_ROOT="${NIGHT_ROOT:-/srv/builds/_night}"; REPO_MIRROR=/srv/igneum.git; NODE_MIRROR=/srv/igneum-node.git
SUBSET="${NIGHT_SUBSET:-0}"; SKIP=" ${NIGHT_SKIP:-} "
DATE=$(date -u +%Y-%m-%d); STAMP=$(date -u +%Y%m%dT%H%M%SZ); T_ALL=$(date +%s)
REPORT_NAME="$DATE$( [ "$SUBSET" = 1 ] && echo -dryrun ).md"
LOGDIR="$NIGHT_ROOT/logs/$STAMP"; mkdir -p "$LOGDIR"
ROWS="$LOGDIR/rows.tsv"; : > "$ROWS"
say() { printf '%s night: %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; }
row() { printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "$4" "$5" >> "$ROWS"; say "$1 | $2 | $3 | ${4}s | $5"; } # stage name status secs detail
skip() { case "$SKIP" in *" $1 "*) return 0 ;; esac; return 1; }
cargo_jobs="${CARGO_BUILD_JOBS:-90}"
# checkout
IG="$NIGHT_ROOT/igneum"; FORK="$IG/vendor/igneum-node"
t0=$(date +%s)
if [ ! -d "$IG/.git" ]; then git clone -q "$REPO_MIRROR" "$IG" || { row checkout repo FAIL 0 "clone failed"; exit 1; }; fi
git -C "$IG" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' || { row checkout repo FAIL 0 "fetch failed"; exit 1; }
git -C "$IG" checkout -q -- . 2>/dev/null; git -C "$IG" clean -qfd -e target -e 'target-*' -e vendor
git -C "$IG" checkout -q -B night-battery origin/master
# earlier reports not yet merged into master ride along
git -C "$IG" checkout -q origin/night-battery -- docs/benchmarks/night 2>/dev/null || true
REPO_SHA=$(git -C "$IG" rev-parse --short HEAD)
if [ "${NIGHT_REEXEC:-0}" != 1 ] && [ -f "$IG/infra/build-server/night/night-battery.sh" ] && ! cmp -s "$0" "$IG/infra/build-server/night/night-battery.sh"; then
say "re-exec from master's copy ($REPO_SHA)"; NIGHT_REEXEC=1 exec bash "$IG/infra/build-server/night/night-battery.sh"
fi
NODE_BRANCH="${NIGHT_NODE_BRANCH:-$(git -C "$NODE_MIRROR" branch --list 'release-*-node' | tr -d ' *' | sort -V | tail -1)}"
[ -n "$NODE_BRANCH" ] || NODE_BRANCH=master
mkdir -p "$IG/vendor"
if [ ! -d "$FORK/.git" ]; then git clone -q --no-checkout "$NODE_MIRROR" "$FORK" || { row checkout fork FAIL 0 "clone failed"; exit 1; }; fi
git -C "$FORK" fetch -q origin '+refs/heads/*:refs/remotes/origin/*'
git -C "$FORK" checkout -q -- . 2>/dev/null; git -C "$FORK" clean -qfd -e target -e 'target-*'
git -C "$FORK" checkout -q -B "$NODE_BRANCH" "origin/$NODE_BRANCH" || { row checkout fork FAIL 0 "no branch $NODE_BRANCH"; exit 1; }
NODE_SHA=$(git -C "$FORK" rev-parse --short HEAD)
row checkout "repo master $REPO_SHA, fork $NODE_BRANCH $NODE_SHA" pass $(( $(date +%s) - t0 )) "$IG; jobs $cargo_jobs; subset $SUBSET"
# helpers
run_to() { # <log> <timeout secs> <cmd...>: runs in the current dir, returns the exit code, 124 on timeout
local log="$1" to="$2"; shift 2
timeout --signal=TERM --kill-after=60 "$to" "$@" > "$log" 2>&1; echo $?
}
counts() { # pass/fail totals from cargo test output
awk '/^test result:/ { for (i = 1; i <= NF; i++) { if ($(i+1) == "passed;") p += $i; if ($(i+1) == "failed;") f += $i } } END { printf "%d passed, %d failed", p, f }' "$1"
}
suite() { # <dir> <label> <cargo test args...>
local dir="$1" label="$2"; shift 2
local log="$LOGDIR/suite-$(echo "$label" | tr '/ ' '__').log" t0 rc c
t0=$(date +%s)
rc=$(cd "$dir" && run_to "$log" 2400 cargo test --release --no-fail-fast "$@")
c=$(counts "$log")
if [ "$rc" = 0 ]; then row suite "$label" pass $(( $(date +%s) - t0 )) "$c"
elif [ "$rc" = 124 ]; then row suite "$label" FAIL $(( $(date +%s) - t0 )) "TIMEOUT 40 min; $c"
elif grep -q '^error\(\[E[0-9]*\]\)\?:' "$log" && ! grep -q '^test result:' "$log"; then row suite "$label" FAIL $(( $(date +%s) - t0 )) "did not compile: $(grep -m1 '^error' "$log" | cut -c1-120)"
else row suite "$label" FAIL $(( $(date +%s) - t0 )) "$c; failed: $(grep -E '^ [a-z_:]+' "$log" | grep -v '^ Finished\|^ Running' | head -3 | tr -d ' ' | tr '\n' ' ' | cut -c1-160)"; fi
}
# suites
if ! skip suites; then
if [ "$SUBSET" = 1 ]; then
suite "$IG/igneum-pow" igneum-pow
for c in kaspa-pow igneum-miner; do suite "$FORK" "fork/$c" -p "$c"; done
else
for d in igneum-pow igneum-census pool proto-vdf app/igneum-app; do [ -f "$IG/$d/Cargo.toml" ] && suite "$IG/$d" "$d"; done
if [ -f "$IG/proving/igneum-prove/Cargo.toml" ]; then
for m in $(cd "$IG/proving/igneum-prove" && cargo metadata --no-deps --format-version 1 2>/dev/null | python3 -c 'import json,sys; print(" ".join(p["name"] for p in json.load(sys.stdin)["packages"]))'); do
case "$m" in *program*|*aggregator*) continue ;; esac # the guests are pinned ELFs, built only by pin-guests.sh
suite "$IG/proving/igneum-prove" "proving/$m" -p "$m"
done
fi
for m in $(cd "$FORK" && cargo metadata --no-deps --format-version 1 2>/dev/null | python3 -c 'import json,sys; print(" ".join(sorted(p["name"] for p in json.load(sys.stdin)["packages"])))'); do
case "$m" in *wasm*) continue ;; esac # browser targets, no host test suite
if [ "$m" = kaspad ]; then suite "$FORK" "fork/kaspad" -p kaspad --features igneum-pow; else suite "$FORK" "fork/$m" -p "$m"; fi
done
fi
else row suite all skip 0 "NIGHT_SKIP"; fi
# fuzz
if ! skip fuzz; then
n=2000; [ "$SUBSET" = 1 ] && n=200
t0=$(date +%s); log="$LOGDIR/fuzz.log"
rc=$(cd "$IG/igneum-pow" && IGNEUM_MIXER_FUZZ=$n IGNEUM_SCRATCH_FUZZ=$n run_to "$log" 7200 cargo test --release --test mixer --test scratch -- fuzz --nocapture)
[ "$rc" = 0 ] && row fuzz "igneum-pow mixer+scratch x$n" pass $(( $(date +%s) - t0 )) "$(grep -h '^fuzz:' "$log" | tr '\n' ';' | cut -c1-200)" || row fuzz "igneum-pow mixer+scratch x$n" FAIL $(( $(date +%s) - t0 )) "rc $rc: $(grep -m1 -E 'panicked|error' "$log" | cut -c1-160)"
else row fuzz igneum-pow skip 0 "NIGHT_SKIP"; fi
# sims
if ! skip sims; then
q=""; [ "$SUBSET" = 1 ] && q="--quick"
t0=$(date +%s); rc=$(cd "$IG/sim" && run_to "$LOGDIR/sim-finality.log" 600 python3 finality_sim.py); [ "$rc" = 0 ] && row sim finality_sim.py pass $(( $(date +%s) - t0 )) "$(grep -c '^|' "$LOGDIR/sim-finality.log") table lines" || row sim finality_sim.py FAIL $(( $(date +%s) - t0 )) "rc $rc"
t0=$(date +%s); rc=$(cd "$IG/sim" && run_to "$LOGDIR/sim-finality-v2.log" 3600 python3 finality_v2.py $q); [ "$rc" = 0 ] && row sim "finality_v2.py $q" pass $(( $(date +%s) - t0 )) "$(grep -c '^|' "$LOGDIR/sim-finality-v2.log") table lines" || row sim "finality_v2.py $q" FAIL $(( $(date +%s) - t0 )) "rc $rc"
t0=$(date +%s); rc=$(cd "$IG/sim/difficulty" && run_to "$LOGDIR/sim-difficulty.log" 5400 python3 sim.py $q); [ "$rc" = 0 ] && row sim "difficulty/sim.py $q" pass $(( $(date +%s) - t0 )) "$(grep -c '^|' "$LOGDIR/sim-difficulty.log") table lines" || row sim "difficulty/sim.py $q" FAIL $(( $(date +%s) - t0 )) "rc $rc"
else row sim all skip 0 "NIGHT_SKIP"; fi
# harness (fast time)
if ! skip harness && [ "$SUBSET" != 1 ]; then
t0=$(date +%s); log="$LOGDIR/harness-build.log"
rc=$(cd "$FORK" && CARGO_TARGET_DIR=target-integration run_to "$log" 3600 cargo build --release -p kaspad -p igneum-miner -p igneum-harness-sim -p igneum-p2p-probe --features kaspad/igneum-pow)
if [ "$rc" = 0 ]; then
row harness build pass $(( $(date +%s) - t0 )) "igneumd igneum-miner igneum-harness-sim igneum-p2p-probe at $NODE_SHA"
REL="$FORK/target-integration/release"
t0=$(date +%s); rc=$(cd "$IG" && IGNEUMD="$REL/igneumd" IGNEUM_MINER="$REL/igneum-miner" IGNEUM_NODE_ROOT="$IG/" run_to "$LOGDIR/harness-finality.log" 3600 node tools/finality-attacks/run.mjs --fast-time)
[ "$rc" = 0 ] && row harness "finality-attacks --fast-time" pass $(( $(date +%s) - t0 )) "$(grep -c -i 'PASS' "$LOGDIR/harness-finality.log") PASS lines" || row harness "finality-attacks --fast-time" FAIL $(( $(date +%s) - t0 )) "rc $rc: $(grep -m1 -i -E 'FAIL|error' "$LOGDIR/harness-finality.log" | cut -c1-160)"
t0=$(date +%s); rc=$(cd "$IG" && IGNEUM_HARNESS_TARGET="$REL" run_to "$LOGDIR/harness-s3s4.log" 3600 node tools/harness/run.mjs s3 s4 --fast-time --no-bench-log)
[ "$rc" = 0 ] && row harness "harness s3 s4 --fast-time" pass $(( $(date +%s) - t0 )) "$(grep -c -i 'PASS' "$LOGDIR/harness-s3s4.log") PASS lines" || row harness "harness s3 s4 --fast-time" FAIL $(( $(date +%s) - t0 )) "rc $rc: $(grep -m1 -i -E 'FAIL|error' "$LOGDIR/harness-s3s4.log" | cut -c1-160)"
t0=$(date +%s); rc=$(cd "$IG" && IGNEUM_EXEC_BIN="$REL" run_to "$LOGDIR/harness-exec-reorg.log" 3600 node tools/exec-sync/reorg.mjs)
[ "$rc" = 0 ] && row harness "exec-sync reorg" pass $(( $(date +%s) - t0 )) "$(grep -c -i 'PASS\|ok' "$LOGDIR/harness-exec-reorg.log") ok lines" || row harness "exec-sync reorg" FAIL $(( $(date +%s) - t0 )) "rc $rc: $(grep -m1 -i -E 'FAIL|error|missing' "$LOGDIR/harness-exec-reorg.log" | cut -c1-160)"
else row harness build FAIL $(( $(date +%s) - t0 )) "$(grep -m1 '^error' "$log" | cut -c1-160)"; fi
else row harness all skip 0 "$( [ "$SUBSET" = 1 ] && echo subset || echo NIGHT_SKIP)"; fi
# clippy
if ! skip clippy; then
dirs="igneum-pow"; [ "$SUBSET" = 1 ] || dirs="igneum-pow igneum-census pool proto-vdf app/igneum-app proving/igneum-prove vendor/igneum-node"
for d in $dirs; do
[ -f "$IG/$d/Cargo.toml" ] || continue
t0=$(date +%s); log="$LOGDIR/clippy-$(echo "$d" | tr '/' '_').log"
feat=""; [ "$d" = vendor/igneum-node ] && feat="--features kaspad/igneum-pow"
rc=$(cd "$IG/$d" && run_to "$log" 3600 cargo clippy --release --all-targets $feat)
w=$(grep -c '^warning: ' "$log"); e=$(grep -c '^error' "$log")
[ "$rc" = 0 ] && row clippy "$d" pass $(( $(date +%s) - t0 )) "$w warnings" || row clippy "$d" FAIL $(( $(date +%s) - t0 )) "rc $rc, $e errors, $w warnings: $(grep -m1 '^error' "$log" | cut -c1-120)"
done
else row clippy all skip 0 "NIGHT_SKIP"; fi
# audit
if ! skip audit; then
locks="igneum-pow/Cargo.lock vendor/igneum-node/Cargo.lock"; [ "$SUBSET" = 1 ] || locks=$(cd "$IG" && find . -name Cargo.lock -not -path '*/target*' -not -path './vendor/igneum-node/*' | sed 's|^\./||'; echo vendor/igneum-node/Cargo.lock)
for l in $locks; do
d=$(dirname "$l"); [ -f "$IG/$l" ] || continue
t0=$(date +%s); log="$LOGDIR/audit-$(echo "$d" | tr '/' '_').log"
rc=$(cd "$IG/$d" && run_to "$log" 600 cargo audit --color never)
v=$(grep -c '^Crate:' "$log"); wn=$(grep -c -i '^warning:' "$log")
[ "$rc" = 0 ] && row audit "$d" pass $(( $(date +%s) - t0 )) "$v vulnerabilities, $wn warnings" || row audit "$d" FAIL $(( $(date +%s) - t0 )) "rc $rc: $v vulnerabilities ($(grep -A1 '^Crate:' "$log" | grep -v '^--' | awk '{ print $2 }' | paste -sd, - | cut -c1-120)), $wn warnings"
done
else row audit all skip 0 "NIGHT_SKIP"; fi
# report
OUTDIR="$IG/docs/benchmarks/night"; mkdir -p "$OUTDIR"; OUT="$OUTDIR/$REPORT_NAME"
PREV=$(ls "$OUTDIR"/*.md 2>/dev/null | grep -v "/$REPORT_NAME$" | grep -v -- '-dryrun' | sort | tail -1)
[ "$SUBSET" = 1 ] && PREV=$(ls "$OUTDIR"/*-dryrun.md 2>/dev/null | grep -v "/$REPORT_NAME$" | sort | tail -1)
python3 - "$ROWS" "$OUT" "${PREV:-}" "$DATE" "$REPO_SHA" "$NODE_BRANCH" "$NODE_SHA" "$(( $(date +%s) - T_ALL ))" "$SUBSET" "$cargo_jobs" "$LOGDIR" <<'PY'
import re, sys, os
rows_f, out, prev, date, repo_sha, node_branch, node_sha, secs, subset, jobs, logdir = sys.argv[1:]
rows = [l.rstrip("\n").split("\t") for l in open(rows_f) if l.strip()]
def fmt(s):
s = int(s); return f"{s // 60} min {s % 60:02d} s" if s >= 60 else f"{s} s"
n_pass = sum(1 for r in rows if r[2] == "pass"); n_fail = sum(1 for r in rows if r[2] == "FAIL"); n_skip = sum(1 for r in rows if r[2] == "skip")
lines = [f"# Night battery {date}{' (dry run, subset)' if subset == '1' else ''}", "",
f"igneum-build-1, {fmt(secs)} wall, one build slot (CARGO_BUILD_JOBS {jobs}), repo master {repo_sha}, fork {node_branch} {node_sha}. "
f"{n_pass} pass, {n_fail} FAIL, {n_skip} skip. Logs on the box: `{logdir}`. Written by `infra/build-server/night/night-battery.sh`.", "",
"| Stage | What | Result | Time | Detail |", "|---|---|---|---|---|"]
for st, what, res, t, det in rows:
res_md = "**FAIL**" if res == "FAIL" else res
lines.append(f"| {st} | {what} | {res_md} | {fmt(t)} | {det.replace('|', '/')} |")
lines += ["", "## New since last night", ""]
if prev and os.path.isfile(prev):
old = {}
for l in open(prev):
m = re.match(r"\| (\w+) \| (.*?) \| (\*\*FAIL\*\*|pass|skip) \| (.*?) \| (.*) \|$", l.rstrip("\n"))
if m: old[(m.group(1), m.group(2))] = m.group(3).strip("*")
new = {(r[0], r[1]): r[2] for r in rows}
changes = []
for k, v in new.items():
if k not in old: changes.append(f"- new row: {k[0]} {k[1]}: {v}")
elif old[k] != v: changes.append(f"- {k[0]} {k[1]}: {old[k]} -> **{v}**")
for k, v in old.items():
if k not in new: changes.append(f"- gone: {k[0]} {k[1]} (was {v})")
hdr = open(prev).read().split("\n")[2] if len(open(prev).read().split("\n")) > 2 else ""
m = re.search(r"repo master (\w+), fork (\S+) (\w+)", hdr)
if m and (m.group(1) != repo_sha or m.group(3) != node_sha):
changes.insert(0, f"- commits moved: repo {m.group(1)} -> {repo_sha}, fork {m.group(2)} {m.group(3)} -> {node_branch} {node_sha}")
lines.append(f"Against `{os.path.basename(prev)}`:")
lines += changes if changes else ["- nothing: every row has the result it had"]
else:
lines.append("No earlier report to compare with: this is the first night.")
open(out, "w").write("\n".join(lines) + "\n")
print(f"{n_pass} pass, {n_fail} FAIL, {n_skip} skip; report {out}")
PY
# commit on night-battery, push to the mirror (never master)
cd "$IG" && git add docs/benchmarks/night && \
git -c user.name=igneum-labs -c user.email=337424239+igneum-labs@users.noreply.github.com commit -q -m "Night battery $DATE$( [ "$SUBSET" = 1 ] && echo ' (dry run)'): $(awk -F'\t' '$3 == "pass" { p++ } $3 == "FAIL" { f++ } END { printf "%d pass, %d FAIL", p, f }' "$ROWS") on master $REPO_SHA, fork $NODE_BRANCH $NODE_SHA
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>" && \
git push -q --force origin night-battery:refs/heads/night-battery && say "pushed night-battery to $REPO_MIRROR ($(git rev-parse --short HEAD)); on the Mac: git fetch build night-battery" || say "commit or push FAILED"
cat "$OUT"
[ "$(awk -F'\t' '$3 == "FAIL"' "$ROWS" | wc -l)" = 0 ]