diff --git a/infra/build-server/night/igneum-night-battery.service b/infra/build-server/night/igneum-night-battery.service new file mode 100644 index 000000000..0ff06cb56 --- /dev/null +++ b/infra/build-server/night/igneum-night-battery.service @@ -0,0 +1,34 @@ +# igneum-build-1: the night battery (infra/build-server/night/night-battery.sh) under one build slot through remote-run.sh. +# Installed by infra/build-server/provision.sh step_night; started by igneum-night-battery.timer at 02:00 Europe/London. +[Unit] +Description=Igneum night battery (every test suite, fuzz, simulators, harnesses, clippy, audit; report on branch night-battery) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=build +Group=build +Nice=19 +IOSchedulingClass=idle +WorkingDirectory=/srv/builds/_night +Environment=BR_DIR=/srv/builds/_night +Environment=BR_CMD=/srv/builds/_bin/night-battery.sh +Environment="BR_LABEL=night battery; agent=night" +Environment=BR_TOOL=night-battery +Environment=BR_KIND=other +Environment=BR_WT=_night +Environment=BR_CRATE=. +Environment=BR_BRANCH=master +Environment=BR_SHA= +Environment=BR_AGENT=night +Environment="BR_COMMAND=night-battery.sh (suites, fuzz, sims, harness, clippy, audit)" +Environment=BR_TARGET=x86_64-unknown-linux-gnu +Environment=IGNEUM_AGENT=night +ExecStart=/bin/bash /srv/builds/_bin/remote-run.sh +TimeoutStartSec=8h +StandardOutput=append:/srv/builds/_log/night-battery.log +StandardError=append:/srv/builds/_log/night-battery.log + +[Install] +WantedBy=multi-user.target diff --git a/infra/build-server/night/igneum-night-battery.timer b/infra/build-server/night/igneum-night-battery.timer new file mode 100644 index 000000000..fdc5b114b --- /dev/null +++ b/infra/build-server/night/igneum-night-battery.timer @@ -0,0 +1,12 @@ +# igneum-build-1: 02:00 London every night (the box's clock is Europe/Berlin; the time zone is named here, so BST and GMT both +# land at 02:00 UK). Not Persistent: a missed night is not run during the day. +[Unit] +Description=Igneum night battery at 02:00 Europe/London + +[Timer] +OnCalendar=*-*-* 02:00:00 Europe/London +Persistent=false +AccuracySec=1min + +[Install] +WantedBy=timers.target diff --git a/infra/build-server/night/night-battery.sh b/infra/build-server/night/night-battery.sh new file mode 100755 index 000000000..4a93d3ef1 --- /dev/null +++ b/infra/build-server/night/night-battery.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +# The night battery on igneum-build-1 (6 October 2026): one invocation, one build slot, every test the repo and the fork have. +# Runs ON the box as user build at 02:00 Europe/London (igneum-night-battery.timer) through remote-run.sh, which holds the +# slot for the whole run, sets CARGO_BUILD_JOBS (90 alone, 45 beside another build) and writes the JSONL line. By hand: +# /srv/builds/_bin/night-battery.sh the full battery (hours) +# NIGHT_SUBSET=1 /srv/builds/_bin/night-battery.sh the dry-run subset (igneum-pow suite and fuzz, two fork crates, quick sims, clippy and audit on igneum-pow) +# NIGHT_NODE_BRANCH=release-0.3.15-node ... the fork branch (default: the newest release-*-node on the mirror) +# NIGHT_SKIP="harness sims" ... skip stages by name +# Stages, each a row (pass, FAIL, skip) with seconds and a detail: +# checkout /srv/builds/_night/igneum from /srv/igneum.git master (the battery re-execs itself from that checkout, so the +# script that runs is master's), vendor/igneum-node from /srv/igneum-node.git at the fork branch +# suites cargo test --release --no-fail-fast per crate: the repo's crates (igneum-pow, igneum-census, pool, proto-vdf, +# app/igneum-app, every member of proving/igneum-prove) and every workspace member of the fork (kaspad with +# --features igneum-pow); the pass and fail counts are read from the `test result:` lines +# fuzz igneum-pow's two fuzz tests at 10x their default (IGNEUM_MIXER_FUZZ and IGNEUM_SCRATCH_FUZZ 2000) +# sims sim/finality_sim.py, sim/finality_v2.py and sim/difficulty/sim.py in full (the subset runs --quick) +# harness the fork's igneumd, igneum-miner, igneum-harness-sim and igneum-p2p-probe built into target-integration, then +# tools/finality-attacks/run.mjs --fast-time, tools/harness/run.mjs s3 s4 --fast-time --no-bench-log and +# tools/exec-sync/reorg.mjs (loopback ports 27200+, 27800+, 29870+; nothing of the live devnet) +# clippy cargo clippy --release --all-targets per crate dir (warnings counted; a row fails on an error) +# audit cargo audit in every directory with a Cargo.lock +# report docs/benchmarks/night/.md (a pass/fail table and "new since last night" against the newest earlier +# report), committed as igneum-labs on branch night-battery (based on master, earlier reports carried over) and +# pushed to /srv/igneum.git night-battery; main merges (`git fetch build night-battery` on the Mac). Never master. +set -uo pipefail +_slots_env="${IGNEUM_BUILD_SLOTS_DIR:-}"; [ -f /etc/profile.d/igneum-build.sh ] && . /etc/profile.d/igneum-build.sh; [ -n "$_slots_env" ] && IGNEUM_BUILD_SLOTS_DIR="$_slots_env" +NIGHT_ROOT="${NIGHT_ROOT:-/srv/builds/_night}"; REPO_MIRROR=/srv/igneum.git; NODE_MIRROR=/srv/igneum-node.git +SUBSET="${NIGHT_SUBSET:-0}"; SKIP=" ${NIGHT_SKIP:-} " +DATE=$(date -u +%Y-%m-%d); STAMP=$(date -u +%Y%m%dT%H%M%SZ); T_ALL=$(date +%s) +REPORT_NAME="$DATE$( [ "$SUBSET" = 1 ] && echo -dryrun ).md" +LOGDIR="$NIGHT_ROOT/logs/$STAMP"; mkdir -p "$LOGDIR" +ROWS="$LOGDIR/rows.tsv"; : > "$ROWS" +say() { printf '%s night: %s\n' "$(date -u +%H:%M:%S)" "$*" >&2; } +row() { printf '%s\t%s\t%s\t%s\t%s\n' "$1" "$2" "$3" "$4" "$5" >> "$ROWS"; say "$1 | $2 | $3 | ${4}s | $5"; } # stage name status secs detail +skip() { case "$SKIP" in *" $1 "*) return 0 ;; esac; return 1; } +cargo_jobs="${CARGO_BUILD_JOBS:-90}" + +# checkout +IG="$NIGHT_ROOT/igneum"; FORK="$IG/vendor/igneum-node" +t0=$(date +%s) +if [ ! -d "$IG/.git" ]; then git clone -q "$REPO_MIRROR" "$IG" || { row checkout repo FAIL 0 "clone failed"; exit 1; }; fi +git -C "$IG" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' || { row checkout repo FAIL 0 "fetch failed"; exit 1; } +git -C "$IG" checkout -q -- . 2>/dev/null; git -C "$IG" clean -qfd -e target -e 'target-*' -e vendor +git -C "$IG" checkout -q -B night-battery origin/master +# earlier reports not yet merged into master ride along +git -C "$IG" checkout -q origin/night-battery -- docs/benchmarks/night 2>/dev/null || true +REPO_SHA=$(git -C "$IG" rev-parse --short HEAD) +if [ "${NIGHT_REEXEC:-0}" != 1 ] && [ -f "$IG/infra/build-server/night/night-battery.sh" ] && ! cmp -s "$0" "$IG/infra/build-server/night/night-battery.sh"; then + say "re-exec from master's copy ($REPO_SHA)"; NIGHT_REEXEC=1 exec bash "$IG/infra/build-server/night/night-battery.sh" +fi +NODE_BRANCH="${NIGHT_NODE_BRANCH:-$(git -C "$NODE_MIRROR" branch --list 'release-*-node' | tr -d ' *' | sort -V | tail -1)}" +[ -n "$NODE_BRANCH" ] || NODE_BRANCH=master +mkdir -p "$IG/vendor" +if [ ! -d "$FORK/.git" ]; then git clone -q --no-checkout "$NODE_MIRROR" "$FORK" || { row checkout fork FAIL 0 "clone failed"; exit 1; }; fi +git -C "$FORK" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' +git -C "$FORK" checkout -q -- . 2>/dev/null; git -C "$FORK" clean -qfd -e target -e 'target-*' +git -C "$FORK" checkout -q -B "$NODE_BRANCH" "origin/$NODE_BRANCH" || { row checkout fork FAIL 0 "no branch $NODE_BRANCH"; exit 1; } +NODE_SHA=$(git -C "$FORK" rev-parse --short HEAD) +row checkout "repo master $REPO_SHA, fork $NODE_BRANCH $NODE_SHA" pass $(( $(date +%s) - t0 )) "$IG; jobs $cargo_jobs; subset $SUBSET" + +# helpers +run_to() { # : runs in the current dir, returns the exit code, 124 on timeout + local log="$1" to="$2"; shift 2 + timeout --signal=TERM --kill-after=60 "$to" "$@" > "$log" 2>&1; echo $? +} +counts() { # pass/fail totals from cargo test output + awk '/^test result:/ { for (i = 1; i <= NF; i++) { if ($(i+1) == "passed;") p += $i; if ($(i+1) == "failed;") f += $i } } END { printf "%d passed, %d failed", p, f }' "$1" +} +suite() { #