7184e5bfc7
- provision.sh step_runner: actions/runner 2.338.0 (sha256 checked) at /opt/actions-runner under a dedicated user `runner`
(no sudo, not in build's group), rustup 1.99.0 pinned with both targets, sccache against /srv/sccache in READ_ONLY mode
on its own server port, Node 22 and mingw from the system, GitHub's svc.sh unit with a Nice 10 drop-in; registered on
igneum-network/igneum as igneum-build-1 (labels self-hosted, linux, x64, igneum-build-1) through
infra/build-server/runner/register.sh (gh as igneum-labs, the token on ssh stdin, never logged). Idempotent after
the env files moved behind svc.sh install (its env.sh rewrites them). libicu74 and python3-numpy added to APT.
- main's slots ruling: SLOTS default 2; remote-run.sh sets CARGO_BUILD_JOBS 90 when it holds the only taken slot and 45
when both are held, BR_MEASURE=1 takes the `measure` file exclusively and excludes builds (builds hold it shared),
lock files open in append mode (the old `exec {fd}>` truncated a busy slot's holder line on every probe), env
IGNEUM_BUILD_SLOTS_DIR and IGNEUM_BUILD_LOG_DIR win over the profile, `--self-test-slots` with five cases (the old
script fails it with JOBS=none); build-remote.sh and cross-remote.sh pass -j only when --jobs is given.
- infra/build-server/prover/cpu-trial.sh: the SP1 CPU prover on one fixture shard under the measure hold with a VmHWM
poller; 6 Oct 2026 run: core 34.2 s, compressed 85.9 s, peak RSS 28.2 GB on 96 threads, so no standing CPU prover.
- docs/plans/ci-self-hosted.md: the proposed runs-on change for ci.yml behind the repository variable IGNEUM_CI_RUNNER
(GitHub-hosted is the fallback), and why windows.yml cannot move to a Linux box. Workflows untouched.
- docs/plans/build-server.md section 7.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
66 lines
4.7 KiB
Bash
66 lines
4.7 KiB
Bash
#!/usr/bin/env bash
|
|
# CPU prover trial on igneum-build-1 (6 October 2026): time the SP1 CPU prover on one real fixture shard with every thread
|
|
# of the box, at nice 19, under one build slot (a timing taken beside another build is not a number: CLAUDE.md).
|
|
# Runs ON the box as user build:
|
|
# infra/build-server/prover/cpu-trial.sh [--fixture <json>] [--host <igneum-prove-host>] [--threads 96] [--mode shard|execute] [--out <dir>]
|
|
# Defaults: the host built from the igneum repo's master on the box (/srv/builds/igneum/proving/igneum-prove/target/release,
|
|
# tools/build-remote.sh from proving/igneum-prove; the `cuda` feature in it changes nothing for SP1_PROVER=cpu), the fixture
|
|
# block-56-transfers.json (the v0 block, one shard), 96 threads, --mode shard --shard 0.
|
|
# Writes <out>/trial-<utc>.{log,json,txt}: the host's own results JSON (execute, core, compressed and verify seconds), the
|
|
# wall time, and the peak resident memory read from /proc/<pid>/status VmHWM every second (no `time` package on the box).
|
|
set -euo pipefail
|
|
[ -f /etc/profile.d/igneum-build.sh ] && . /etc/profile.d/igneum-build.sh
|
|
FIXTURE=/srv/builds/igneum/proving/fixtures/block-56-transfers.json
|
|
HOST=/srv/builds/igneum/proving/igneum-prove/target/release/igneum-prove-host
|
|
THREADS=$(nproc); MODE=shard; OUT=/srv/builds/_log/prover-trial
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
--fixture) FIXTURE="$2"; shift 2 ;; --host) HOST="$2"; shift 2 ;; --threads) THREADS="$2"; shift 2 ;;
|
|
--mode) MODE="$2"; shift 2 ;; --out) OUT="$2"; shift 2 ;; *) echo "unknown argument $1" >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
[ -x "$HOST" ] || { echo "no prover host at $HOST (cd proving/igneum-prove && tools/build-remote.sh -- build --release -p igneum-prove-host)" >&2; exit 2; }
|
|
[ -f "$FIXTURE" ] || { echo "no fixture at $FIXTURE" >&2; exit 2; }
|
|
mkdir -p "$OUT"
|
|
STAMP=$(date -u +%Y%m%dT%H%M%SZ); LOG="$OUT/trial-$STAMP.log"; RES="$OUT/trial-$STAMP.json"; SUM="$OUT/trial-$STAMP.txt"
|
|
# the measurement hold (main's ruling 6 October 2026): remote-run.sh BR_MEASURE=1 takes the `measure` file exclusively, waits
|
|
# for every running build and blocks new ones until this run ends, and writes the run's JSONL line (kind measure)
|
|
RR="${IGNEUM_REMOTE_RUN:-$(dirname "$0")/../remote-run.sh}"; [ -f "$RR" ] || RR=/srv/builds/_bin/remote-run.sh
|
|
[ -f "$RR" ] || { echo "no remote-run.sh beside this script or at /srv/builds/_bin" >&2; exit 2; }
|
|
echo "cpu-trial: host $HOST; fixture $FIXTURE; threads $THREADS; mode $MODE; load $(cut -d' ' -f1-3 /proc/loadavg)" | tee "$LOG"
|
|
"$HOST" --version 2>/dev/null | tee -a "$LOG" || true
|
|
args=("$FIXTURE" --mode "$MODE" --out "$RES"); [ "$MODE" = shard ] && args+=(--shard 0)
|
|
t0=$(date +%s.%N)
|
|
PEAKF=$(mktemp)
|
|
# the command remote-run.sh evals under the measure hold: the host at nice 19 with a VmHWM poller beside it
|
|
CMD="SP1_PROVER=cpu RAYON_NUM_THREADS=$THREADS RUST_LOG=${RUST_LOG:-info} nice -n 19 '$HOST' $(printf '%q ' "${args[@]}") >> '$LOG' 2>&1 & p=\$!; peak=0; while kill -0 \$p 2>/dev/null; do h=\$(awk '/VmHWM/ { print \$2 }' /proc/\$p/status 2>/dev/null || echo 0); [ \${h:-0} -gt \$peak ] && peak=\$h; sleep 1; done; wait \$p; rc=\$?; echo \$peak > '$PEAKF'; ( exit \$rc )"
|
|
BR_MEASURE=1 BR_DIR="$(dirname "$HOST")" BR_CMD="$CMD" BR_LABEL="prover cpu-trial $(basename "$FIXTURE") $THREADS threads; agent=${IGNEUM_AGENT:-box-work}" \
|
|
BR_TOOL=cpu-trial BR_KIND=measure BR_WT=igneum BR_CRATE=proving/igneum-prove BR_BRANCH="${BR_BRANCH:-master}" BR_SHA="${BR_SHA:-}" BR_AGENT="${IGNEUM_AGENT:-box-work}" \
|
|
BR_COMMAND="igneum-prove-host $(basename "$FIXTURE") --mode $MODE (SP1_PROVER=cpu, $THREADS threads, nice 19)" BR_TARGET=x86_64-unknown-linux-gnu \
|
|
bash "$RR" 2>&1 | tee -a "$LOG"; rc=${PIPESTATUS[0]}
|
|
peak=$(cat "$PEAKF" 2>/dev/null || echo 0); rm -f "$PEAKF"
|
|
t1=$(date +%s.%N)
|
|
wall=$(python3 -c "print(round($t1 - $t0, 1))")
|
|
{
|
|
echo "cpu-trial $STAMP: exit $rc, wall ${wall} s (hold included), peak RSS $(( peak / 1024 )) MB (VmHWM), threads $THREADS, nice 19, measure hold, load at end $(cut -d' ' -f1-3 /proc/loadavg)"
|
|
echo "host: $HOST ($(stat -c %s "$HOST") B, sha256 $(sha256sum "$HOST" | cut -c1-16)...)"
|
|
echo "fixture: $FIXTURE"
|
|
if [ -f "$RES" ]; then
|
|
python3 - "$RES" <<'PY'
|
|
import json, sys
|
|
d = json.load(open(sys.argv[1]))
|
|
def walk(o, p=""):
|
|
if isinstance(o, dict):
|
|
for k, v in o.items(): walk(v, f"{p}.{k}" if p else k)
|
|
elif isinstance(o, list):
|
|
for i, v in enumerate(o): walk(v, f"{p}[{i}]")
|
|
else:
|
|
s = str(o)
|
|
if any(t in p.lower() for t in ("sec", "time", "cycles", "bytes", "verified", "prover", "shard", "program", "stage")) and len(s) < 80:
|
|
print(f" {p} = {s}")
|
|
walk(d)
|
|
PY
|
|
fi
|
|
echo "log lines with times:"; grep -E -i 'prove|core|compress|verif|execute|cycles| s\b|seconds' "$LOG" | grep -v '^cpu-trial' | tail -20 | sed 's/^/ /'
|
|
} | tee "$SUM"
|
|
exit "$rc"
|