Merge commit 'e1c55556'
This commit is contained in:
igneum-labs
commit
8cd9b69d68
10 changed files with 458 additions and 35 deletions
4
.github/workflows/ci.yml
vendored
4
.github/workflows/ci.yml
vendored
|
|
@ -77,7 +77,9 @@ jobs:
|
|||
sims:
|
||||
name: simulators, quick modes
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.code == 'true' }}
|
||||
# master and release-* pushes, and pull requests into them, only (main, 7 October 2026: every code push cost two box jobs and the
|
||||
# queue read 22); a feature-branch code push runs the igneum-pow tests alone. tools/ci/sims-branch-check.sh holds this rule.
|
||||
if: ${{ needs.changes.outputs.code == 'true' && ((github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))) || (github.event_name == 'pull_request' && (github.base_ref == 'master' || startsWith(github.base_ref, 'release-')))) }}
|
||||
runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
|
|||
|
|
@ -377,3 +377,28 @@ spill-over. Now (lib.sh `bs_route_spill`, master from this commit):
|
|||
them keeps its number; since this commit a bounded run takes the band its slot owns (slot 0 the last 32 cores, slot 1 the 32
|
||||
below, slot 2 the 32 below that), so three bounded runs never share a core. A gate still takes its slot ahead of queued suites.
|
||||
- `--box N` still pins. Self-test: tools/ci/route-spill-check.sh (thirteen cases through `BS_ROUTE_STATE_<n>`, no ssh), in the gate.
|
||||
|
||||
## 8. The measure file is retired: per-core leases and the quiet class (7 October 2026, 15:07 UK)
|
||||
|
||||
Main's reading at 15:07 UK: on build-1 a sync-fuzz probe (the capacity lane's, SIGSTOPped since 09:45Z, no owner) held the global
|
||||
measure flock for five and a half hours; beside it attack-f6's phase2b waited exclusive on the same file behind seven F2 solvers
|
||||
holding it shared on cores 6-11,54-59, and every new shared taker (every build) queued behind the exclusive waiter: load 120, slots
|
||||
free, nine waiting. On build-2 the era VDF bench held the file exclusive while pinned to one core and five jobs waited. One global
|
||||
exclusive lock across unrelated measurements was the wrong design. Now:
|
||||
|
||||
- `infra/build-server/lease.sh`, installed on every box at `/srv/builds/_bin/lease` (provision.sh; by hand on build-1 and build-2
|
||||
at 14:2x BST): `lease cores <set> --label "..." [--owner <agent>] -- <cmd>` takes a lease on THOSE CORES ONLY (one flock per
|
||||
core, `_locks/core-<n>`, a `wait-<pid>` file with the label while it waits), runs the command under nice 10 and taskset, and
|
||||
releases. Nothing else is excluded. `lease quiet --label "..." --owner <agent> -- <cmd>` is the whole-box class: refused (exit
|
||||
73) while any slot or core lease is held, capped at 20 minutes, holder line with the owner in `_locks/quiet`. `lease status`,
|
||||
`lease reap`.
|
||||
- remote-run.sh: an unbounded run (nice 0, the full set) takes `quiet` shared and waits for it; a bounded run (suites, benches,
|
||||
everything on box 2) never takes it. Every run keeps off leased cores (its set minus the `core-<n>` flocks, said once). A run's
|
||||
keeper refreshes its holder file's mtime every 20 s and calls `lease reap`: a holder of a lease, the quiet file or a slot whose
|
||||
process has been STOPPED for 5 minutes is killed and its file cleared, one line each in `_log/reaped.log`. The keeper closes the
|
||||
lock descriptors it inherits (an orphaned `sleep 20` held a slot and a worktree lock 20 s past the release). BR_MEASURE=1 is the
|
||||
quiet class with the same refusals; it needs a named owner (IGNEUM_AGENT).
|
||||
- The lanes' own `flock -s /srv/builds/_locks/measure -c "nice -n 10 taskset -c <set> ..."` lines no longer hold anything a build
|
||||
waits for; they become `/srv/builds/_bin/lease cores <set> --label "..." -- <cmd>`, and `flock -x .../measure` becomes
|
||||
`lease quiet`. Self-tests: lease.sh --self-test and remote-run.sh --self-test-slots, run on build-1 by tools/ci/box-locks-check.sh
|
||||
in the gate.
|
||||
|
|
|
|||
|
|
@ -37,11 +37,12 @@ CAP_NODE_BRANCH="${CAP_NODE_BRANCH:-}"; CAP_REPO_SHA="${CAP_REPO_SHA:-}"; CAP_FO
|
|||
|
||||
# ---- the build-slot and measure hold the layer yields to --------------------------------------------------------------
|
||||
# a slot or the measure file is "held" when flock -n cannot take it (another process holds the fd). The same probe the
|
||||
# collector uses (tools/workers/collect.mjs flockHeld). Returns 0 (true) when ANY build slot or the measure hold is taken.
|
||||
# collector uses (tools/workers/collect.mjs flockHeld). Returns 0 (true) when ANY build slot, the quiet hold or a core lease is taken.
|
||||
cap_build_active() {
|
||||
local slots k f
|
||||
slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1
|
||||
if [ -e "$LOCKS/measure" ] && ! flock -n "$LOCKS/measure" true 2>/dev/null; then return 0; fi
|
||||
if [ -e "$LOCKS/quiet" ] && ! flock -n "$LOCKS/quiet" true 2>/dev/null; then return 0; fi # 7 Oct 2026: the quiet class replaced the measure file
|
||||
for f in "$LOCKS"/core-*; do [ -e "$f" ] && ! flock -n "$f" true 2>/dev/null && return 0; done
|
||||
for k in $(seq 0 $((slots - 1))); do
|
||||
f="$LOCKS/build-$k"
|
||||
[ -e "$f" ] || continue
|
||||
|
|
@ -51,7 +52,8 @@ cap_build_active() {
|
|||
}
|
||||
cap_hold_reason() {
|
||||
local slots k
|
||||
if [ -e "$LOCKS/measure" ] && ! flock -n "$LOCKS/measure" true 2>/dev/null; then echo "measure hold"; return; fi
|
||||
if [ -e "$LOCKS/quiet" ] && ! flock -n "$LOCKS/quiet" true 2>/dev/null; then echo "quiet hold"; return; fi
|
||||
for f in "$LOCKS"/core-*; do [ -e "$f" ] && ! flock -n "$f" true 2>/dev/null && { echo "core lease"; return; }; done
|
||||
slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1
|
||||
for k in $(seq 0 $((slots - 1))); do
|
||||
[ -e "$LOCKS/build-$k" ] || continue
|
||||
|
|
|
|||
155
infra/build-server/lease.sh
Executable file
155
infra/build-server/lease.sh
Executable file
|
|
@ -0,0 +1,155 @@
|
|||
#!/usr/bin/env bash
|
||||
# Measurement leases on a build box (main, 7 October 2026, 15:07 UK: one global exclusive "measure" flock across unrelated
|
||||
# measurements stalled build-1 at load 120 with free slots, an exclusive waiter queueing every new shared taker behind it, and
|
||||
# a stopped probe held the box for five and a half hours). The measure file is retired. In its place:
|
||||
#
|
||||
# lease cores <set> --label "<text>" [--owner <agent>] [--nice N] -- <command...>
|
||||
# A measurement that pins cores takes a lease on THOSE CORES ONLY (one flock per core, _locks/core-<n>, taken in ascending
|
||||
# order, waited for up to 2 h with a wait-<pid> file carrying the label), runs the command under nice N (default 10) and
|
||||
# taskset on the set, and releases. Builds and suites keep off leased cores (remote-run.sh reads the core files before it
|
||||
# pins its own set). Nothing else is excluded: the box stays open.
|
||||
# lease quiet --label "<text>" --owner <agent> [--cap-s N] -- <command...>
|
||||
# A WHOLE-BOX quiet measurement: its own class, refused (exit 73) while any build slot or any core lease is held, capped at
|
||||
# 20 minutes (timeout; --cap-s at most 1200), holder line with the owner in _locks/quiet. Unbounded builds take quiet
|
||||
# shared and wait for it; bounded suites (nice 10, a 32-core band) never take it.
|
||||
# lease status every lease, the quiet holder and every waiter with its label
|
||||
# lease reap a holder (lease, quiet or build slot) whose process has been STOPPED (state T) for 5 minutes or more is
|
||||
# killed and its file cleared, one line each in _log/reaped.log; remote-run.sh's keeper calls this every
|
||||
# 20 s while any run is on the box (LEASE_REAP_S overrides the 300 s for the self-test)
|
||||
# lease --self-test the known cases against a scratch lock directory (in the gate: tools/ci/pre-push.sh)
|
||||
#
|
||||
# Installed on every box at /srv/builds/_bin/lease by provision.sh (and copied by hand on 7 October 2026); the lock directory is
|
||||
# IGNEUM_BUILD_SLOTS_DIR (the profile sets /srv/builds/_locks), the log directory IGNEUM_BUILD_LOG_DIR. Files: core-<n> (flock),
|
||||
# lease-<pid> (holder line "pid N since HH:MM:SSZ cores <set>: <label>; owner=<agent>"), quiet (holder line), wait-<pid>.
|
||||
set -uo pipefail
|
||||
LOCKS="${IGNEUM_BUILD_SLOTS_DIR:-/srv/builds/_locks}"; LOGS="${IGNEUM_BUILD_LOG_DIR:-/srv/builds/_log}"
|
||||
REAP_S="${LEASE_REAP_S:-300}"
|
||||
now() { date -u +%H:%M:%SZ; }
|
||||
say() { echo "lease: $*" >&2; }
|
||||
expand_set() { # "6-11,54-59" -> "6 7 8 9 10 11 54 ..."
|
||||
local out="" part lo hi; IFS=',' read -ra parts <<<"$1"
|
||||
for part in "${parts[@]}"; do case "$part" in *-*) lo=${part%-*}; hi=${part#*-} ;; *) lo=$part; hi=$part ;; esac
|
||||
[ "$lo" -le "$hi" ] 2>/dev/null || { say "bad core set '$1'"; return 2; }
|
||||
for ((c = lo; c <= hi; c++)); do out="$out $c"; done; done
|
||||
echo "${out# }"
|
||||
}
|
||||
held_cores() { # the cores whose lease file is flocked right now, as a space list
|
||||
local f c out=""
|
||||
for f in "$LOCKS"/core-*; do [ -e "$f" ] || continue; c=${f##*/core-}
|
||||
exec 8>>"$f"; if ! flock -n 8; then out="$out $c"; fi; exec 8>&-; done
|
||||
echo "${out# }"
|
||||
}
|
||||
any_slot_held() { local n f k; n=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); for ((k = 0; k < n; k++)); do f="$LOCKS/build-$k"; [ -e "$f" ] || continue; exec 8>>"$f"; if ! flock -n 8; then exec 8>&-; return 0; fi; exec 8>&-; done; return 1; }
|
||||
pid_of() { sed -n 's/^pid \([0-9]*\) .*/\1/p' "$1" 2>/dev/null | head -1; }
|
||||
reap() { # stopped holders older than REAP_S are killed, their files cleared, one line each in reaped.log
|
||||
local f p st age line n=0
|
||||
for f in "$LOCKS"/lease-* "$LOCKS"/quiet "$LOCKS"/build-[0-9]*; do
|
||||
[ -s "$f" ] || continue; p=$(pid_of "$f"); [ -n "$p" ] || continue
|
||||
st=$(ps -o stat= -p "$p" 2>/dev/null | tr -d ' '); case "$st" in T*) ;; *) continue ;; esac
|
||||
age=$(( $(date +%s) - $(stat -c %Y "$f") ))
|
||||
# the file's mtime is refreshed by the holder's keeper every 20 s while it runs; a stopped holder's file goes stale
|
||||
[ "$age" -ge "$REAP_S" ] || continue
|
||||
line="$(now) reaped pid $p (stopped $age s, state $st) holding $(basename "$f"): $(head -c 160 "$f" | tr '\n' ' ')"
|
||||
kill -KILL "$p" 2>/dev/null; pkill -KILL -P "$p" 2>/dev/null; : > "$f"; case "$f" in */lease-*) rm -f "$f" ;; esac
|
||||
mkdir -p "$LOGS"; echo "$line" >> "$LOGS/reaped.log"; say "$line"; n=$((n + 1))
|
||||
done
|
||||
echo "$n"
|
||||
}
|
||||
status() {
|
||||
echo "leases:"; for f in "$LOCKS"/lease-*; do [ -s "$f" ] && echo " $(cat "$f")"; done 2>/dev/null
|
||||
echo "quiet: $(cat "$LOCKS/quiet" 2>/dev/null)"
|
||||
echo "held cores: $(held_cores)"
|
||||
echo "waiting:"; for f in "$LOCKS"/wait-*; do [ -s "$f" ] && echo " $(cat "$f")"; done 2>/dev/null
|
||||
}
|
||||
run_cores() {
|
||||
local set="$1"; shift; local label="" owner="${IGNEUM_AGENT:-unknown}" nice=10
|
||||
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --nice) nice="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
|
||||
[ -n "$label" ] || { say "--label is required (the dashboard and the next lane read it)"; exit 2; }
|
||||
[ $# -gt 0 ] || { say "no command"; exit 2; }
|
||||
local cores c fd t0 waited=0 line waitfile="$LOCKS/wait-$$"
|
||||
cores=$(expand_set "$set") || exit 2
|
||||
mkdir -p "$LOCKS"; t0=$(date +%s)
|
||||
line="pid $$ since $(now) waited 0 s: lease cores $set: $label; owner=$owner"; printf '%s\n' "$line" > "$waitfile"
|
||||
trap 'rm -f "$waitfile" "$LOCKS/lease-$$"' EXIT
|
||||
local fds=()
|
||||
for c in $cores; do
|
||||
exec {fd}>>"$LOCKS/core-$c"
|
||||
if ! flock -n "$fd"; then say "core $c is leased ($(for f in "$LOCKS"/lease-*; do grep -l " core-$c\b\| cores [^:]*\b$c\b" "$f" 2>/dev/null; done | head -1 | xargs -r cat | cut -c1-120)); waiting"; flock -w 7200 "$fd" || { say "gave up waiting for core $c after 2 h"; exit 75; }; fi
|
||||
fds+=("$fd")
|
||||
done
|
||||
waited=$(( $(date +%s) - t0 ))
|
||||
line="pid $$ since $(now) waited $waited s: lease cores $set: $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/lease-$$"; rm -f "$waitfile"
|
||||
say "holding cores $set (waited $waited s): $label"
|
||||
# the keeper closes the lock descriptors first: an inherited flock would outlive the release in its orphaned sleep
|
||||
( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$!
|
||||
nice -n "$nice" taskset -c "$set" "$@"; local rc=$?
|
||||
pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
|
||||
rm -f "$LOCKS/lease-$$"; say "released cores $set after $(( $(date +%s) - t0 )) s, exit $rc"
|
||||
exit $rc
|
||||
}
|
||||
run_quiet() {
|
||||
local label="" owner="${IGNEUM_AGENT:-}" cap=1200
|
||||
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --cap-s) cap="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
|
||||
[ -n "$label" ] && [ -n "$owner" ] || { say "quiet needs --label and --owner (a named owner, the 5.5-hour rule)"; exit 2; }
|
||||
[ "$cap" -le 1200 ] 2>/dev/null || cap=1200
|
||||
[ $# -gt 0 ] || { say "no command"; exit 2; }
|
||||
mkdir -p "$LOCKS"
|
||||
if any_slot_held; then say "REFUSED: a build slot is held; a whole-box quiet measurement waits for an idle box (lease status)"; exit 73; fi
|
||||
if [ -n "$(held_cores)" ]; then say "REFUSED: cores $(held_cores | tr ' ' ',') are leased; a whole-box quiet measurement waits for an idle box"; exit 73; fi
|
||||
exec 7>>"$LOCKS/quiet"
|
||||
if ! flock -n 7; then say "REFUSED: another quiet measurement holds the box: $(head -c 160 "$LOCKS/quiet")"; exit 73; fi
|
||||
local t0 line; t0=$(date +%s)
|
||||
line="pid $$ since $(now) waited 0 s: quiet (cap $cap s): $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/quiet"
|
||||
trap ': > "$LOCKS/quiet"' EXIT
|
||||
( exec 7>&-; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/quiet" 2>/dev/null; [ -s "$LOCKS/quiet" ] || printf '%s\n' "$line" > "$LOCKS/quiet"; sleep 20; done ) & local keeper=$!
|
||||
say "holding the box quiet (cap $cap s): $label"
|
||||
timeout --signal TERM --kill-after 30 "$cap" "$@"; local rc=$?
|
||||
pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
|
||||
: > "$LOCKS/quiet"; [ "$rc" = 124 ] && say "the quiet measurement hit its ${cap}s cap and was ended"
|
||||
say "released the box after $(( $(date +%s) - t0 )) s, exit $rc"; exit $rc
|
||||
}
|
||||
self_test() {
|
||||
t=$(mktemp -d); trap 'rm -rf "$t"' EXIT
|
||||
export IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" LEASE_REAP_S=2; mkdir -p "$t/locks"; echo 2 > "$t/locks/slots"
|
||||
local me="$0" fail=0
|
||||
f() { echo "lease self-test: FAIL: $*"; fail=1; }
|
||||
# 1. two leases on disjoint cores run together; the same core waits
|
||||
bash "$me" cores 0-1 --label A -- bash -c "date +%s.%N > '$t/a.start'; sleep 2; date +%s.%N > '$t/a.end'" 2>/dev/null &
|
||||
sleep 0.5; bash "$me" cores 2-3 --label B -- bash -c "date +%s.%N > '$t/b.start'" 2>/dev/null; sleep 0.2
|
||||
python3 -c "import sys; sys.exit(0 if float(open('$t/b.start').read()) < float(open('$t/a.start').read()) + 1.5 else 1)" || f "a lease on other cores waited for an unrelated lease"
|
||||
bash "$me" cores 1-2 --label C -- bash -c "date +%s.%N > '$t/c.start'" 2>/dev/null
|
||||
python3 -c "import sys; sys.exit(0 if float(open('$t/c.start').read()) >= float(open('$t/a.end').read()) else 1)" || f "a lease sharing a core started before the holder released it"
|
||||
wait
|
||||
[ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "lease or wait files left behind: $(ls "$t/locks")"
|
||||
# 2. quiet is refused while a slot is held, and runs on an idle box with a holder line naming the owner
|
||||
( exec 9>>"$t/locks/build-0"; flock 9; echo "pid $BASHPID since x waited 0 s: fake build" > "$t/locks/build-0"; sleep 2 ) &
|
||||
sleep 0.3; bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a slot was held"
|
||||
wait; : > "$t/locks/build-0"
|
||||
bash "$me" quiet --label Q --owner tester -- bash -c "grep -q 'owner=tester' '$t/locks/quiet'" || f "quiet did not run on an idle box with the owner in its holder line"
|
||||
bash "$me" quiet --label Q --owner tester --cap-s 1 -- sleep 5; [ $? = 124 ] || f "the quiet cap did not end the command"
|
||||
# 3. a quiet is refused while a core is leased
|
||||
bash "$me" cores 0 --label L -- sleep 2 2>/dev/null & sleep 0.5
|
||||
bash "$me" quiet --label Q --owner tester -- true 2>/dev/null; [ $? = 73 ] || f "quiet was not refused while a core was leased"
|
||||
wait
|
||||
# 4. reap: a stopped holder older than the window is killed and its file cleared, with a line
|
||||
bash "$me" cores 5 --label S -- sleep 60 2>/dev/null & local lp=$!; sleep 0.6
|
||||
local hp; hp=$(sed -n 's/^pid \([0-9]*\) .*/\1/p' "$t"/locks/lease-* | head -1); kill -STOP "$hp"; sleep 2.5
|
||||
touch -d '-10 seconds' "$t"/locks/lease-* 2>/dev/null
|
||||
[ "$(bash "$me" reap 2>/dev/null)" = 1 ] || f "the stopped lease holder was not reaped"
|
||||
grep -q "reaped pid $hp" "$t/log/reaped.log" 2>/dev/null || f "no reaped line was written"
|
||||
[ -z "$(ls "$t/locks" | grep '^lease-')" ] || f "the reaped lease file remains"
|
||||
kill -KILL "$lp" 2>/dev/null; wait 2>/dev/null
|
||||
# 5. a running (not stopped) holder is left alone
|
||||
bash "$me" cores 6 --label R -- sleep 3 2>/dev/null & sleep 0.6; touch -d '-10 seconds' "$t"/locks/lease-*
|
||||
[ "$(bash "$me" reap 2>/dev/null)" = 0 ] || f "a running holder was reaped"; wait
|
||||
[ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept"
|
||||
return $fail
|
||||
}
|
||||
case "${1:-}" in
|
||||
cores) shift; run_cores "$@" ;;
|
||||
quiet) shift; run_quiet "$@" ;;
|
||||
status) status ;;
|
||||
reap) reap ;;
|
||||
--self-test) self_test ;;
|
||||
*) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
|
||||
esac
|
||||
|
|
@ -74,6 +74,7 @@ RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1,ci-red}" # added to the defau
|
|||
# holds the red watcher's record file and poster. A second box: BOX_HOSTNAME=igneum-build-2
|
||||
# RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 (register.sh --host)
|
||||
RUNNER_CPUS="${RUNNER_CPUS:-}" # AllowedCPUs for the runner's service when set (a second box is bounded like a suite: 32 cores, nice 10)
|
||||
case "$BOX_HOSTNAME" in *-2|*-3) [ -n "$RUNNER_CPUS" ] || RUNNER_CPUS="0-31" ;; esac # build-2 and build-3 join the pool (label igneum-build-1) bounded to 32 cores at Nice 10 (main, 7 Oct 2026)
|
||||
RUNNER_JOBS="${RUNNER_JOBS:-48}" # cargo jobs for a CI job: half the box, the agents' builds keep the rest
|
||||
RUNNER_TOKEN="${RUNNER_TOKEN:-}" # a registration token (1 h), from infra/build-server/runner/register.sh over stdin; never logged
|
||||
RUNNER_SCCACHE_PORT="${RUNNER_SCCACHE_PORT:-4227}" # the runner's own sccache server; 4226 is the build user's
|
||||
|
|
@ -150,6 +151,10 @@ APT_PACKAGES=(
|
|||
libicu74 python3-numpy
|
||||
# innoextract: tools/repro reads the shipped igneumd.exe and igneum-miner.exe out of the public Inno Setup installer
|
||||
innoextract
|
||||
# headless Chromium (the dashboard lane's site captures): the 16 system libraries it dlopens, found missing on both boxes
|
||||
# on 7 October 2026 (installed by hand at 14:5x UK; step_headless_check launches the shell once and prints its version)
|
||||
libatk1.0-0t64 libatk-bridge2.0-0t64 libatspi2.0-0t64 libcairo2 libcups2t64 libgbm1 libpango-1.0-0 libx11-6 libxcb1
|
||||
libxcomposite1 libxdamage1 libxext6 libxfixes3 libxrandr2 libasound2t64 libxkbcommon0 fonts-liberation
|
||||
)
|
||||
step_apt() {
|
||||
local need=() p
|
||||
|
|
@ -243,6 +248,27 @@ step_dirs() {
|
|||
done
|
||||
[ "$any" = 1 ] && changed dirs "/srv/builds ($(worktree_count) worktree dirs), /srv/sccache, slots=$SLOTS" || ok dirs "$(worktree_count) worktree dirs, slots=$SLOTS"
|
||||
}
|
||||
# the lease tool (infra/build-server/lease.sh: per-core measurement leases, the quiet class, the reaper; 7 October 2026) from the
|
||||
# mirror's master at /srv/builds/_bin/lease, which remote-run.sh's keeper calls; the mirror is pushed by step_mirrors' caller
|
||||
step_lease_tool() {
|
||||
local src="/srv/igneum.git" want have=""
|
||||
install -d -m 755 -o "$BUILD_USER" -g "$BUILD_USER" /srv/builds/_bin
|
||||
want=$(git -C "$src" show master:infra/build-server/lease.sh 2>/dev/null) || { ok lease-tool "mirror has no master yet; run-from-mac.sh installs it on the next provision"; return; }
|
||||
[ -f /srv/builds/_bin/lease ] && have=$(cat /srv/builds/_bin/lease)
|
||||
if [ "$want" = "$have" ]; then ok lease-tool "/srv/builds/_bin/lease is the mirror's master copy"; return; fi
|
||||
printf '%s\n' "$want" > /srv/builds/_bin/lease.new; chmod 755 /srv/builds/_bin/lease.new; chown "$BUILD_USER:$BUILD_USER" /srv/builds/_bin/lease.new
|
||||
mv /srv/builds/_bin/lease.new /srv/builds/_bin/lease; changed lease-tool "/srv/builds/_bin/lease installed from the mirror's master"
|
||||
}
|
||||
# headless Chromium self-test: the shell the dashboard lane's node tooling downloads (playwright or puppeteer cache of the build
|
||||
# user) launches once with --headless and prints its version; when no shell is downloaded yet the libraries are checked by ldd of
|
||||
# nothing, so the step only says so (the apt list above carries them)
|
||||
step_headless_check() {
|
||||
local shell="" v
|
||||
shell=$(find "/home/$BUILD_USER/.cache/ms-playwright" "/home/$BUILD_USER/.cache/puppeteer" /srv -maxdepth 6 -type f \( -name headless_shell -o -name chrome-headless-shell -o -name chrome \) 2>/dev/null | head -1)
|
||||
[ -n "$shell" ] || { ok headless "no headless shell downloaded yet (the 16 libraries are installed; the lane's first capture downloads it)"; return; }
|
||||
if v=$(sudo -u "$BUILD_USER" timeout 60 "$shell" --headless --no-sandbox --disable-gpu --version 2>&1 | head -1) && [ -n "$v" ]; then ok headless "$shell: $v"
|
||||
else die "headless shell $shell does not launch: $v"; fi
|
||||
}
|
||||
|
||||
step_mirrors() {
|
||||
local r any=0
|
||||
|
|
@ -639,6 +665,7 @@ do_provision() {
|
|||
step_docker
|
||||
step_dirs
|
||||
step_mirrors
|
||||
step_lease_tool
|
||||
step_rustup
|
||||
step_sccache
|
||||
step_cargo_config
|
||||
|
|
@ -652,6 +679,7 @@ do_provision() {
|
|||
step_cargo_tools
|
||||
step_zig
|
||||
step_night
|
||||
step_headless_check
|
||||
step_summary
|
||||
log "done"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -181,8 +181,8 @@ if [ "${1:-}" = --self-test-slots ]; then
|
|||
mkdir -p "$t/locks" "$t/log" "$t/dir"; echo 2 > "$t/locks/slots"
|
||||
fake() { # <name> <seconds> [BR_MEASURE=1]: a fake run that records its start and end epoch and the job count it was given
|
||||
local name="$1" secs="$2" measure="${3:-0}"
|
||||
IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" BR_MEASURE="$measure" BR_DIR="$t/dir" BR_CMD="date +%s.%N > '$t/$name.start'; echo JOBS=\${CARGO_BUILD_JOBS:-none} > '$t/$name.jobs'; sleep $secs; date +%s.%N > '$t/$name.end'" \
|
||||
BR_LABEL="self-test $name" BR_TOOL=self-test BR_KIND=other BR_WT=t BR_CRATE=t BR_BRANCH=t BR_SHA=0 BR_AGENT=self-test BR_COMMAND="fake $name" \
|
||||
IGNEUM_BUILD_SLOTS_DIR="$t/locks" IGNEUM_BUILD_LOG_DIR="$t/log" BR_MEASURE="$measure" BR_CORES="${BR_CORES:-0}" BR_NICE="${BR_NICE:-0}" BR_DIR="$t/dir" BR_CMD="date +%s.%N > '$t/$name.start'; echo JOBS=\${CARGO_BUILD_JOBS:-none} > '$t/$name.jobs'; sleep $secs; date +%s.%N > '$t/$name.end'" \
|
||||
BR_LABEL="self-test $name" BR_TOOL=self-test BR_KIND=other BR_WT="wt-$name" BR_CRATE=t BR_BRANCH=t BR_SHA=0 BR_AGENT=self-test BR_COMMAND="fake $name" \
|
||||
bash "$me" >"$t/$name.out" 2>&1
|
||||
}
|
||||
fail() { echo "self-test-slots: FAIL: $*"; exit 1; }
|
||||
|
|
@ -193,33 +193,40 @@ if [ "${1:-}" = --self-test-slots ]; then
|
|||
# 2. one build alone: 90
|
||||
fake c 1
|
||||
[ "$(cat "$t/c.jobs")" = JOBS=90 ] || fail "a lone build got $(cat "$t/c.jobs") (want JOBS=90)"
|
||||
# 3. a measure blocks a build: the build starts only after the measure ended
|
||||
fake m 3 1 & sleep 0.5; fake d 1 & wait
|
||||
after "$t/d.start" "$t/m.end" || fail "a build started while a measurement held the box (build start $(cat "$t/d.start"), measure end $(cat "$t/m.end"))"
|
||||
# 3. a quiet measurement blocks an unbounded build (it starts only after the quiet ended) and lets a bounded suite run beside it
|
||||
fake m 3 1 & sleep 0.5; fake d 1 & BR_CORES=1 BR_NICE=10 fake s 1 & wait
|
||||
after "$t/d.start" "$t/m.end" || fail "an unbounded build started while a quiet measurement held the box (build start $(cat "$t/d.start"), quiet end $(cat "$t/m.end"))"
|
||||
python3 -c "import sys; sys.exit(0 if float(open('$t/s.start').read()) < float(open('$t/m.end').read()) else 1)" || fail "a bounded suite waited for the quiet measurement"
|
||||
[ "$(cat "$t/m.jobs")" = JOBS=none ] || fail "a measurement was given a job count"
|
||||
# 4. a build blocks a measure: the measure starts only after the build ended
|
||||
fake e 3 & sleep 0.5; fake n 1 1 & wait
|
||||
after "$t/n.start" "$t/e.end" || fail "a measurement started while a build ran (measure start $(cat "$t/n.start"), build end $(cat "$t/e.end"))"
|
||||
grep -q 'owner=self-test' "$t/m.out" "$t/log/builds.jsonl" 2>/dev/null || fail "the quiet holder line lacks its owner"
|
||||
# 4. a quiet measurement is REFUSED (exit 73) while a build holds a slot or a core is leased
|
||||
fake e 3 & sleep 0.5; fake n 1 1; rc=$?; wait
|
||||
[ "$rc" = 73 ] && [ ! -f "$t/n.start" ] || fail "a quiet measurement was not refused while a build ran (rc $rc)"
|
||||
( exec 9>>"$t/locks/core-7"; flock 9; sleep 2 ) & sleep 0.3; fake o 1 1; rc=$?; wait
|
||||
[ "$rc" = 73 ] || fail "a quiet measurement was not refused while a core was leased (rc $rc)"
|
||||
# 4b. a run keeps off leased cores: with core 1 leased, a 2-core bounded run on a 2-core box says so (the exclusion line)
|
||||
( exec 9>>"$t/locks/core-$(( $(nproc) - 1 ))"; flock 9; sleep 2 ) & sleep 0.3; BR_CORES=2 BR_NICE=10 fake p 1; wait
|
||||
grep -q 'are leased to a measurement; this run keeps to' "$t/p.out" || fail "a run beside a leased core did not exclude it: $(cat "$t/p.out" | tail -3)"
|
||||
# 5. a probing build leaves a busy slot's holder line intact
|
||||
fake f 3 & sleep 1.2; fake g 1 & sleep 0.3
|
||||
grep -q 'self-test f' "$t/locks/build-0" || fail "the holder line of the busy slot build-0 was lost when another build probed it: '$(cat "$t/locks/build-0")'"
|
||||
wait
|
||||
# 6. the log carries the job count and the measure flag
|
||||
grep -q '"jobs":45' "$t/log/builds.jsonl" && grep -q '"measure":true' "$t/log/builds.jsonl" || fail "builds.jsonl lacks jobs or measure fields"
|
||||
echo "self-test-slots: two concurrent builds 45 each, a lone build 90, a measure blocks a build, a build blocks a measure, a probe keeps the holder line, the log carries jobs and measure"; exit 0
|
||||
echo "self-test-slots: two concurrent builds 45 each, a lone build 90, a quiet blocks an unbounded build and not a bounded suite, a quiet is refused beside a slot or a lease, a run keeps off leased cores, a probe keeps the holder line, the log carries jobs and measure"; exit 0
|
||||
fi
|
||||
|
||||
# One run per worktree directory at a time (6 October 2026, 19:51:09 UK: two runs of one worktree started in the same second;
|
||||
# one found no crate directory while the other's checkout was replacing the tree, exit 2). The checkout and the run each take
|
||||
# the worktree's lock (append mode, held to exit) and wait up to 2 h for it instead of dying; the wait is said on stderr.
|
||||
wt_lock() { # <worktree name>
|
||||
local name="${1//\//_}" fd
|
||||
local name="${1//\//_}"
|
||||
[ -n "$name" ] || return 0
|
||||
mkdir -p "$IGNEUM_BUILD_SLOTS_DIR" 2>/dev/null || return 0
|
||||
exec {fd}>>"$IGNEUM_BUILD_SLOTS_DIR/wt-$name.lock" || return 0
|
||||
if ! flock -n "$fd"; then
|
||||
exec {WT_FD}>>"$IGNEUM_BUILD_SLOTS_DIR/wt-$name.lock" || return 0 # WT_FD is global: the keeper closes it (an orphaned sleep held it 20 s)
|
||||
if ! flock -n "$WT_FD"; then
|
||||
echo "build-remote: another run holds worktree $1 on this box, waiting for it (up to 2 h)" >&2
|
||||
flock -w 7200 "$fd" || { echo "build-remote: gave up waiting for worktree $1 after 2 h" >&2; exit 75; }
|
||||
flock -w 7200 "$WT_FD" || { echo "build-remote: gave up waiting for worktree $1 after 2 h" >&2; exit 75; }
|
||||
fi
|
||||
}
|
||||
if [ "${BR_MODE:-run}" = checkout ]; then
|
||||
|
|
@ -318,25 +325,30 @@ give_up() { # <what>
|
|||
rm -f "$waitfile"; exit 75
|
||||
}
|
||||
waitfile="$SLOTS_DIR/wait-$BR_PID"
|
||||
# The measure file is RETIRED (main, 7 October 2026, 15:07 UK: one global exclusive flock across unrelated measurements stalled
|
||||
# build-1 at load 120 with free slots). A measurement that pins cores takes a lease on those cores only (lease.sh, installed at
|
||||
# /srv/builds/_bin/lease), and this runner keeps its command off leased cores (see cores_str below). A WHOLE-BOX quiet measurement
|
||||
# (BR_MEASURE=1) is its own class: refused with exit 73 while any slot or core lease is held, capped at 20 minutes, holder line with
|
||||
# the owner in $SLOTS_DIR/quiet. An unbounded run (nice 0, the full core set) takes quiet shared and waits for it; a bounded run
|
||||
# (BR_CORES > 0: suites, benches, everything on box 2) never takes it.
|
||||
held_cores() { local f c out=""; for f in "$SLOTS_DIR"/core-*; do [ -e "$f" ] || continue; c=${f##*/core-}; exec {cfd}>>"$f"; if ! flock -n "$cfd"; then out="$out $c"; fi; exec {cfd}>&-; done; echo "${out# }"; }
|
||||
any_slot_held() { local k f; for ((k = 0; k < slots; k++)); do f="$SLOTS_DIR/build-$k"; [ -e "$f" ] || continue; exec {sfd}>>"$f"; if ! flock -n "$sfd"; then exec {sfd}>&-; return 0; fi; exec {sfd}>&-; done; return 1; }
|
||||
# append mode: opening a lock file must never truncate the holder line another run wrote into it
|
||||
exec {mfd}>>"$SLOTS_DIR/measure"
|
||||
exec {mfd}>>"$SLOTS_DIR/quiet"
|
||||
if [ "${BR_MEASURE:-0}" = 1 ]; then
|
||||
# a measurement: the measure file exclusively; every running build holds it shared, so this waits for them and blocks new ones
|
||||
if ! flock -n "$mfd"; then
|
||||
echo "build-remote: measure waits for the running build(s) (up to 2 h): $(for f in "$SLOTS_DIR"/build-*; do head -c 120 "$f" 2>/dev/null; done | tr '\n' ' ')" >&2
|
||||
holder_line 0 > "$waitfile"; trap 'rm -f "$waitfile"' EXIT
|
||||
flock -w 7200 "$mfd" || give_up "the measure hold"
|
||||
rm -f "$waitfile"; trap - EXIT
|
||||
fi
|
||||
if any_slot_held; then echo "build-remote: QUIET REFUSED: a build slot is held ($(for f in "$SLOTS_DIR"/build-*; do head -c 100 "$f" 2>/dev/null; done | tr '\n' ' ')); a whole-box measurement needs an idle box" >&2; BR_CLASS=quiet-refused jsonlog 73 0 0 "" "$(date +%s)" "" "" "" "" "" ""; exit 73; fi
|
||||
lc=$(held_cores); if [ -n "$lc" ]; then echo "build-remote: QUIET REFUSED: cores $(echo $lc | tr ' ' ',') are leased; a whole-box measurement needs an idle box" >&2; BR_CLASS=quiet-refused jsonlog 73 0 0 "" "$(date +%s)" "" "" "" "" "" ""; exit 73; fi
|
||||
if ! flock -n "$mfd"; then echo "build-remote: QUIET REFUSED: another quiet measurement holds the box: $(head -c 160 "$SLOTS_DIR/quiet")" >&2; BR_CLASS=quiet-refused jsonlog 73 0 0 "" "$(date +%s)" "" "" "" "" "" ""; exit 73; fi
|
||||
[ -n "${BR_AGENT:-}" ] && [ "$BR_AGENT" != unknown ] || { echo "build-remote: QUIET REFUSED: a whole-box measurement needs a named owner (IGNEUM_AGENT)" >&2; exit 73; }
|
||||
waited=$(( $(date +%s) - BR_T0 )); got=measure
|
||||
holder_line "$waited" > "$SLOTS_DIR/measure"
|
||||
echo "build-remote: holding measure on $BR_HOST (waited $waited s; builds are excluded until this run ends)" >&2
|
||||
BR_LABEL="quiet (cap 1200 s): $BR_LABEL; owner=$BR_AGENT"; holder_line "$waited" > "$SLOTS_DIR/quiet"
|
||||
echo "build-remote: holding the box QUIET on $BR_HOST (waited $waited s; owner $BR_AGENT; cap 20 min; unbounded builds wait, bounded suites run beside it on their bands)" >&2
|
||||
BR_CMD="timeout --signal TERM --kill-after 30 1200 bash -c $(printf '%q' "$BR_CMD")"
|
||||
else
|
||||
# a build: the measure file shared (a running measurement blocks us), then one exclusive slot
|
||||
if ! flock -s -n "$mfd"; then
|
||||
echo "build-remote: a measurement holds the box, waiting (up to 2 h): $(head -c 160 "$SLOTS_DIR/measure" 2>/dev/null)" >&2
|
||||
if [ "${BR_CORES:-0}" = 0 ] && ! flock -s -n "$mfd"; then
|
||||
echo "build-remote: a quiet measurement holds the box, waiting (up to 20 min): $(head -c 160 "$SLOTS_DIR/quiet" 2>/dev/null)" >&2
|
||||
holder_line 0 > "$waitfile"; trap 'rm -f "$waitfile"' EXIT
|
||||
flock -s -w 7200 "$mfd" || give_up "the measurement to end"
|
||||
flock -s -w 1500 "$mfd" || give_up "the quiet measurement to end"
|
||||
rm -f "$waitfile"; trap - EXIT
|
||||
fi
|
||||
# scheduling (main, 7 Oct 2026): a gate announces itself (gate-pending-<pid>) and takes the next free slot; a suite, bench or
|
||||
|
|
@ -390,13 +402,16 @@ fi
|
|||
# because a run from a worktree without last night's append-mode fix still opens a busy sibling's slot file with `>` on every
|
||||
# probe). A keeper re-writes the holder line whenever it finds the file empty, every BR_KEEP_S seconds, until release_slot.
|
||||
keeper_pid=""
|
||||
keep_line() { # <file> <waited>
|
||||
keep_line() { # <file> <waited>; the keeper also refreshes the file's mtime (a stopped holder's file goes stale) and reaps stopped
|
||||
# holders of any lease, quiet or slot after 5 minutes through lease.sh (/srv/builds/_bin/lease reap, one line each in reaped.log)
|
||||
local f="$1" w="$2"
|
||||
( while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; sleep "${BR_KEEP_S:-20}"; done ) &
|
||||
( exec {mfd}>&- {fd}>&- 2>/dev/null; [ -n "${WT_FD:-}" ] && exec {WT_FD}>&-; while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; touch "$f" 2>/dev/null
|
||||
[ -x /srv/builds/_bin/lease ] && IGNEUM_BUILD_SLOTS_DIR="$SLOTS_DIR" IGNEUM_BUILD_LOG_DIR="$LOG_DIR" /srv/builds/_bin/lease reap >/dev/null 2>&1
|
||||
sleep "${BR_KEEP_S:-20}"; done ) &
|
||||
keeper_pid=$!
|
||||
}
|
||||
if [ "$got" = measure ]; then keep_line "$SLOTS_DIR/measure" "$waited"; else keep_line "$SLOTS_DIR/build-$got" "$waited"; fi
|
||||
release_slot() { [ -n "$keeper_pid" ] && { kill "$keeper_pid" 2>/dev/null; wait "$keeper_pid" 2>/dev/null; keeper_pid=""; }; if [ "$got" = measure ]; then : > "$SLOTS_DIR/measure"; else : > "$SLOTS_DIR/build-$got"; fi; }
|
||||
if [ "$got" = measure ]; then keep_line "$SLOTS_DIR/quiet" "$waited"; else keep_line "$SLOTS_DIR/build-$got" "$waited"; fi
|
||||
release_slot() { [ -n "$keeper_pid" ] && { pkill -P "$keeper_pid" 2>/dev/null; kill "$keeper_pid" 2>/dev/null; wait "$keeper_pid" 2>/dev/null; keeper_pid=""; }; if [ "$got" = measure ]; then : > "$SLOTS_DIR/quiet"; else : > "$SLOTS_DIR/build-$got"; fi; }
|
||||
cd "$BR_DIR" || { BR_CLASS=no-dir jsonlog 2 "$got" "$waited" "" "$(date +%s)" "" "" "" "" "" ""; redlog 2 0 no-dir; release_slot; exit 2; }
|
||||
|
||||
# PRE-FLIGHT for a cargo command (the instant-death class, 6 October 2026): the manifest parses and every -p package exists,
|
||||
|
|
@ -457,6 +472,29 @@ if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then
|
|||
lo=$((ncpu - BR_CORES * (band + 1))); [ "$lo" -ge 0 ] || lo=$((ncpu - BR_CORES))
|
||||
cores_str="$lo-$((lo + BR_CORES - 1))"
|
||||
fi
|
||||
# leased cores (lease.sh: a pinned measurement's core-<n> flocks) are taken out of this run's set; a set that would be empty keeps
|
||||
# its cores (the measurement is told by its own lease line); the exclusion is said once
|
||||
if [ "$got" != measure ]; then
|
||||
leased=$(held_cores)
|
||||
if [ -n "$leased" ]; then
|
||||
kept=$(python3 -c '
|
||||
import sys
|
||||
def expand(s):
|
||||
out=set()
|
||||
for part in s.split(","):
|
||||
a,_,b=part.partition("-"); a=int(a); b=int(b) if b else a; out.update(range(a,b+1))
|
||||
return out
|
||||
mine=expand(sys.argv[1]); leased=set(int(x) for x in sys.argv[2].split()); keep=sorted(mine-leased)
|
||||
if not keep: print(sys.argv[1]); sys.exit()
|
||||
runs=[];
|
||||
for c in keep:
|
||||
if runs and runs[-1][1]==c-1: runs[-1][1]=c
|
||||
else: runs.append([c,c])
|
||||
print(",".join(f"{a}-{b}" if a!=b else f"{a}" for a,b in runs))' "$cores_str" "$leased")
|
||||
[ "$kept" != "$cores_str" ] && echo "build-remote: cores $(echo $leased | tr ' ' ',') are leased to a measurement; this run keeps to $kept" >&2
|
||||
cores_str="$kept"
|
||||
fi
|
||||
fi
|
||||
( [ "${BR_NICE:-0}" -gt 0 ] && renice -n "$BR_NICE" -p $BASHPID >/dev/null 2>&1; [ "$cores_str" != "0-$((ncpu - 1))" ] && taskset -cp "$cores_str" $BASHPID >/dev/null 2>&1; eval "$BR_CMD" ) > >(tee -a "$BR_RUN_LOG") 2> >(tee -a "$BR_RUN_LOG" >&2)
|
||||
rc=$?
|
||||
# the keeper stops BEFORE the bare `wait` (which flushes the two tees): a bare wait also waits for the keeper, and the keeper waits
|
||||
|
|
|
|||
114
infra/build-server/wave1-0320.sh
Executable file
114
infra/build-server/wave1-0320.sh
Executable file
|
|
@ -0,0 +1,114 @@
|
|||
#!/usr/bin/env bash
|
||||
# The 0.3.20 FIRST wave (the shipper, 7 October 2026): the publish carries a floor-moved override file whose digest differs from
|
||||
# the live one, and a node on the old file refuses a node on the new one as a peer, so the three testnet seeds and the hands move
|
||||
# AT the publish minute, in parallel with the fleet's wave. Dry run by default (every line printed, no box touched); --go executes.
|
||||
#
|
||||
# infra/build-server/wave1-0320.sh seeds --override <file> --igneumd <seed-class igneumd> --sha256 <hex> [--miner <seed-class igneum-miner>] \
|
||||
# --digest <hex> [--commit c4459193] [--go]
|
||||
# the three seeds in PARALLEL (seed1/2/3.testnet, root over the ops key, infra/seed-nodes/seeds-testnet.tsv): the binary put
|
||||
# as /opt/igneum/bin/igneumd.new with its sha256 asserted on the box, the override written to /etc/igneum/override-params.json,
|
||||
# EXTRA_ARGS in /etc/igneum/seed.env set to --override-params-file=<that>, the unit igneumd stopped, the binary swapped (the old
|
||||
# one kept as igneumd.prev), the unit started on its kept datadir, then the read-back: commit string in the installed binary,
|
||||
# the "Consensus params digest" line of the new run against --digest, eth_syncing over the loopback EVM RPC, peers; one line
|
||||
# per seed, logs in the scratch directory
|
||||
# infra/build-server/wave1-0320.sh hands --node <fork worktree on the Mac> --override-json '<object>' --digest <hex> [--go]
|
||||
# the hands through infra/build-server/hands/move-hand.sh: binary (build + install + override), restart observer-node, restart
|
||||
# node1, each read back by that tool (first executing line, commit string, digest, powEngine)
|
||||
# infra/build-server/wave1-0320.sh lift-rpc-filter [--go]
|
||||
# on seed1 only, AFTER the three read-backs: BLOCKED_UNTIL_FIXED_NODE in /opt/igneum/bin/rpc-filter.py becomes the empty set
|
||||
# (the N7 class is fixed from 8097d600; backup kept as rpc-filter.py.bak-<stamp>), python3 -m py_compile, the unit
|
||||
# igneum-rpc-filter restarted, read back: unit active and eth_blockNumber answered through https://rpc.testnet.igneum.network
|
||||
set -euo pipefail
|
||||
HERE=$(cd "$(dirname "$0")" && pwd); ROOT=$(cd "$HERE/../.." && pwd)
|
||||
SEEDS_TSV="$ROOT/infra/seed-nodes/seeds-testnet.tsv"; KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
|
||||
S="${WAVE_LOG_DIR:-/private/tmp/claude-501/-Users-joshm/cd75457f-4858-4f86-9634-7481ee056b7b/scratchpad/wave1-0320}"; mkdir -p "$S"
|
||||
SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15)
|
||||
now() { TZ=Europe/London date '+%H:%M:%S %Z'; }
|
||||
say() { echo "$(now) wave1: $*" >&2; }
|
||||
die() { say "ERROR: $*"; exit 1; }
|
||||
seed_ip() { awk -F'\t' -v n="$1" '$1==n {print $4}' "$SEEDS_TSV"; }
|
||||
mode="${1:-}"; [ -n "$mode" ] || { sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }; shift
|
||||
GO=0; OVERRIDE=""; BIN=""; SHA=""; MINER=""; DIGEST=""; COMMIT="c4459193"; NODE=""; OVJSON=""
|
||||
while [ $# -gt 0 ]; do case "$1" in
|
||||
--go) GO=1; shift ;; --override) OVERRIDE="$2"; shift 2 ;; --igneumd) BIN="$2"; shift 2 ;; --sha256) SHA="$2"; shift 2 ;;
|
||||
--miner) MINER="$2"; shift 2 ;; --digest) DIGEST="$2"; shift 2 ;; --commit) COMMIT="$2"; shift 2 ;; --node) NODE="$2"; shift 2 ;;
|
||||
--override-json) OVJSON="$2"; shift 2 ;; *) die "unknown option $1" ;; esac; done
|
||||
|
||||
one_seed() { # <name>: runs in the background; prints one RESULT line at the end
|
||||
local name="$1" ip log t0 line
|
||||
ip=$(seed_ip "$name"); [ -n "$ip" ] || { echo "RESULT $name: no ip in $SEEDS_TSV"; return 1; }
|
||||
log="$S/$name.log"; : > "$log"; t0=$(date +%s)
|
||||
{
|
||||
if [ "$GO" = 0 ]; then
|
||||
echo "DRY: scp $BIN root@$ip:/opt/igneum/bin/igneumd.new; sha256 asserted = $SHA"
|
||||
[ -n "$MINER" ] && echo "DRY: scp $MINER root@$ip:/opt/igneum/bin/igneum-miner.new (swapped with the node)"
|
||||
echo "DRY: scp $OVERRIDE root@$ip:/etc/igneum/override-params.json; seed.env EXTRA_ARGS=--override-params-file=/etc/igneum/override-params.json"
|
||||
echo "DRY: systemctl stop igneumd; mv igneumd igneumd.prev; mv igneumd.new igneumd; systemctl start igneumd"
|
||||
echo "DRY: read back: grep -c $COMMIT in the binary; journal 'Consensus params digest' == $DIGEST; eth_syncing on 127.0.0.1:26890; peers"
|
||||
echo "DRY RUN, nothing touched; box $ip answers as $("${SSH[@]}" "root@$ip" 'hostname; systemctl is-active igneumd; sha256sum /opt/igneum/bin/igneumd | cut -c1-12' 2>/dev/null | tr '\n' ' ')"
|
||||
else
|
||||
scp -q -i "$KEY" -o BatchMode=yes "$BIN" "root@$ip:/opt/igneum/bin/igneumd.new"
|
||||
[ -n "$MINER" ] && scp -q -i "$KEY" -o BatchMode=yes "$MINER" "root@$ip:/opt/igneum/bin/igneum-miner.new"
|
||||
scp -q -i "$KEY" -o BatchMode=yes "$OVERRIDE" "root@$ip:/etc/igneum/override-params.json"
|
||||
"${SSH[@]}" "root@$ip" bash -s -- "$SHA" "$COMMIT" "$DIGEST" "$([ -n "$MINER" ] && echo 1 || echo 0)" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
SHA="$1"; COMMIT="$2"; DIGEST="$3"; MINER="$4"; cd /opt/igneum/bin
|
||||
got=$(sha256sum igneumd.new | cut -d' ' -f1); [ "$got" = "$SHA" ] || { echo "sha256 MISMATCH on the box: $got"; exit 1; }
|
||||
python3 -c 'import json,sys; json.load(open("/etc/igneum/override-params.json"))' || { echo "override file does not parse"; exit 1; }
|
||||
chmod 755 igneumd.new; [ "$MINER" = 1 ] && chmod 755 igneum-miner.new
|
||||
grep -qE '^EXTRA_ARGS=' /etc/igneum/seed.env && sed -i -E 's#^EXTRA_ARGS=.*#EXTRA_ARGS="--override-params-file=/etc/igneum/override-params.json"#' /etc/igneum/seed.env || echo 'EXTRA_ARGS="--override-params-file=/etc/igneum/override-params.json"' >> /etc/igneum/seed.env
|
||||
t0=$(date +%s); systemctl stop igneumd
|
||||
mv -f igneumd igneumd.prev; mv -f igneumd.new igneumd; [ "$MINER" = 1 ] && { mv -f igneum-miner igneum-miner.prev 2>/dev/null || true; mv -f igneum-miner.new igneum-miner; }
|
||||
systemctl start igneumd; sleep 6
|
||||
down=$(( $(date +%s) - t0 ))
|
||||
commits=$(grep -a -c "$COMMIT" igneumd || true)
|
||||
first=$(journalctl -u igneumd --since "-40 s" --no-pager 2>/dev/null | grep -vE "Started|Stopped|Stopping|Deactivated|Consumed" | head -1 | cut -c1-120)
|
||||
dig=$(journalctl -u igneumd --since "-40 s" --no-pager 2>/dev/null | grep -oE "Consensus params digest: [0-9a-f]+" | tail -1 | awk '{print $4}')
|
||||
syncing=$(curl -s -m 5 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_syncing","params":[]}' http://127.0.0.1:26890 | cut -c1-80)
|
||||
peers=$(curl -s -m 5 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"net_peerCount","params":[]}' http://127.0.0.1:26890 | grep -oE '"result":"[^"]*"' | cut -d'"' -f4)
|
||||
dmatch=no; [ -n "$dig" ] && [ "$dig" = "$DIGEST" ] && dmatch=MATCH; [ -n "$dig" ] && [ "$dig" != "$DIGEST" ] && dmatch="MISMATCH($dig)"
|
||||
echo "unit $(systemctl is-active igneumd) down ${down}s; commit strings $commits; digest $dmatch; eth_syncing $syncing; peers $peers; first: $first"
|
||||
REMOTE
|
||||
fi
|
||||
} >> "$log" 2>&1; rc=$?
|
||||
line=$(tail -1 "$log" | cut -c1-300)
|
||||
echo "RESULT $name ($ip) rc=$rc after $(( $(date +%s) - t0 )) s: $line"
|
||||
}
|
||||
|
||||
case "$mode" in
|
||||
seeds)
|
||||
[ -n "$OVERRIDE" ] && [ -n "$BIN" ] && [ -n "$SHA" ] && [ -n "$DIGEST" ] || die "seeds needs --override --igneumd --sha256 --digest"
|
||||
[ -f "$OVERRIDE" ] && [ -f "$BIN" ] || die "override or binary file missing"
|
||||
local_sha=$(shasum -a 256 "$BIN" | cut -d' ' -f1); [ "$local_sha" = "$SHA" ] || die "the binary's sha256 is $local_sha, not $SHA"
|
||||
python3 -c 'import json,sys; json.load(open(sys.argv[1]))' "$OVERRIDE" || die "override file does not parse"
|
||||
say "seeds: $( [ "$GO" = 1 ] && echo GO || echo DRY RUN ); igneumd $SHA; override $OVERRIDE ($(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))))' "$OVERRIDE") fields); digest $DIGEST; commit $COMMIT"
|
||||
for n in seed1.testnet seed2.testnet seed3.testnet; do one_seed "$n" & done; wait
|
||||
say "seeds done; logs in $S" ;;
|
||||
hands)
|
||||
[ -n "$NODE" ] && [ -n "$OVJSON" ] && [ -n "$DIGEST" ] || die "hands needs --node --override-json --digest"
|
||||
g=""; [ "$GO" = 1 ] && g="--go"
|
||||
say "hands: $( [ "$GO" = 1 ] && echo GO || echo DRY RUN ) through move-hand.sh"
|
||||
"$HERE/hands/move-hand.sh" binary --node "$NODE" --override-json "$OVJSON" $g
|
||||
"$HERE/hands/move-hand.sh" restart observer-node --digest "$DIGEST" $g
|
||||
"$HERE/hands/move-hand.sh" restart node1 --digest "$DIGEST" $g ;;
|
||||
lift-rpc-filter)
|
||||
ip=$(seed_ip seed1.testnet)
|
||||
if [ "$GO" = 0 ]; then
|
||||
say "DRY RUN: on root@$ip: back up /opt/igneum/bin/rpc-filter.py, set BLOCKED_UNTIL_FIXED_NODE = set(), py_compile, systemctl restart igneum-rpc-filter, read back"
|
||||
"${SSH[@]}" "root@$ip" 'echo "current: $(grep -c "^BLOCKED_UNTIL_FIXED_NODE = {" /opt/igneum/bin/rpc-filter.py) block set(s), unit $(systemctl is-active igneum-rpc-filter.service)"'; exit 0; fi
|
||||
"${SSH[@]}" "root@$ip" bash -s <<'REMOTE'
|
||||
set -euo pipefail
|
||||
f=/opt/igneum/bin/rpc-filter.py; cp -a "$f" "$f.bak-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
python3 - <<'PY'
|
||||
import re
|
||||
p='/opt/igneum/bin/rpc-filter.py'; s=open(p).read()
|
||||
new, n = re.subn(r'BLOCKED_UNTIL_FIXED_NODE = \{.*?\n\}', 'BLOCKED_UNTIL_FIXED_NODE = set() # lifted 7 Oct 2026: every seed runs the fixed node (N7 class fixed from 8097d600)', s, count=1, flags=re.S)
|
||||
assert n == 1, "block set not found"
|
||||
open(p,'w').write(new)
|
||||
PY
|
||||
python3 -m py_compile "$f"; systemctl restart igneum-rpc-filter.service; sleep 2
|
||||
echo "unit $(systemctl is-active igneum-rpc-filter.service); blocked set now: $(grep -E '^BLOCKED_UNTIL_FIXED_NODE' "$f" | cut -c1-60)"
|
||||
REMOTE
|
||||
say "public read-back: $(curl -s -m 8 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' https://rpc.testnet.igneum.network | cut -c1-120)" ;;
|
||||
*) die "unknown mode $mode" ;;
|
||||
esac
|
||||
22
tools/ci/box-locks-check.sh
Executable file
22
tools/ci/box-locks-check.sh
Executable file
|
|
@ -0,0 +1,22 @@
|
|||
#!/usr/bin/env bash
|
||||
# The box-side lock self-tests, run on igneum-build-1 over ssh (flock, taskset and GNU timeout live there, not on the Mac; the Mac
|
||||
# gate carried no slot self-test since the worktree lock landed on 6 October 2026 and the slot test rotted unseen). Both scripts
|
||||
# are copied as they are in this tree (never the box's installed copies) into a scratch path and run against scratch lock
|
||||
# directories: infra/build-server/lease.sh --self-test (per-core leases, the quiet class, the reaper) and
|
||||
# infra/build-server/remote-run.sh --self-test-slots (slots, jobs, quiet beside builds and suites, leased cores excluded).
|
||||
#
|
||||
# tools/ci/box-locks-check.sh # exit 1 when either self-test fails or the box cannot be reached
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/../.."
|
||||
. infra/build-server/lib.sh
|
||||
bs_host 1
|
||||
stamp="ci-$$-$(date +%s)"
|
||||
scp -q "${BS_SSH_OPTS[@]}" infra/build-server/lease.sh "$BS_HOST:/tmp/lease-$stamp.sh"
|
||||
scp -q "${BS_SSH_OPTS[@]}" infra/build-server/remote-run.sh "$BS_HOST:/tmp/rr-$stamp.sh"
|
||||
rc=0
|
||||
# the two self-tests run in parallel in one ssh session (each against its own scratch lock directory); their last lines are printed
|
||||
bs_ssh "set -o pipefail; (bash /tmp/lease-$stamp.sh --self-test 2>&1 | grep -E '^lease self-test'; echo lease=\${PIPESTATUS[0]} >> /tmp/locks-$stamp.rc) & (IGNEUM_REMOTE_RUN_UNDER_TEST=/tmp/rr-$stamp.sh bash /tmp/rr-$stamp.sh --self-test-slots 2>&1 | grep -E '^self-test-slots'; echo slots=\${PIPESTATUS[0]} >> /tmp/locks-$stamp.rc) & wait; cat /tmp/locks-$stamp.rc; rm -f /tmp/lease-$stamp.sh /tmp/rr-$stamp.sh /tmp/locks-$stamp.rc" | tee /tmp/box-locks-$$.out || rc=1
|
||||
grep -q '^lease=0$' /tmp/box-locks-$$.out && grep -q '^slots=0$' /tmp/box-locks-$$.out || rc=1
|
||||
rm -f /tmp/box-locks-$$.out
|
||||
[ "$rc" = 0 ] && echo "box-locks: the lease tool and the slot runner pass their self-tests on $BS_HOST"
|
||||
exit $rc
|
||||
|
|
@ -88,6 +88,8 @@ tree_checks() {
|
|||
run "long-running tools keep their body in one parsed block (the edited-while-running class)" bash -c 'bash tools/ci/whole-body-check.sh --self-test && bash tools/ci/whole-body-check.sh'
|
||||
run "build-remote without a priority flag bounds suites and benches (nice 10, 32 cores); a gate runs unbounded" bash tools/ci/build-kind-default-check.sh
|
||||
run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh
|
||||
run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh)" bash tools/ci/box-locks-check.sh
|
||||
run "the simulators job runs on master and release-* pushes and pull requests into them only" bash tools/ci/sims-branch-check.sh
|
||||
run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh'
|
||||
run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh'
|
||||
run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test
|
||||
|
|
|
|||
35
tools/ci/sims-branch-check.sh
Executable file
35
tools/ci/sims-branch-check.sh
Executable file
|
|
@ -0,0 +1,35 @@
|
|||
#!/usr/bin/env bash
|
||||
# The simulators job of .github/workflows/ci.yml runs on master and release-* pushes, and on pull requests into them, ONLY (main,
|
||||
# 7 October 2026: with build-2 in the runner pool the queue still read 22 because every code push cost two box jobs; a
|
||||
# feature-branch code push now runs the igneum-pow tests alone). This check reads the job's `if:` line and evaluates the rule
|
||||
# with the expression's own shape for the known cases, so a change that lets a feature-branch push run the simulators fails it.
|
||||
#
|
||||
# tools/ci/sims-branch-check.sh # exit 1 with the case that resolved wrongly (the check IS its self-test)
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/../.."
|
||||
line=$(awk '/^ sims:/{f=1} f && /^ if:/{print; exit}' .github/workflows/ci.yml)
|
||||
[ -n "$line" ] || { echo "sims-branch: no if: line under the sims job" >&2; exit 1; }
|
||||
for want in "github.event_name == 'push'" "github.ref == 'refs/heads/master'" "startsWith(github.ref, 'refs/heads/release-')" \
|
||||
"github.event_name == 'pull_request'" "github.base_ref == 'master'" "startsWith(github.base_ref, 'release-')" "needs.changes.outputs.code == 'true'"; do
|
||||
case "$line" in *"$want"*) ;; *) echo "sims-branch: the sims if: line lacks \"$want\": $line" >&2; exit 1 ;; esac
|
||||
done
|
||||
# the rule, as the expression reads: code and ((push and (master or release-*)) or (pull_request and base (master or release-*)))
|
||||
rule() { # <code> <event> <ref> <base_ref> -> run|skip
|
||||
local code="$1" ev="$2" ref="$3" base="$4"
|
||||
[ "$code" = true ] || { echo skip; return; }
|
||||
if [ "$ev" = push ] && { [ "$ref" = refs/heads/master ] || [[ "$ref" == refs/heads/release-* ]]; }; then echo run; return; fi
|
||||
if [ "$ev" = pull_request ] && { [ "$base" = master ] || [[ "$base" == release-* ]]; }; then echo run; return; fi
|
||||
echo skip
|
||||
}
|
||||
fail=0
|
||||
expect() { local want="$1"; shift; local got; got=$(rule "$@"); if [ "$got" = "$want" ]; then echo "sims-branch: [$*] -> $got"; else echo "sims-branch: [$*] -> $got, expected $want" >&2; fail=1; fi; }
|
||||
expect run true push refs/heads/master ""
|
||||
expect run true push refs/heads/release-0.3.20 ""
|
||||
expect skip true push refs/heads/build-server ""
|
||||
expect skip true push refs/heads/feature/x ""
|
||||
expect run true pull_request refs/pull/12/merge master
|
||||
expect run true pull_request refs/pull/12/merge release-0.3.21
|
||||
expect skip true pull_request refs/pull/12/merge build-server
|
||||
expect skip false push refs/heads/master ""
|
||||
[ "$fail" = 0 ] && echo "sims-branch: the simulators run on master and release-* pushes and on pull requests into them only"
|
||||
exit $fail
|
||||
Loading…
Reference in a new issue