6836115d65
infra/build-server/capacity: igneum-capacity.service (user build, Nice 19, SCHED_IDLE, CPUQuota leaving 8 threads free) runs run.sh, a controller over a priority queue of jobs that pauses (SIGSTOP) the instant a build slot or the measure hold is taken (5 s poll of /srv/builds/_locks) and resumes after. Jobs, each with a dry-run and a 10-min smoke: pow fuzz (continuous, seed base advances), sync-request fuzz against a throwaway pruned node on loopback (the Horizon 28-unwrap gate, igneum-p2p-probe sync-fuzz), GHOSTDAG and finality sweeps seeds 1 to 1,000, model.py sweeps cached, clippy+audit per recent branch. Summaries to /srv/workers/capacity.json; the worker dashboard gains a Background lane (collect.mjs doc.background, page renderBackground). Night battery stops and restarts the layer. docs/plans/build-server.md section 8. igneum-pow fuzz tests and ghostdag_sim.py / attacks.py gain a seed-base knob for the continuous sweeps. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
84 lines
5 KiB
Bash
Executable file
84 lines
5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# The capacity controller on igneum-build-1 (infra/build-server/capacity). Runs the job queue in priority order, one job
|
|
# at a time, each for a bounded slice; a 5 s poll of /srv/builds/_locks SIGSTOPs the running job the instant any build
|
|
# slot or the measure hold is taken and SIGCONTs it when they clear. The layer never takes a build slot and never writes
|
|
# under /srv/builds/<worktree>. Started as user build by igneum-capacity.service (Nice 19, chrt -i 0 wrapper, CPUQuota
|
|
# leaving 8 threads free). Killed by the night battery's start and restarted after (the service's ExecStartPre/StopPost).
|
|
#
|
|
# run.sh the controller loop (systemd runs this)
|
|
# run.sh --once one pass through the weighted sequence, then exit (for a manual check)
|
|
# run.sh --dry-run call every job's --dry-run in priority order and exit
|
|
#
|
|
# The weighted sequence (CAP_SEQUENCE) gives the earlier, higher-priority jobs more turns. Default:
|
|
# pow-fuzz pow-fuzz pow-fuzz sync-fuzz sync-fuzz sim-sweeps model-sweeps clippy-audit
|
|
# Each turn runs one job for CAP_SLICE_S (default 1800 s) through run_slice, which backgrounds the job in its own
|
|
# process group and pauses/resumes it with the lock poll.
|
|
set -uo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; export CAP_JOB=run
|
|
. "$HERE/../capacity/lib.sh" 2>/dev/null || . "$HERE/lib.sh"
|
|
|
|
SLICE_S="${CAP_SLICE_S:-1800}"
|
|
POLL_S="${CAP_POLL_S:-5}"
|
|
SEQUENCE="${CAP_SEQUENCE:-pow-fuzz pow-fuzz pow-fuzz sync-fuzz sync-fuzz sim-sweeps model-sweeps clippy-audit}"
|
|
JOBS_DIR="$HERE/jobs"
|
|
|
|
"$NODE_BIN" "$SUMMARY" init 2>/dev/null || true
|
|
|
|
if [ "${1:-}" = --dry-run ]; then
|
|
for j in pow-fuzz sync-fuzz sim-sweeps model-sweeps clippy-audit; do
|
|
cap_say "dry-run $j"; bash "$JOBS_DIR/$j.sh" --dry-run || cap_say "$j dry-run returned $?"
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
# run_slice <job>: background the job in its own process group, poll the locks, STOP while a build or measure holds,
|
|
# CONT when clear, enforce the slice as a wall clock, reap at the end. The job writes its own summary; the controller
|
|
# updates the top-level state and current_job.
|
|
run_slice() {
|
|
local job="$1" log="$CAP_LOG/$job.slice.log"; mkdir -p "$CAP_LOG"
|
|
cap_say "slice start: $job (${SLICE_S}s)"
|
|
# own process group via setsid so STOP/CONT reach the whole cargo/python tree
|
|
setsid bash "$JOBS_DIR/$job.sh" --slice-s "$SLICE_S" > "$log" 2>&1 &
|
|
local pid=$! pgid; pgid=$(ps -o pgid= -p "$pid" 2>/dev/null | tr -d ' '); [ -n "$pgid" ] || pgid="$pid"
|
|
local paused=0 end=$(( $(date +%s) + SLICE_S + 120 )) # a 2 min grace over the slice for the job's own teardown
|
|
while kill -0 "$pid" 2>/dev/null; do
|
|
if cap_build_active; then
|
|
if [ "$paused" = 0 ]; then kill -STOP -"$pgid" 2>/dev/null && paused=1; local why; why=$(cap_hold_reason); cap_say "pause $job ($why)"; printf '{"state":"paused","current_job":%s,"paused_reason":%s,"slice_s":%s}' "$(cap_json_str "$job")" "$(cap_json_str "$why")" "$SLICE_S" | cap_controller_summary; fi
|
|
else
|
|
if [ "$paused" = 1 ]; then kill -CONT -"$pgid" 2>/dev/null; paused=0; cap_say "resume $job"; printf '{"state":"running","current_job":%s,"paused_reason":null,"slice_s":%s}' "$(cap_json_str "$job")" "$SLICE_S" | cap_controller_summary; fi
|
|
fi
|
|
# a safety stop so a wedged job cannot hold the turn forever; a paused job's clock does not advance it past end+grace
|
|
if [ "$paused" = 0 ] && [ "$(date +%s)" -ge "$end" ]; then cap_say "slice over time, stopping $job"; kill -INT -"$pgid" 2>/dev/null; sleep 3; kill -KILL -"$pgid" 2>/dev/null; break; fi
|
|
sleep "$POLL_S"
|
|
done
|
|
[ "$paused" = 1 ] && kill -CONT -"$pgid" 2>/dev/null
|
|
wait "$pid" 2>/dev/null; local rc=$?
|
|
cap_say "slice end: $job rc $rc"
|
|
}
|
|
|
|
once() {
|
|
for job in $SEQUENCE; do
|
|
[ -f "$JOBS_DIR/$job.sh" ] || { cap_say "no job $job"; continue; }
|
|
# wait out a running build before starting a slice (do not even launch during a build)
|
|
while cap_build_active; do
|
|
printf '{"state":"waiting","current_job":null,"paused_reason":%s,"slice_s":%s}' "$(cap_json_str "$(cap_hold_reason)")" "$SLICE_S" | cap_controller_summary
|
|
sleep "$POLL_S"
|
|
done
|
|
printf '{"state":"running","current_job":%s,"paused_reason":null,"slice_s":%s,"host":%s}' "$(cap_json_str "$job")" "$SLICE_S" "$(cap_json_str "$(hostname)")" | cap_controller_summary
|
|
run_slice "$job"
|
|
done
|
|
}
|
|
|
|
cap_sync_checkout >/dev/null 2>&1 || cap_say "initial checkout had trouble (jobs will retry)"
|
|
if [ "${1:-}" = --once ]; then once; printf '{"state":"idle","current_job":null,"paused_reason":null}' | cap_controller_summary; exit 0; fi
|
|
|
|
trap 'cap_say "controller stopping"; printf "{\"state\":\"stopped\",\"current_job\":null}" | cap_controller_summary; exit 0' INT TERM
|
|
cap_say "controller start (sequence: $SEQUENCE; slice ${SLICE_S}s; poll ${POLL_S}s)"
|
|
cycle=0
|
|
while true; do
|
|
cycle=$((cycle + 1))
|
|
# refresh the checkout at the top of each cycle (cheap when nothing moved); yields if a build is active
|
|
while cap_build_active; do sleep "$POLL_S"; done
|
|
cap_sync_checkout >/dev/null 2>&1 || true
|
|
once
|
|
done
|