Merge build-server e48a512c into master on the box mirror (GitHub suspended; the box gate stands in for CI)
# Conflicts: # infra/build-server/remote-run.sh
This commit is contained in:
igneum-labs
commit
2ebcdd3241
5 changed files with 261 additions and 21 deletions
64
infra/build-server/cloud-sweep.sh
Executable file
64
infra/build-server/cloud-sweep.sh
Executable file
|
|
@ -0,0 +1,64 @@
|
|||
#!/usr/bin/env bash
|
||||
# Rented SWEEP WORKERS on Hetzner Cloud (main's order, 7 October 2026): CCX63 (48 dedicated cores) or CCX53 (32) by the hour,
|
||||
# provisioned like a build box minus the runner (provision.sh ROLE=sweep: build user with the Mac's key, toolchain, sccache,
|
||||
# the bare mirrors, the lease tool with its pool, ufw), for the adversarial sweeps that queue on build-1 and build-2. Destroyed
|
||||
# after. Within USD 1,500 a day, a cost line at every USD 100 (cost). Token at ~/.config/igneum/hetzner-token (never printed),
|
||||
# key igneum-ops (the Mac's ~/.ssh/igneum_ed25519), hcloud CLI.
|
||||
#
|
||||
# infra/build-server/cloud-sweep.sh up <n> [--type ccx63|ccx53] [--location fsn1|nbg1|hel1] create igneum-sweep-<n>, provision, host line
|
||||
# infra/build-server/cloud-sweep.sh list the workers with uptime and cost so far
|
||||
# infra/build-server/cloud-sweep.sh cost one line: hours, EUR, USD so far (all workers)
|
||||
# infra/build-server/cloud-sweep.sh down <n> [--force] refuse while a lease is held (lease status), then delete
|
||||
#
|
||||
# Host files: ~/.config/igneum/sweep-<n> ("build@<ip>", what a lane's ssh line reads). The pool on a worker: cores 8 to ncpu-1
|
||||
# (40 on a CCX63, 24 on a CCX53); the lanes use the same `lease pool <threads> -- cmd` line as on the boxes.
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
export HCLOUD_TOKEN; HCLOUD_TOKEN="$(tr -d '[:space:]' < "$HOME/.config/igneum/hetzner-token")"
|
||||
HC="${HCLOUD:-/opt/homebrew/bin/hcloud}"; KEY="$HOME/.ssh/igneum_ed25519"
|
||||
EUR_USD="${EUR_USD:-1.09}"
|
||||
die() { echo "cloud-sweep: $*" >&2; exit 1; }
|
||||
hourly() { "$HC" server-type describe "$1" -o json | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["prices"][0]["price_hourly"]["gross"])'; }
|
||||
case "${1:-}" in
|
||||
up)
|
||||
n="${2:-}"; [ -n "$n" ] || die "up <n>"; shift 2; TYPE=ccx63; LOC=fsn1
|
||||
while [ $# -gt 0 ]; do case "$1" in --type) TYPE="$2"; shift 2 ;; --location) LOC="$2"; shift 2 ;; *) die "unknown $1" ;; esac; done
|
||||
NAME="igneum-sweep-$n"
|
||||
if ! "$HC" server describe "$NAME" >/dev/null 2>&1; then
|
||||
"$HC" server create --name "$NAME" --type "$TYPE" --image ubuntu-24.04 --location "$LOC" --ssh-key igneum-ops --label igneum=sweep --label role=sweep >/dev/null
|
||||
echo "created $NAME ($TYPE, $LOC) at $(TZ=UTC date +%H:%M:%SZ)"
|
||||
fi
|
||||
IP=$("$HC" server ip "$NAME")
|
||||
ssh-keygen -R "$IP" >/dev/null 2>&1 || true
|
||||
for i in $(seq 1 60); do ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 "root@$IP" true 2>/dev/null && break; sleep 5; done
|
||||
ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 "root@$IP" true || die "$NAME at $IP: ssh never answered"
|
||||
# cloud-init may still hold apt for a minute after first boot
|
||||
ssh -i "$KEY" -o BatchMode=yes "root@$IP" 'cloud-init status --wait >/dev/null 2>&1 || true'
|
||||
ssh -i "$KEY" -o BatchMode=yes -o ServerAliveInterval=15 "root@$IP" "ROLE=sweep SLOTS=1 WORKTREES='' BOX_HOSTNAME=$NAME P2P_PORTS='' bash -s" < "$HERE/provision.sh" 2>&1 | grep -E "^(changed|ok| |summary|die|error)" | tail -12
|
||||
# the lease tool straight from this checkout (the worker's mirror has no master until a lane pushes one)
|
||||
ssh -i "$KEY" -o BatchMode=yes "root@$IP" 'install -d -m 755 -o build -g build /srv/builds/_bin /srv/builds/_locks; cat > /srv/builds/_bin/lease.new && chmod 755 /srv/builds/_bin/lease.new && chown build:build /srv/builds/_bin/lease.new && mv /srv/builds/_bin/lease.new /srv/builds/_bin/lease' < "$HERE/lease.sh"
|
||||
printf 'build@%s\n' "$IP" > "$HOME/.config/igneum/sweep-$n"
|
||||
echo "SWEEP WORKER $NAME build@$IP ($TYPE, $LOC, $(ssh -i "$KEY" -o BatchMode=yes "build@$IP" nproc) threads, pool cores 8 to $(( $(ssh -i "$KEY" -o BatchMode=yes "build@$IP" nproc) - 1 ))), lease tool $(ssh -i "$KEY" -o BatchMode=yes "build@$IP" 'sha256sum /srv/builds/_bin/lease | cut -c1-8'), $(hourly "$TYPE") EUR/h; host file ~/.config/igneum/sweep-$n"
|
||||
;;
|
||||
list|cost)
|
||||
"$HC" server list -l igneum=sweep -o json | python3 -c '
|
||||
import sys,json,datetime,subprocess,os
|
||||
d=json.load(sys.stdin); now=datetime.datetime.now(datetime.timezone.utc); tot_h=0.0; tot_eur=0.0; rate=float(os.environ.get("EUR_USD","1.09")); mode=sys.argv[1]
|
||||
for s in d:
|
||||
t=s["server_type"]["name"]; created=datetime.datetime.fromisoformat(s["created"].replace("Z","+00:00")); h=(now-created).total_seconds()/3600
|
||||
pr=float([p for p in s["server_type"]["prices"] if p["location"]==s["datacenter"]["location"]["name"]][0]["price_hourly"]["gross"])
|
||||
eur=h*pr; tot_h+=h; tot_eur+=eur
|
||||
if mode=="list": print(f"{s[\"name\"]} {s[\"public_net\"][\"ipv4\"][\"ip\"]} {t} {s[\"datacenter\"][\"location\"][\"name\"]} {s[\"status\"]} up {h:.2f} h, EUR {eur:.2f} so far")
|
||||
print(f"COST sweep workers: {len(d)} running, {tot_h:.2f} worker-hours, EUR {tot_eur:.2f} = USD {tot_eur*rate:.2f} so far (gross, hourly, EUR/USD {rate})")
|
||||
' "$1"
|
||||
;;
|
||||
down)
|
||||
n="${2:-}"; [ -n "$n" ] || die "down <n>"; NAME="igneum-sweep-$n"; IP=$("$HC" server ip "$NAME" 2>/dev/null || true)
|
||||
if [ -n "$IP" ] && [ "${3:-}" != --force ]; then
|
||||
held=$(ssh -i "$KEY" -o BatchMode=yes -o ConnectTimeout=8 "build@$IP" '/srv/builds/_bin/lease status 2>/dev/null | sed -n "/^leases:/,/^quiet:/p" | grep -c "^ pid"' 2>/dev/null || echo 0)
|
||||
[ "$held" = 0 ] || die "$NAME holds $held lease(s) (lease status on build@$IP); --force to delete anyway"
|
||||
fi
|
||||
"$HC" server delete "$NAME" >/dev/null && echo "deleted $NAME at $(TZ=UTC date +%H:%M:%SZ)"; rm -f "$HOME/.config/igneum/sweep-$n"
|
||||
;;
|
||||
*) sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
|
||||
esac
|
||||
|
|
@ -12,6 +12,23 @@
|
|||
# A WHOLE-BOX quiet measurement: its own class, refused (exit 73) while any build slot or any core lease is held, capped at
|
||||
# 20 minutes (timeout; --cap-s at most 1200), holder line with the owner in _locks/quiet. Unbounded builds take quiet
|
||||
# shared and wait for it; bounded suites (nice 10, a 32-core band) never take it.
|
||||
# lease pool <threads> --label "<text>" [--owner <agent>] [--min N] [--nice N] -- <command...>
|
||||
# A SWEEP or any thread-bound job (main, 7 October 2026, 20:17 UK: adversarial binaries started by hand at --threads 64 to 89,
|
||||
# several beside each other on a box, read load 601): takes up to <threads> FREE cores from the bounded pool (cores 8 to
|
||||
# ncpu-1, the same per-core flocks the bounded builds lease, the first 8 cores reserved for the nodes and the box), never
|
||||
# fewer than --min (default the smaller of <threads> and 16), waiting up to 2 h in 10 s steps with a wait-<pid> line while
|
||||
# fewer are free; then runs the command under nice N (default 10) pinned to the cores taken, with "{cores}" in any
|
||||
# argument replaced by the COUNT taken and "{cpuset}" by the set, and LEASE_CORES / LEASE_CPUSET exported. The sum of
|
||||
# pooled threads on a box therefore never passes 88. Rule: no sweep starts except through `lease pool`.
|
||||
# PRIORITY CLASSES (main, 7 Oct 2026 20:37 UK: the class v5 census sat behind eleven adversarial waiters): release (release
|
||||
# builds and canaries) > v5 ("v5 gate", "v5 kit") > measure (measurement rows) > adv (adversarial sweeps). The class comes
|
||||
# from --class <release|v5|measure|adv>, else from the owner (release, shipper, build-server: release; class-v5: v5;
|
||||
# measure*: measure) or the label ("release", "canary", "pair": release; "v5 gate", "v5 kit": v5; "measure": measure), else
|
||||
# adv. A higher-class waiter takes the next freed cores before any lower-class waiter regardless of arrival order (a lower
|
||||
# class yields while a higher class waits). A lower-class holder above LEASE_PREEMPT_MIN_CORES (32) threads is pre-empted
|
||||
# (SIGTERM to its command, which every sweep honours at its shard boundary; the lane re-queues it) when a higher class has
|
||||
# waited LEASE_PREEMPT_S (120 s), newest holder first, one per period, logged in _log/preempt.log. The class stands in
|
||||
# every wait and lease line (lease status).
|
||||
# lease status every lease, the quiet holder and every waiter with its label
|
||||
# lease reap a holder (lease, quiet or build slot) whose process has been STOPPED (state T) for 5 minutes or more is
|
||||
# killed and its file cleared, one line each in _log/reaped.log; remote-run.sh's keeper calls this every
|
||||
|
|
@ -87,6 +104,73 @@ run_cores() {
|
|||
rm -f "$LOCKS/lease-$$"; say "released cores $set after $(( $(date +%s) - t0 )) s, exit $rc"
|
||||
exit $rc
|
||||
}
|
||||
run_pool() {
|
||||
local want="$1"; shift; local label="" owner="${IGNEUM_AGENT:-unknown}" nice=10 min="" reserve="${LEASE_POOL_RESERVE:-8}" class="" rank
|
||||
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --nice) nice="$2"; shift 2 ;; --min) min="$2"; shift 2 ;; --class) class="$2"; shift 2 ;; --priority) class=v5; shift ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
|
||||
if [ -z "$class" ]; then
|
||||
case "$owner" in release|shipper|build-server) class=release ;; class-v5) class=v5 ;; measure*) class=measure ;; esac
|
||||
[ -n "$class" ] || case "$label" in *release*|*canary*|*pair*) class=release ;; *"v5 gate"*|*"v5 kit"*) class=v5 ;; *measure*) class=measure ;; *) class=adv ;; esac
|
||||
fi
|
||||
case "$class" in release) rank=0 ;; v5) rank=1 ;; measure) rank=2 ;; adv) rank=3 ;; *) say "unknown class $class (release, v5, measure, adv)"; exit 2 ;; esac
|
||||
[ "$want" -ge 1 ] 2>/dev/null || { say "pool needs a thread count"; exit 2; }
|
||||
[ -n "$label" ] || { say "--label is required (the dashboard and the next lane read it)"; exit 2; }
|
||||
[ $# -gt 0 ] || { say "no command"; exit 2; }
|
||||
local ncpu; ncpu="${LEASE_POOL_NCPU:-$(nproc)}"; local size=$((ncpu - reserve)); [ "$want" -gt "$size" ] && want=$size
|
||||
[ -n "$min" ] || min=$(( want < 16 ? want : 16 )); [ "$min" -gt "$want" ] && min=$want
|
||||
local t0 waited=0 line waitfile="$LOCKS/wait-$$" fds=() cores=() c fd taken said=0
|
||||
mkdir -p "$LOCKS"; t0=$(date +%s)
|
||||
line="pid $$ since $(now) waited 0 s: lease pool $want (min $min) class $class/$rank: $label; owner=$owner"; printf '%s\n' "$line" > "$waitfile"
|
||||
trap 'rm -f "$waitfile" "$LOCKS/lease-$$"' EXIT
|
||||
local last_preempt=0 preempt_s="${LEASE_PREEMPT_S:-120}" preempt_min="${LEASE_PREEMPT_MIN_CORES:-32}" step="${LEASE_POOL_STEP_S:-10}" higher
|
||||
while :; do
|
||||
fds=(); cores=(); taken=0
|
||||
# a waiter yields while a HIGHER class waits (its wait line carries "class <name>/<rank>")
|
||||
higher=$(cat "$LOCKS"/wait-* 2>/dev/null | grep -v "^pid $$ " | sed -n 's/.* class [a-z0-9]*\/\([0-9]\):.*/\1/p' | awk -v r="$rank" '$1 < r' | head -1)
|
||||
if [ -z "$higher" ]; then
|
||||
for ((c = reserve; c < ncpu && taken < want; c++)); do
|
||||
exec {fd}>>"$LOCKS/core-$c"
|
||||
if flock -n "$fd"; then fds+=("$fd"); cores+=("$c"); taken=$((taken + 1)); else exec {fd}>&-; fi
|
||||
done
|
||||
[ "$taken" -ge "$min" ] && break
|
||||
for fd in "${fds[@]}"; do exec {fd}>&-; done
|
||||
fi
|
||||
waited=$(( $(date +%s) - t0 )); [ "$waited" -ge 7200 ] && { say "gave up waiting for $min free pool cores after 2 h"; exit 75; }
|
||||
[ "$said" = 0 ] && { say "$taken of $min pool cores free ($(held_cores | wc -w) leased)$([ -n "$higher" ] && echo ', a higher class waits ahead'); waiting"; said=1; }
|
||||
printf '%s\n' "pid $$ since $(now) waited $waited s: lease pool $want (min $min, $taken free) class $class/$rank: $label; owner=$owner" > "$waitfile"
|
||||
# pre-emption: this class has waited preempt_s with the pool short: TERM to the NEWEST lower-class holder above preempt_min cores
|
||||
if [ -z "$higher" ] && [ "$waited" -ge "$preempt_s" ] && [ $(( $(date +%s) - last_preempt )) -ge "$preempt_s" ]; then
|
||||
local vf vpid vrank vcores
|
||||
for vf in $(ls -t "$LOCKS"/lease-* 2>/dev/null); do
|
||||
vrank=$(sed -n 's/.* class [a-z0-9]*\/\([0-9]\) cmd .*/\1/p' "$vf" | head -1); [ -n "$vrank" ] || continue
|
||||
vcores=$(sed -n 's/.* lease pool \([0-9]*\) of .*/\1/p' "$vf" | head -1)
|
||||
[ "$vrank" -gt "$rank" ] && [ "${vcores:-0}" -gt "$preempt_min" ] || continue
|
||||
vpid=$(sed -n 's/.* cmd \([0-9]*\);.*/\1/p' "$vf" | head -1)
|
||||
[ -n "$vpid" ] && kill -0 "$vpid" 2>/dev/null || continue
|
||||
kill -TERM "$vpid" 2>/dev/null; last_preempt=$(date +%s); mkdir -p "$LOGS"
|
||||
echo "$(now) class $class waiter pid $$ ($label; owner=$owner) waited $waited s: TERM to cmd $vpid of $(head -c 200 "$vf" | tr '\n' ' ')" >> "$LOGS/preempt.log"
|
||||
say "pool short for $waited s: sent TERM to the newest lower-class holder's command (pid $vpid, $vcores cores)"
|
||||
break
|
||||
done
|
||||
fi
|
||||
sleep "$step"
|
||||
done
|
||||
waited=$(( $(date +%s) - t0 ))
|
||||
local set; set=$(printf '%s,' "${cores[@]}"); set=${set%,}
|
||||
line="pid $$ since $(now) waited $waited s: lease pool $taken of $want cores $set class $class/$rank: $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/lease-$$"; rm -f "$waitfile"
|
||||
say "holding $taken pool cores ($set, waited $waited s, class $class): $label"
|
||||
local args=() a; for a in "$@"; do a=${a//\{cores\}/$taken}; a=${a//\{cpuset\}/$set}; args+=("$a"); done
|
||||
export LEASE_CORES="$taken" LEASE_CPUSET="$set"
|
||||
local cmdpid
|
||||
if [ "${LEASE_NO_PIN:-0}" = 1 ]; then "${args[@]}" & else nice -n "$nice" taskset -c "$set" "${args[@]}" & fi; cmdpid=$!
|
||||
# the command's pid in the holder line (a rank P waiter's TERM goes to it, never to the lease or its keeper)
|
||||
line="pid $$ since $(now) waited $waited s: lease pool $taken of $want cores $set class $class/$rank cmd $cmdpid; $label; owner=$owner"; printf '%s\n' "$line" > "$LOCKS/lease-$$"
|
||||
( for fd in "${fds[@]}"; do exec {fd}>&-; done; while kill -0 $$ 2>/dev/null; do touch "$LOCKS/lease-$$" 2>/dev/null; [ -s "$LOCKS/lease-$$" ] || printf '%s\n' "$line" > "$LOCKS/lease-$$"; sleep 20; done ) & local keeper=$!
|
||||
trap 'kill -TERM "$cmdpid" 2>/dev/null' TERM INT
|
||||
wait "$cmdpid"; local rc=$?; trap - TERM INT
|
||||
pkill -P "$keeper" 2>/dev/null; kill "$keeper" 2>/dev/null; wait "$keeper" 2>/dev/null
|
||||
rm -f "$LOCKS/lease-$$"; say "released $taken pool cores after $(( $(date +%s) - t0 )) s, exit $rc"
|
||||
exit $rc
|
||||
}
|
||||
run_quiet() {
|
||||
local label="" owner="${IGNEUM_AGENT:-}" cap=1200
|
||||
while [ $# -gt 0 ]; do case "$1" in --label) label="$2"; shift 2 ;; --owner) owner="$2"; shift 2 ;; --cap-s) cap="$2"; shift 2 ;; --) shift; break ;; *) say "unknown option $1"; exit 2 ;; esac; done
|
||||
|
|
@ -147,14 +231,42 @@ self_test() {
|
|||
# 5. a running (not stopped) holder is left alone
|
||||
bash "$me" cores 6 --label R -- sleep 6 2>/dev/null & for i in $(seq 1 50); do ls "$t"/locks/lease-* >/dev/null 2>&1 && break; sleep 0.1; done; sleep 0.3; touch -d '-10 seconds' "$t"/locks/lease-*
|
||||
[ "$(bash "$me" reap 2>/dev/null)" = 0 ] || f "a running holder was reaped"; wait
|
||||
[ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept"
|
||||
# 6. pool: with 12 cores (8 reserved, 4 in the pool) a pool run takes what is free, substitutes the count, and the next waits
|
||||
export LEASE_POOL_NCPU=12
|
||||
bash "$me" pool 88 --min 2 --label P1 -- bash -c "echo \$LEASE_CORES > '$t/p1.n'; sleep 6; date +%s.%N > '$t/p1.end'" 2>/dev/null & for i in $(seq 1 60); do [ -s "$t/p1.n" ] && break; sleep 0.1; done
|
||||
[ "$(cat "$t/p1.n" 2>/dev/null)" = 4 ] || f "a pool run did not take the 4 free pool cores (got '$(cat "$t/p1.n" 2>/dev/null)')"
|
||||
bash "$me" pool 3 --min 1 --label P2 -- bash -c "date +%s.%N > '$t/p2.start'; echo {cores} > '$t/p2.n'" 2>/dev/null
|
||||
python3 -c "import sys; sys.exit(0 if float(open('$t/p2.start').read()) >= float(open('$t/p1.end').read()) else 1)" || f "a pool run started while the pool was full instead of waiting"
|
||||
[ "$(cat "$t/p2.n" 2>/dev/null)" = 3 ] || f "{cores} was not replaced by the count taken (got '$(cat "$t/p2.n" 2>/dev/null)')"
|
||||
wait
|
||||
# 7. CLASS ORDER (known-failed first): an adv holder fills the pool; a v5 waiter and then an adv waiter queue; when the holder
|
||||
# ends, the v5 waiter runs first and the adv waiter only after it, whatever the arrival order
|
||||
export LEASE_PREEMPT_S=100 LEASE_POOL_STEP_S=1
|
||||
bash "$me" pool 4 --min 4 --label HOLD --owner adv-test -- bash -c "date +%s > '$t/h.start'; sleep 4" 2>/dev/null & for i in $(seq 1 60); do [ -s "$t/h.start" ] && break; sleep 0.1; done; sleep 0.3
|
||||
bash "$me" pool 4 --min 4 --label "v5 gate test" --owner class-v5 -- bash -c "date +%s.%N > '$t/v.start'; sleep 2; date +%s.%N > '$t/v.end'" 2>/dev/null & sleep 1.2
|
||||
bash "$me" pool 4 --min 2 --label SWEEP --owner adv-test -- bash -c "date +%s.%N > '$t/s.start'" 2>/dev/null &
|
||||
wait
|
||||
[ -s "$t/v.start" ] && [ -s "$t/s.start" ] || f "class-order fixture did not run both waiters"
|
||||
python3 -c "import sys; sys.exit(0 if float(open('$t/s.start').read()) >= float(open('$t/v.end').read()) else 1)" || f "a sweep took cores a waiting v5 gate asked for"
|
||||
# 8. PRE-EMPTION: an adv holder above the core floor fills the pool; a release waiter pre-empts it (TERM to its command) after
|
||||
# LEASE_PREEMPT_S and runs
|
||||
export LEASE_PREEMPT_S=1 LEASE_PREEMPT_MIN_CORES=1
|
||||
bash "$me" pool 4 --min 4 --label ADV --owner adv-test -- bash -c "trap 'echo termed > \"$t/adv.termed\"; exit 0' TERM; date +%s > '$t/adv.start'; sleep 30 & wait" 2>/dev/null & for i in $(seq 1 60); do [ -s "$t/adv.start" ] && break; sleep 0.1; done; sleep 0.3
|
||||
bash "$me" pool 4 --min 2 --label "release canary test" --owner release -- bash -c "date +%s > '$t/p.start'" 2>/dev/null
|
||||
[ -s "$t/adv.termed" ] || f "the release waiter did not pre-empt the adv holder"
|
||||
[ -s "$t/p.start" ] || f "the release waiter did not run after the pre-emption"
|
||||
wait; unset LEASE_PREEMPT_S LEASE_POOL_STEP_S LEASE_POOL_NCPU LEASE_PREEMPT_MIN_CORES
|
||||
grep -q "class release waiter" "$t/log/preempt.log" 2>/dev/null || f "no preempt.log line"
|
||||
[ -z "$(ls "$t/locks" | grep -E '^(lease|wait)-')" ] || f "pool lease or wait files left behind: $(ls "$t/locks")"
|
||||
[ "$fail" = 0 ] && echo "lease self-test: disjoint leases run together, a shared core waits, quiet is refused beside a slot or a lease and capped, a stopped holder is reaped after the window, a running one is kept, a pool run takes the free pool cores and the next waits for them, a higher class is served before a lower one whatever the arrival order, a release waiter pre-empts an adv holder above the floor"
|
||||
return $fail
|
||||
}
|
||||
case "${1:-}" in
|
||||
cores) shift; run_cores "$@" ;;
|
||||
pool) shift; run_pool "$@" ;;
|
||||
quiet) shift; run_quiet "$@" ;;
|
||||
status) status ;;
|
||||
reap) reap ;;
|
||||
--self-test) self_test ;;
|
||||
*) sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
|
||||
*) sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
|
||||
esac
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ case "$BOX_HOSTNAME" in *-2|*-4) [ -n "$SLOTS_GIVEN" ] || SLOTS=3 ;; esac # bu
|
|||
WORKERS_HOST="${WORKERS_HOST:-build.igneum.network}" # the dashboard feed's HTTPS name (A record in deSEC, 6 Oct 2026)
|
||||
SSH_PUBKEY="${SSH_PUBKEY:-}"
|
||||
BUILD_USER=build
|
||||
ROLE="${ROLE:-box}" # box (the default) or sweep: a rented cloud sweep worker, provisioned minus the runner (7 Oct 2026)
|
||||
BUILD_HOME=/home/$BUILD_USER
|
||||
# the GitHub Actions runner (step_runner, 6 October 2026 evening): a dedicated user, never build and never root
|
||||
RUNNER_USER=runner
|
||||
|
|
@ -673,14 +674,21 @@ do_provision() {
|
|||
step_profile
|
||||
step_node
|
||||
step_sshd
|
||||
step_caddy
|
||||
step_cuda
|
||||
step_ufw
|
||||
step_runner
|
||||
step_cargo_tools
|
||||
step_zig
|
||||
step_night
|
||||
step_headless_check
|
||||
if [ "$ROLE" = sweep ]; then
|
||||
# a rented SWEEP WORKER (Hetzner Cloud CCX, by the hour, main's order 7 Oct 2026): the build user, toolchain, sccache, mirrors,
|
||||
# lease tool and firewall only; no dashboard feed, no CUDA, no CI runner, no night battery, no Chromium, no zig
|
||||
step_ufw
|
||||
step_cargo_tools
|
||||
else
|
||||
step_caddy
|
||||
step_cuda
|
||||
step_ufw
|
||||
step_runner
|
||||
step_cargo_tools
|
||||
step_zig
|
||||
step_night
|
||||
step_headless_check
|
||||
fi
|
||||
step_summary
|
||||
log "done"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -208,14 +208,16 @@ if [ "${1:-}" = --self-test-slots ]; then
|
|||
# 4b. a run keeps off leased cores: with core 1 leased, a 2-core bounded run on a 2-core box says so (the exclusion line)
|
||||
# the lease outlives the fake's slot take and settle (a loaded box took them past 2 s and the first version read no lease)
|
||||
( exec 9>>"$t/locks/core-$(( $(nproc) - 1 ))"; flock 9; sleep 12 ) & sleep 0.5; BR_CORES=2 BR_NICE=10 fake p 1; wait
|
||||
grep -q 'are leased to a measurement; this run keeps to' "$t/p.out" || fail "a run beside a leased core did not exclude it: $(cat "$t/p.out" | tail -3)"
|
||||
# the bounded run leases its own pool cores, so the measurement's core is never among them (the pool line names the set)
|
||||
grep -q 'bounded pool: 2 core(s) leased' "$t/p.out" || fail "a bounded run did not lease its pool cores: $(cat "$t/p.out" | tail -3)"
|
||||
grep -E 'bounded pool' "$t/p.out" | grep -qE "[(,]$(( $(nproc) - 1 ))[,.]" && fail "a bounded run leased the measurement's core"
|
||||
# 5. a probing build leaves a busy slot's holder line intact
|
||||
fake f 3 & sleep 1.2; fake g 1 & sleep 0.3
|
||||
grep -q 'self-test f' "$t/locks/build-0" || fail "the holder line of the busy slot build-0 was lost when another build probed it: '$(cat "$t/locks/build-0")'"
|
||||
wait
|
||||
# 6. the log carries the job count and the measure flag
|
||||
grep -q '"jobs":44' "$t/log/builds.jsonl" && grep -q '"measure":true' "$t/log/builds.jsonl" || fail "builds.jsonl lacks jobs or measure fields"
|
||||
echo "self-test-slots: two concurrent builds 44 each, a lone build 88, a quiet blocks an unbounded build and not a bounded suite, a quiet is refused beside a slot or a lease, a run keeps off leased cores, a probe keeps the holder line, the log carries jobs and measure"; exit 0
|
||||
echo "self-test-slots: two concurrent builds 44 each, a lone build 88, a quiet blocks an unbounded build and not a bounded suite, a quiet is refused beside a slot or a lease, a bounded run leases its pool cores clear of a measurement, a probe keeps the holder line, the log carries jobs and measure"; exit 0
|
||||
fi
|
||||
|
||||
# One run per worktree directory at a time (6 October 2026, 19:51:09 UK: two runs of one worktree started in the same second;
|
||||
|
|
@ -408,7 +410,7 @@ keeper_pid=""
|
|||
keep_line() { # <file> <waited>; the keeper also refreshes the file's mtime (a stopped holder's file goes stale) and reaps stopped
|
||||
# holders of any lease, quiet or slot after 5 minutes through lease.sh (/srv/builds/_bin/lease reap, one line each in reaped.log)
|
||||
local f="$1" w="$2"
|
||||
( exec {mfd}>&- {fd}>&- 2>/dev/null; [ -n "${WT_FD:-}" ] && exec {WT_FD}>&-; while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; touch "$f" 2>/dev/null
|
||||
( exec {mfd}>&- {fd}>&- 2>/dev/null; [ -n "${WT_FD:-}" ] && exec {WT_FD}>&-; for pfd in "${pool_fds[@]}"; do exec {pfd}>&-; done; while kill -0 "$BR_PID" 2>/dev/null; do [ -s "$f" ] || holder_line "$w" > "$f" 2>/dev/null; touch "$f" 2>/dev/null
|
||||
[ -x /srv/builds/_bin/lease ] && IGNEUM_BUILD_SLOTS_DIR="$SLOTS_DIR" IGNEUM_BUILD_LOG_DIR="$LOG_DIR" /srv/builds/_bin/lease reap >/dev/null 2>&1
|
||||
sleep "${BR_KEEP_S:-20}"; done ) &
|
||||
keeper_pid=$!
|
||||
|
|
@ -469,18 +471,42 @@ BR_RUN_LOG="$RUN_LOG_DIR/$BR_HOST-$BR_T0-$BR_PID.log"; export BR_RUN_LOG
|
|||
# (since the third slot on build-2, 7 Oct 2026: a bounded run takes the band its SLOT owns, counted from the top: slot 0 the last N
|
||||
# cores, slot 1 the N below, slot 2 the N below that, so three bounded runs never share a core; a band below core 0 falls back to
|
||||
# the last N)
|
||||
# (the founder, 7 Oct 2026 19:4x BST, both boxes to near max: a bounded run takes the LAST N cores, N = 88 by default, leaving the first 8
|
||||
# to the release builds, the seed and the observer processes; with N above half the box the slots share the band, and nice 10 plus
|
||||
# the per-slot jobs rule keep two suites fair; a smaller N (IGNEUM_BOUND_CORES) returns to disjoint bands per slot when it fits)
|
||||
# The bounded POOL (main, 7 Oct 2026 20:0x BST: with every bounded run taking 88 threads the boxes read load 280 to 350; the ceiling
|
||||
# is per BOX, not per job). Cores 8 to 95 (the first 8 reserved for the release builds, the seed and the observers) form one pool of
|
||||
# 88 per box; a bounded run leases free cores from it (one flock per core, the same core-<n> files a pinned measurement leases, so
|
||||
# measurements and suites share the pool), up to BR_CORES (88 by default, or the caller's --jobs), and never fewer than
|
||||
# BR_POOL_MIN (16): when fewer are free it waits in the queue (its wait-<pid> file says so) and takes them as they free; CARGO_BUILD_JOBS
|
||||
# becomes the count taken and taskset the set taken. So the sum of bounded threads on a box never passes 88. An unbounded run
|
||||
# (nice 0, a gate or a release build) is outside the pool and keeps the full set.
|
||||
ncpu=$(nproc); cores_str="0-$((ncpu - 1))"
|
||||
if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then
|
||||
band=0; case "${got:-}" in ''|measure) ;; *) band=$got ;; esac
|
||||
lo=$((ncpu - BR_CORES * (band + 1))); [ "$lo" -ge 0 ] || lo=$((ncpu - BR_CORES))
|
||||
cores_str="$lo-$((lo + BR_CORES - 1))"
|
||||
pool_fds=(); pool_cores=()
|
||||
pool_take() { # <want> <min>: lease up to <want> free pool cores, at least <min>, waiting up to 2 h; sets pool_cores/pool_fds
|
||||
local want="$1" min="$2" t0 c fd taken
|
||||
t0=$(date +%s)
|
||||
while :; do
|
||||
taken=0; pool_fds=(); pool_cores=()
|
||||
for ((c = ${BR_POOL_RESERVE:-8}; c < ncpu && taken < want; c++)); do
|
||||
exec {fd}>>"$SLOTS_DIR/core-$c"
|
||||
if flock -n "$fd"; then pool_fds+=("$fd"); pool_cores+=("$c"); taken=$((taken + 1)); else exec {fd}>&-; fi
|
||||
done
|
||||
if [ "$taken" -ge "$min" ]; then return 0; fi
|
||||
for fd in "${pool_fds[@]}"; do exec {fd}>&-; done; pool_fds=(); pool_cores=()
|
||||
[ -f "$waitfile" ] || { echo "build-remote: the bounded pool has $taken free core(s) (want $want, at least $min): waiting for the pool" >&2; holder_line "$(( $(date +%s) - BR_T0 ))" > "$waitfile"; }
|
||||
[ $(( $(date +%s) - t0 )) -lt 7200 ] || give_up "the bounded pool"
|
||||
sleep 10
|
||||
done
|
||||
}
|
||||
if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ] && [ "$got" != measure ]; then
|
||||
want="$BR_CORES"; [ "${BR_JOBS_CAP:-0}" -gt 0 ] && [ "$BR_JOBS_CAP" -lt "$want" ] && want="$BR_JOBS_CAP" # the class cap, lowered by an explicit --jobs
|
||||
pool_take "$want" "$(( want < ${BR_POOL_MIN:-16} ? want : ${BR_POOL_MIN:-16} ))"
|
||||
rm -f "$waitfile" 2>/dev/null
|
||||
cores_str=$(printf '%s,' "${pool_cores[@]}"); cores_str=${cores_str%,}
|
||||
BR_JOBS=${#pool_cores[@]}; export CARGO_BUILD_JOBS="$BR_JOBS" BR_JOBS
|
||||
echo "build-remote: bounded pool: ${#pool_cores[@]} core(s) leased ($(echo "$cores_str" | cut -c1-60)...), CARGO_BUILD_JOBS=$BR_JOBS" >&2
|
||||
fi
|
||||
# leased cores (lease.sh: a pinned measurement's core-<n> flocks) are taken out of this run's set; a set that would be empty keeps
|
||||
# its cores (the measurement is told by its own lease line); the exclusion is said once
|
||||
if [ "$got" != measure ]; then
|
||||
if [ "$got" != measure ] && [ "${#pool_cores[@]}" = 0 ]; then
|
||||
leased=$(held_cores)
|
||||
if [ -n "$leased" ]; then
|
||||
kept=$(python3 -c '
|
||||
|
|
|
|||
30
tools/site-deploy-from-mirror.sh
Executable file
30
tools/site-deploy-from-mirror.sh
Executable file
|
|
@ -0,0 +1,30 @@
|
|||
#!/usr/bin/env bash
|
||||
# Deploy the public site from the BOX MIRROR's master (GitHub dark, 7 October 2026): a GIT-LESS export of the site tree at build/master
|
||||
# (or the commit given), the site built in it (node site/build.mjs, the downloads snapshot read from the dl host), then the Vercel CLI
|
||||
# from this Mac with the igneum team config (~/.config/igneum/vercel, --scope igneum; the box holds no token by R6) against the
|
||||
# site project (.vercel/project.json of ~/Projects/igneum/site). Reads nothing from GitHub. Prints the commit served and the
|
||||
# production URL with the UTC time; the worktree is removed after.
|
||||
#
|
||||
# tools/site-deploy-from-mirror.sh [<commit or ref, default build/master>]
|
||||
set -euo pipefail
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
REF="${1:-build/master}"
|
||||
git fetch -q build master
|
||||
SHA=$(git rev-parse "$REF"); SHORT=${SHA:0:8}
|
||||
# a GIT-LESS export (main, 7 Oct 2026 19:4x BST: the worktree deploy G72XWh48k was BLOCKED by the team's commit-author check, which
|
||||
# cannot resolve 337424239+[removed] while the GitHub account is suspended; an export carries no Git
|
||||
# metadata, so the check does not apply). The deploy directory must hold no .git; the check below refuses otherwise.
|
||||
W=$(mktemp -d "${TMPDIR:-/tmp}/site-deploy.XXXXXX")
|
||||
trap 'rm -rf "$W"' EXIT
|
||||
git archive --format=tar "$SHA" site | tar -x -C "$W"
|
||||
[ -e "$W/.git" ] || [ -e "$W/site/.git" ] && { echo "the deploy directory carries a .git; refusing (the author check would block it)" >&2; exit 1; }
|
||||
LINK="$HOME/Projects/igneum/site/.vercel/project.json"; [ -f "$LINK" ] || { echo "no site project link at $LINK" >&2; exit 1; }
|
||||
# the Vercel project's Root Directory is "site": the deploy runs from the export's root with site/ inside it
|
||||
mkdir -p "$W/.vercel"; cp "$LINK" "$W/.vercel/project.json"
|
||||
echo "site export at $SHORT ($(git log -1 --format='%ci %s' "$SHA" | cut -c1-90)); no .git in $W"
|
||||
( cd "$W/site" && node build.mjs ) 2>&1 | tail -3
|
||||
echo "deploying $(TZ=UTC date +%H:%M:%SZ)"
|
||||
OUT=$( cd "$W" && npx --yes vercel@latest --global-config "$HOME/.config/igneum/vercel" --scope igneum deploy --prod --yes 2>&1 ) || { echo "$OUT" | tail -5 >&2; exit 1; }
|
||||
URL=$(echo "$OUT" | grep -oE 'https://[a-z0-9.-]*vercel\.app' | tail -1)
|
||||
echo "SITE DEPLOYED commit $SHORT at $(TZ=UTC date +%H:%M:%SZ) UTC ($(TZ=Europe/London date +%H:%M) BST): $URL -> https://igneum.network"
|
||||
echo "$OUT" | grep -iE "Production:|Aliased|error" | head -3
|
||||
Loading…
Reference in a new issue