64 lines
5.6 KiB
Bash
Executable file
64 lines
5.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Rented SWEEP WORKERS on Hetzner Cloud (main's order, 7 October 2026): CCX63 (48 dedicated cores) or CCX53 (32) by the hour,
|
|
# provisioned like a build box minus the runner (provision.sh ROLE=sweep: build user with the Mac's key, toolchain, sccache,
|
|
# the bare mirrors, the lease tool with its pool, ufw), for the adversarial sweeps that queue on build-1 and build-2. Destroyed
|
|
# after. Within USD 1,500 a day, a cost line at every USD 100 (cost). Token at ~/.config/igneum/hetzner-token (never printed),
|
|
# key igneum-ops (the Mac's ~/.ssh/igneum_ed25519), hcloud CLI.
|
|
#
|
|
# infra/build-server/cloud-sweep.sh up <n> [--type ccx63|ccx53] [--location fsn1|nbg1|hel1] create igneum-sweep-<n>, provision, host line
|
|
# infra/build-server/cloud-sweep.sh list the workers with uptime and cost so far
|
|
# infra/build-server/cloud-sweep.sh cost one line: hours, EUR, USD so far (all workers)
|
|
# infra/build-server/cloud-sweep.sh down <n> [--force] refuse while a lease is held (lease status), then delete
|
|
#
|
|
# Host files: ~/.config/igneum/sweep-<n> ("build@<ip>", what a lane's ssh line reads). The pool on a worker: cores 8 to ncpu-1
|
|
# (40 on a CCX63, 24 on a CCX53); the lanes use the same `lease pool <threads> -- cmd` line as on the boxes.
|
|
set -euo pipefail
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
export HCLOUD_TOKEN; HCLOUD_TOKEN="$(tr -d '[:space:]' < "$HOME/.config/igneum/hetzner-token")"
|
|
HC="${HCLOUD:-/opt/homebrew/bin/hcloud}"; KEY="$HOME/.ssh/igneum_ed25519"
|
|
EUR_USD="${EUR_USD:-1.09}"
|
|
die() { echo "cloud-sweep: $*" >&2; exit 1; }
|
|
hourly() { "$HC" server-type describe "$1" -o json | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["prices"][0]["price_hourly"]["gross"])'; }
|
|
case "${1:-}" in
|
|
up)
|
|
n="${2:-}"; [ -n "$n" ] || die "up <n>"; shift 2; TYPE=ccx63; LOC=fsn1
|
|
while [ $# -gt 0 ]; do case "$1" in --type) TYPE="$2"; shift 2 ;; --location) LOC="$2"; shift 2 ;; *) die "unknown $1" ;; esac; done
|
|
NAME="igneum-sweep-$n"
|
|
if ! "$HC" server describe "$NAME" >/dev/null 2>&1; then
|
|
"$HC" server create --name "$NAME" --type "$TYPE" --image ubuntu-24.04 --location "$LOC" --ssh-key igneum-ops --label igneum=sweep --label role=sweep >/dev/null
|
|
echo "created $NAME ($TYPE, $LOC) at $(TZ=UTC date +%H:%M:%SZ)"
|
|
fi
|
|
IP=$("$HC" server ip "$NAME")
|
|
ssh-keygen -R "$IP" >/dev/null 2>&1 || true
|
|
for i in $(seq 1 60); do ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 "root@$IP" true 2>/dev/null && break; sleep 5; done
|
|
ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 "root@$IP" true || die "$NAME at $IP: ssh never answered"
|
|
# cloud-init may still hold apt for a minute after first boot
|
|
ssh -i "$KEY" -o BatchMode=yes "root@$IP" 'cloud-init status --wait >/dev/null 2>&1 || true'
|
|
ssh -i "$KEY" -o BatchMode=yes -o ServerAliveInterval=15 "root@$IP" "ROLE=sweep SLOTS=1 WORKTREES='' BOX_HOSTNAME=$NAME P2P_PORTS='' bash -s" < "$HERE/provision.sh" 2>&1 | grep -E "^(changed|ok| |summary|die|error)" | tail -12
|
|
# the lease tool straight from this checkout (the worker's mirror has no master until a lane pushes one)
|
|
ssh -i "$KEY" -o BatchMode=yes "root@$IP" 'install -d -m 755 -o build -g build /srv/builds/_bin /srv/builds/_locks; cat > /srv/builds/_bin/lease.new && chmod 755 /srv/builds/_bin/lease.new && chown build:build /srv/builds/_bin/lease.new && mv /srv/builds/_bin/lease.new /srv/builds/_bin/lease' < "$HERE/lease.sh"
|
|
printf 'build@%s\n' "$IP" > "$HOME/.config/igneum/sweep-$n"
|
|
echo "SWEEP WORKER $NAME build@$IP ($TYPE, $LOC, $(ssh -i "$KEY" -o BatchMode=yes "build@$IP" nproc) threads, pool cores 8 to $(( $(ssh -i "$KEY" -o BatchMode=yes "build@$IP" nproc) - 1 ))), lease tool $(ssh -i "$KEY" -o BatchMode=yes "build@$IP" 'sha256sum /srv/builds/_bin/lease | cut -c1-8'), $(hourly "$TYPE") EUR/h; host file ~/.config/igneum/sweep-$n"
|
|
;;
|
|
list|cost)
|
|
"$HC" server list -l igneum=sweep -o json | python3 -c '
|
|
import sys,json,datetime,subprocess,os
|
|
d=json.load(sys.stdin); now=datetime.datetime.now(datetime.timezone.utc); tot_h=0.0; tot_eur=0.0; rate=float(os.environ.get("EUR_USD","1.09")); mode=sys.argv[1]
|
|
for s in d:
|
|
t=s["server_type"]["name"]; created=datetime.datetime.fromisoformat(s["created"].replace("Z","+00:00")); h=(now-created).total_seconds()/3600
|
|
pr=float([p for p in s["server_type"]["prices"] if p["location"]==s["datacenter"]["location"]["name"]][0]["price_hourly"]["gross"])
|
|
eur=h*pr; tot_h+=h; tot_eur+=eur
|
|
if mode=="list": print(f"{s[\"name\"]} {s[\"public_net\"][\"ipv4\"][\"ip\"]} {t} {s[\"datacenter\"][\"location\"][\"name\"]} {s[\"status\"]} up {h:.2f} h, EUR {eur:.2f} so far")
|
|
print(f"COST sweep workers: {len(d)} running, {tot_h:.2f} worker-hours, EUR {tot_eur:.2f} = USD {tot_eur*rate:.2f} so far (gross, hourly, EUR/USD {rate})")
|
|
' "$1"
|
|
;;
|
|
down)
|
|
n="${2:-}"; [ -n "$n" ] || die "down <n>"; NAME="igneum-sweep-$n"; IP=$("$HC" server ip "$NAME" 2>/dev/null || true)
|
|
if [ -n "$IP" ] && [ "${3:-}" != --force ]; then
|
|
held=$(ssh -i "$KEY" -o BatchMode=yes -o ConnectTimeout=8 "build@$IP" '/srv/builds/_bin/lease status 2>/dev/null | sed -n "/^leases:/,/^quiet:/p" | grep -c "^ pid"' 2>/dev/null || echo 0)
|
|
[ "$held" = 0 ] || die "$NAME holds $held lease(s) (lease status on build@$IP); --force to delete anyway"
|
|
fi
|
|
"$HC" server delete "$NAME" >/dev/null && echo "deleted $NAME at $(TZ=UTC date +%H:%M:%SZ)"; rm -f "$HOME/.config/igneum/sweep-$n"
|
|
;;
|
|
*) sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
|
|
esac
|