#!/usr/bin/env bash
# Rented SWEEP WORKERS on Hetzner Cloud (main's order, 7 October 2026): CCX63 (48 dedicated cores) or CCX53 (32) by the hour,
# provisioned like a build box minus the runner (provision.sh ROLE=sweep: build user with the Mac's key, toolchain, sccache,
# the bare mirrors, the lease tool with its pool, ufw), for the adversarial sweeps that queue on build-1 and build-2. Destroyed
# after. Within USD 1,500 a day, a cost line at every USD 100 (cost). Token at ~/.config/igneum/hetzner-token (never printed),
# key igneum-ops (the Mac's ~/.ssh/igneum_ed25519), hcloud CLI.
#
#   infra/build-server/cloud-sweep.sh up <n> [--type ccx63|ccx53] [--location fsn1|nbg1|hel1]   create igneum-sweep-<n>, provision, host line
#   infra/build-server/cloud-sweep.sh list                                                  the workers with uptime and cost so far
#   infra/build-server/cloud-sweep.sh cost                                                  one line: hours, EUR, USD so far (all workers)
#   infra/build-server/cloud-sweep.sh down <n> [--force]                                    refuse while a lease is held (lease status), then delete
#
# Host files: ~/.config/igneum/sweep-<n> ("build@<ip>", what a lane's ssh line reads). The pool on a worker: cores 8 to ncpu-1
# (40 on a CCX63, 24 on a CCX53); the lanes use the same `lease pool <threads> -- cmd` line as on the boxes.
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
export HCLOUD_TOKEN; HCLOUD_TOKEN="$(tr -d '[:space:]' < "$HOME/.config/igneum/hetzner-token")"
HC="${HCLOUD:-/opt/homebrew/bin/hcloud}"; KEY="$HOME/.ssh/igneum_ed25519"
EUR_USD="${EUR_USD:-1.09}"
die() { echo "cloud-sweep: $*" >&2; exit 1; }
hourly() { "$HC" server-type describe "$1" -o json | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["prices"][0]["price_hourly"]["gross"])'; }
case "${1:-}" in
  up)
    n="${2:-}"; [ -n "$n" ] || die "up <n>"; shift 2; TYPE=ccx63; LOC=fsn1
    while [ $# -gt 0 ]; do case "$1" in --type) TYPE="$2"; shift 2 ;; --location) LOC="$2"; shift 2 ;; *) die "unknown $1" ;; esac; done
    NAME="igneum-sweep-$n"
    if ! "$HC" server describe "$NAME" >/dev/null 2>&1; then
      "$HC" server create --name "$NAME" --type "$TYPE" --image ubuntu-24.04 --location "$LOC" --ssh-key igneum-ops --label igneum=sweep --label role=sweep >/dev/null
      echo "created $NAME ($TYPE, $LOC) at $(TZ=UTC date +%H:%M:%SZ)"
    fi
    IP=$("$HC" server ip "$NAME")
    ssh-keygen -R "$IP" >/dev/null 2>&1 || true
    for i in $(seq 1 60); do ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 "root@$IP" true 2>/dev/null && break; sleep 5; done
    ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 "root@$IP" true || die "$NAME at $IP: ssh never answered"
    # cloud-init may still hold apt for a minute after first boot
    ssh -i "$KEY" -o BatchMode=yes "root@$IP" 'cloud-init status --wait >/dev/null 2>&1 || true'
    ssh -i "$KEY" -o BatchMode=yes -o ServerAliveInterval=15 "root@$IP" "ROLE=sweep SLOTS=1 WORKTREES='' BOX_HOSTNAME=$NAME P2P_PORTS='' bash -s" < "$HERE/provision.sh" 2>&1 | grep -E "^(changed|ok|  |summary|die|error)" | tail -12
    # the lease tool straight from this checkout (the worker's mirror has no master until a lane pushes one)
    ssh -i "$KEY" -o BatchMode=yes "root@$IP" 'install -d -m 755 -o build -g build /srv/builds/_bin /srv/builds/_locks; cat > /srv/builds/_bin/lease.new && chmod 755 /srv/builds/_bin/lease.new && chown build:build /srv/builds/_bin/lease.new && mv /srv/builds/_bin/lease.new /srv/builds/_bin/lease' < "$HERE/lease.sh"
    printf 'build@%s\n' "$IP" > "$HOME/.config/igneum/sweep-$n"
    echo "SWEEP WORKER $NAME build@$IP ($TYPE, $LOC, $(ssh -i "$KEY" -o BatchMode=yes "build@$IP" nproc) threads, pool cores 8 to $(( $(ssh -i "$KEY" -o BatchMode=yes "build@$IP" nproc) - 1 ))), lease tool $(ssh -i "$KEY" -o BatchMode=yes "build@$IP" 'sha256sum /srv/builds/_bin/lease | cut -c1-8'), $(hourly "$TYPE") EUR/h; host file ~/.config/igneum/sweep-$n"
    ;;
  list|cost)
    "$HC" server list -l igneum=sweep -o json | python3 -c '
import sys,json,datetime,subprocess,os
d=json.load(sys.stdin); now=datetime.datetime.now(datetime.timezone.utc); tot_h=0.0; tot_eur=0.0; rate=float(os.environ.get("EUR_USD","1.09")); mode=sys.argv[1]
for s in d:
    t=s["server_type"]["name"]; created=datetime.datetime.fromisoformat(s["created"].replace("Z","+00:00")); h=(now-created).total_seconds()/3600
    pr=float([p for p in s["server_type"]["prices"] if p["location"]==s["datacenter"]["location"]["name"]][0]["price_hourly"]["gross"])
    eur=h*pr; tot_h+=h; tot_eur+=eur
    if mode=="list": print(f"{s[\"name\"]} {s[\"public_net\"][\"ipv4\"][\"ip\"]} {t} {s[\"datacenter\"][\"location\"][\"name\"]} {s[\"status\"]} up {h:.2f} h, EUR {eur:.2f} so far")
print(f"COST sweep workers: {len(d)} running, {tot_h:.2f} worker-hours, EUR {tot_eur:.2f} = USD {tot_eur*rate:.2f} so far (gross, hourly, EUR/USD {rate})")
' "$1"
    ;;
  down)
    n="${2:-}"; [ -n "$n" ] || die "down <n>"; NAME="igneum-sweep-$n"; IP=$("$HC" server ip "$NAME" 2>/dev/null || true)
    if [ -n "$IP" ] && [ "${3:-}" != --force ]; then
      held=$(ssh -i "$KEY" -o BatchMode=yes -o ConnectTimeout=8 "build@$IP" '/srv/builds/_bin/lease status 2>/dev/null | sed -n "/^leases:/,/^quiet:/p" | grep -c "^  pid"' 2>/dev/null || echo 0)
      [ "$held" = 0 ] || die "$NAME holds $held lease(s) (lease status on build@$IP); --force to delete anyway"
    fi
    "$HC" server delete "$NAME" >/dev/null && echo "deleted $NAME at $(TZ=UTC date +%H:%M:%SZ)"; rm -f "$HOME/.config/igneum/sweep-$n"
    ;;
  *) sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;;
esac