igneum/infra/cloud-devnet/create.sh

#!/usr/bin/env bash
# Create the cloud devnet VMs: N nodes, regions round-robin, one firewall, one SSH key. Writes nodes.tsv.
# Hetzner Cloud through `hcloud` (primary). DigitalOcean through `doctl` with PROVIDER=digitalocean.
# Spends money from the moment the servers exist (hourly billing on both providers). It prints the plan and
# the provider's live price and asks for "yes" first (YES=1 skips the question).
#
# Before the first run: `hcloud context create igneum` (paste the project's API token; the project is created by
# Josh in the Hetzner console, not by this script) or `doctl auth init`.
# usage: ./create.sh            create N nodes
#        ./create.sh builder    create only the builder VM (provision.sh does this itself when it needs one)
#
# NET_MODE=private (the default since 4 Oct 2026, see config.sh): one private network per Hetzner network zone,
# the lowest-index node of each zone is its public IPv4 gateway (NAT and one forwarded p2p port per private node),
# every other node is created without public addresses. Re-running the script is safe: servers that exist are kept
# and attached to their zone network if they are not in it yet; nodes.tsv is rewritten from the live state.

. "$(dirname "$0")/lib/common.sh"

what="${1:-nodes}"
mkdir -p "$BUILD_DIR"

# ---- SSH key -------------------------------------------------------------------------------------------------
if [ ! -f "$SSH_KEY_FILE" ]; then
  log "no key at $SSH_KEY_FILE, generating an ed25519 pair (no passphrase; it only opens these test VMs)"
  ssh-keygen -q -t ed25519 -N "" -C "igneum-devnet" -f "$SSH_KEY_FILE"
fi
PUBKEY_FILE="$SSH_KEY_FILE.pub"
[ -f "$PUBKEY_FILE" ] || die "missing $PUBKEY_FILE"

# ---- Hetzner ------------------------------------------------------------------------------------------------------
hetzner_prepare() {
  need hcloud "brew install hcloud"
  hcloud context active >/dev/null 2>&1 || die "no active hcloud context: hcloud context create igneum"
  if ! hcloud ssh-key describe "$SSH_KEY_NAME" >/dev/null 2>&1; then
    hcloud ssh-key create --name "$SSH_KEY_NAME" --public-key-from-file "$PUBKEY_FILE" >/dev/null
    log "uploaded ssh key $SSH_KEY_NAME"
  fi
  if ! hcloud firewall describe "$PREFIX-devnet" >/dev/null 2>&1; then
    hcloud firewall create --name "$PREFIX-devnet" --label igneum=devnet >/dev/null
    hcloud firewall add-rule "$PREFIX-devnet" --direction in --protocol tcp --port 22 --source-ips 0.0.0.0/0 --source-ips ::/0 --description ssh >/dev/null
    hcloud firewall add-rule "$PREFIX-devnet" --direction in --protocol tcp --port "$P2P_PORT" --source-ips 0.0.0.0/0 --source-ips ::/0 --description igneum-p2p >/dev/null
    hcloud firewall add-rule "$PREFIX-devnet" --direction in --protocol icmp --source-ips 0.0.0.0/0 --source-ips ::/0 --description ping >/dev/null
    log "created firewall $PREFIX-devnet (in: 22, $P2P_PORT, icmp; RPC never leaves loopback)"
  fi
  if [ "$NET_MODE" = private ]; then
    local lo=$(( FWD_PORT_BASE + 1 )) hi=$(( FWD_PORT_BASE + 99 ))
    if ! hcloud firewall describe "$PREFIX-devnet" -o json | python3 -c "import json,sys; r=json.load(sys.stdin)['rules']; sys.exit(0 if any(x.get('port')=='$lo-$hi' for x in r) else 1)"; then
      hcloud firewall add-rule "$PREFIX-devnet" --direction in --protocol tcp --port "$lo-$hi" --source-ips 0.0.0.0/0 --source-ips ::/0 --description igneum-p2p-forwarded >/dev/null
      log "firewall: opened tcp $lo-$hi (the gateways' forwarded p2p ports)"
    fi
    for z in $(zones_in_plan); do
      net=$(network_name "$z")
      if ! hcloud network describe "$net" >/dev/null 2>&1; then
        hcloud network create --name "$net" --ip-range "$(zone_net "$z")" --label igneum=devnet --label zone="$z" >/dev/null
        hcloud network add-subnet "$net" --type cloud --network-zone "$z" --ip-range "$(zone_net "$z")" >/dev/null
        log "created network $net $(zone_net "$z") (zone $z)"
      fi
    done
  fi
}

# net hourly price of a server type at a location, from the API (cached per run in build/prices.tsv)
hetzner_hourly() {   # type location
  local f="$BUILD_DIR/prices.tsv" p
  p=$(awk -F'\t' -v t="$1" -v l="$2" '$1 == t && $2 == l { print $3; exit }' "$f" 2>/dev/null)
  if [ -z "$p" ]; then
    hcloud server-type describe "$1" -o json 2>/dev/null | python3 -c 'import json,sys; t=json.load(sys.stdin); [print(t["name"] + "\t" + p["location"] + "\t" + p["price_hourly"]["net"]) for p in t["prices"]]' >> "$f"
    p=$(awk -F'\t' -v t="$1" -v l="$2" '$1 == t && $2 == l { print $3; exit }' "$f")
  fi
  printf '%s' "${p:-0}"
}
# the plan's hourly cost: every node at the live API price, plus USD 0.60/month per public IPv4
hetzner_plan_cost() {
  local i reg t a total=0 ips=0
  for i in $(seq 1 "$N"); do
    reg=$(region_of "$i"); t=$(type_for_index "$i" "$reg"); a=$(access_of "$i")
    total=$(python3 -c "print($total + $(hetzner_hourly "$t" "$reg"))"); [ "$a" = public ] && ips=$((ips + 1))
  done
  total=$(python3 -c "print(round($total + $ips * 0.60 / 730, 4))")
  log "cost of this plan at the live API prices (net USD): $total per hour, about $(python3 -c "print(round($total * 730))") per month, $(python3 -c "print(round($total * 6, 2))") for an evening of 6 h; $ips public IPv4 at 0.60/month each"
}

hetzner_create_one() {   # name type location [role] [access]  (access public|private, private only in NET_MODE=private)
  local name="$1" type="$2" loc="$3" role="${4:-node}" access="${5:-public}" net="" z
  local -a netargs=()
  if [ "$NET_MODE" = private ]; then
    z=$(zone_of_region "$loc"); [ -n "$z" ] || die "$loc is in no zone of ZONES"
    net=$(network_name "$z")
    if [ "$access" = private ]; then netargs=(--without-ipv4 --without-ipv6 --network "$net"); else netargs=(--without-ipv6 --network "$net"); fi
  fi
  if hcloud server describe "$name" >/dev/null 2>&1; then
    log "$name exists, keeping it"
    if [ -n "$net" ] && ! hcloud server describe "$name" -o json | python3 -c "import json,sys; s=json.load(sys.stdin); sys.exit(0 if s['private_net'] else 1)"; then
      hcloud server attach-to-network "$name" --network "$net" >/dev/null && log "attached $name to $net"
    fi
    return
  fi
  hcloud server create --name "$name" --type "$type" --image "$IMAGE" --location "$loc" "${netargs[@]}" \
    --ssh-key "$SSH_KEY_NAME" --firewall "$PREFIX-devnet" --label igneum=devnet --label role="$role" --label access="$access" >/dev/null
  log "created $name ($type, $loc, $access)"
}

# public IPv4 (or "-") and first private IP (or "-") of a server
hetzner_addrs() { hcloud server describe "$1" -o json | python3 -c 'import json,sys; s=json.load(sys.stdin); v4=(s["public_net"].get("ipv4") or {}).get("ip") or "-"; p=s["private_net"][0]["ip"] if s["private_net"] else "-"; print(v4 + "\t" + p)'; }
hetzner_ip() { hcloud server ip "$1"; }
# access of node index i in the plan: the zone gateway (lowest index of the zone) is public, the rest private
access_of() { if [ "$NET_MODE" = private ] && [ "$(gateway_index_for_zone "$(zone_of_region "$(region_of "$1")")")" != "$1" ]; then printf 'private'; else printf 'public'; fi; }

# ---- DigitalOcean --------------------------------------------------------------------------------------------------
do_prepare() {
  need doctl "brew install doctl"
  doctl account get >/dev/null 2>&1 || die "doctl is not authenticated: doctl auth init"
  DO_KEY_ID=$(doctl compute ssh-key list --format ID,Name --no-header | awk -v n="$SSH_KEY_NAME" '$2 == n { print $1 }')
  if [ -z "$DO_KEY_ID" ]; then
    DO_KEY_ID=$(doctl compute ssh-key import "$SSH_KEY_NAME" --public-key-file "$PUBKEY_FILE" --format ID --no-header)
    log "imported ssh key $SSH_KEY_NAME ($DO_KEY_ID)"
  fi
  DO_FW_ID=$(doctl compute firewall list --format ID,Name --no-header | awk -v n="$PREFIX-devnet" '$2 == n { print $1 }')
  if [ -z "$DO_FW_ID" ]; then
    DO_FW_ID=$(doctl compute firewall create --name "$PREFIX-devnet" --tag-names "$PREFIX-devnet" \
      --inbound-rules "protocol:tcp,ports:22,address:0.0.0.0/0,address:::/0 protocol:tcp,ports:$P2P_PORT,address:0.0.0.0/0,address:::/0 protocol:icmp,address:0.0.0.0/0,address:::/0" \
      --outbound-rules "protocol:tcp,ports:all,address:0.0.0.0/0,address:::/0 protocol:udp,ports:all,address:0.0.0.0/0,address:::/0 protocol:icmp,address:0.0.0.0/0,address:::/0" \
      --format ID --no-header)
    log "created firewall $PREFIX-devnet ($DO_FW_ID), applied by tag"
  fi
}

do_price() { log "live price list for $1 (USD):"; doctl compute size list --format Slug,Memory,VCPUs,Disk,PriceMonthly,PriceHourly | grep -E "^Slug|^$1 " || true; }

do_create_one() {   # name size region
  local name="$1" size="$2" reg="$3"
  if doctl compute droplet get "$name" >/dev/null 2>&1; then log "$name exists, keeping it"; return; fi
  doctl compute droplet create "$name" --size "$size" --image "$DO_IMAGE" --region "$reg" --ssh-keys "$DO_KEY_ID" \
    --tag-names "$PREFIX-devnet,role:${4:-node}" --wait >/dev/null
  log "created $name ($size, $reg)"
}

do_ip() { doctl compute droplet get "$1" --format PublicIPv4 --no-header; }

# ---- plan and confirm --------------------------------------------------------------------------------------------
if [ "$PROVIDER" = digitalocean ]; then
  do_prepare; TYPE="$DO_SIZE"; BTYPE="$DO_BUILDER_SIZE"
else
  hetzner_prepare; TYPE="${SERVER_TYPE:-by-location}"; BTYPE="$BUILDER_TYPE"
fi

if [ "$what" = builder ]; then
  log "plan: 1 builder VM $BTYPE in $(region_of 1) on $PROVIDER (deleted by provision.sh after the build unless KEEP_BUILDER=1)"
  if [ "$PROVIDER" = digitalocean ]; then do_price "$BTYPE"; else log "builder $BTYPE in $(region_of 1): USD $(hetzner_hourly "$BTYPE" "$(region_of 1)")/h net"; fi
  confirm "create the builder now (hourly billing starts)?"
  if [ "$PROVIDER" = digitalocean ]; then do_create_one "$PREFIX-builder" "$BTYPE" "$(region_of 1)" builder; ip=$(do_ip "$PREFIX-builder")
  elif [ "$NET_MODE" = private ]; then
    # no public address (the primary IP limit); it sits behind the zone gateway of region 1, ssh jumps through it
    require_nodes; hetzner_create_one "$PREFIX-builder" "$BTYPE" "$(region_of 1)" builder private; ip=$(hetzner_addrs "$PREFIX-builder" | cut -f2)
    for try in $(seq 1 12); do nssh "$ip" true >/dev/null 2>&1 && break; sleep 10; done
    nscp "$HERE/net/private-node.sh" "$SSH_USER@$ip:/root/private-node.sh"
    nssh "$ip" "bash /root/private-node.sh '$(zone_hetzner_gw "$(zone_of_region "$(region_of 1)")")'" | sed 's/^/[builder] /'
  else hetzner_create_one "$PREFIX-builder" "$BTYPE" "$(region_of 1)" builder; ip=$(hetzner_ip "$PREFIX-builder"); fi
  printf '%s\n' "$ip" > "$BUILD_DIR/builder.ip"
  log "builder $ip (saved to build/builder.ip)"
  exit 0
fi

log "plan: $N nodes ($IMAGE) on $PROVIDER, regions round-robin:"
for i in $(seq 1 "$N"); do
  reg=$(region_of "$i"); t="$TYPE"; [ "$PROVIDER" = digitalocean ] || t=$(type_for_index "$i" "$reg")
  a=$(access_of "$i"); [ "$a" = public ] && [ "$NET_MODE" = private ] && a="public (zone gateway: NAT + port forwards)"
  printf '  %s  %s  %s  %s\n' "$(node_name "$i")" "$reg" "$t" "$a"
done
[ "$NET_MODE" = private ] && log "private mode: $(zones_in_plan | wc -l | tr -d ' ') zone networks ($NET_PREFIX.<zone>.0/24), $(for i in $(seq 1 "$N"); do access_of "$i"; printf '\n'; done | grep -c public) public IPv4 primary IPs in total"
if [ "$PROVIDER" = digitalocean ]; then
  do_price "$TYPE"
  log "DigitalOcean s-2vcpu-4gb: USD 24 per node per month (USD 0.036/h, DO pricing page): 20 nodes USD 480/mo, USD 0.71/h"
else
  hetzner_plan_cost
fi
confirm "create $N servers now (hourly billing starts)?"

: > "$NODES_FILE.tmp"
for i in $(seq 1 "$N"); do
  name=$(node_name "$i"); reg=$(region_of "$i")
  if [ "$PROVIDER" = digitalocean ]; then do_create_one "$name" "$TYPE" "$reg"; else hetzner_create_one "$name" "$(type_for_index "$i" "$reg")" "$reg" node "$(access_of "$i")"; fi
done
log "waiting 20 s for the servers to boot, then collecting addresses"
sleep 20
# pass 1: public addresses (the gateways' IPs are needed for the private rows); pass 2: the rows
for i in $(seq 1 "$N"); do
  name=$(node_name "$i"); reg=$(region_of "$i"); a=$(access_of "$i")
  if [ "$PROVIDER" = digitalocean ]; then ip=$(do_ip "$name"); pub="$ip"; priv="-"
  else IFS=$'\t' read -r pub priv <<< "$(hetzner_addrs "$name")"; ip="$pub"; fi
  if [ "$NET_MODE" = private ]; then ip="$priv"; fi
  printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$name" "$i" "$reg" "$ip" "$a" "$pub" "$P2P_PORT" >> "$NODES_FILE.tmp"
done
if [ "$NET_MODE" = private ]; then
  # private rows: pub = the zone gateway's public IPv4, port = FWD_PORT_BASE + index
  NODES_FILE="$NODES_FILE.tmp" python3 - "$NODES_FILE.tmp" "$FWD_PORT_BASE" "$(printf '%s' "$ZONES")" <<'PY'
import sys
path, base, zones = sys.argv[1], int(sys.argv[2]), sys.argv[3]
zone = {}
for part in zones.split(";"):
    z, _, locs = part.split(":"); [zone.__setitem__(l, z) for l in locs.split()]
rows = [l.rstrip("\n").split("\t") for l in open(path) if l.strip()]
gw = {zone[r[2]]: r[5] for r in rows if r[4] == "public"}
out = []
for r in rows:
    if r[4] == "private":
        if zone[r[2]] not in gw: sys.exit("zone %s has no public gateway node" % zone[r[2]])
        r[5] = gw[zone[r[2]]]; r[6] = str(base + int(r[1]))
    out.append("\t".join(r))
open(path, "w").write("\n".join(out) + "\n")
PY
fi
mv "$NODES_FILE.tmp" "$NODES_FILE"
cp "$NODES_FILE" "$BUILD_DIR/nodes-$(date -u +%Y%m%d-%H%M%S).tsv"
log "wrote $NODES_FILE (name, index, region, ip, access, pub, port):"
cat "$NODES_FILE"
if [ "$NET_MODE" = private ] && [ "$PROVIDER" != digitalocean ]; then
  log "gateways: waiting for ssh, then routes, NAT and port forwards"
  for n in $(public_nodes); do for try in $(seq 1 12); do nssh "$(node_ip "$n")" true >/dev/null 2>&1 && break; sleep 10; done; done
  "$HERE/net/setup.sh" gateways
  log "private nodes: waiting for ssh through the gateways, then the uplink check"
  for n in $(private_nodes); do for try in $(seq 1 12); do nssh "$(node_ip "$n")" true >/dev/null 2>&1 && break; sleep 10; done; done
  "$HERE/net/setup.sh" nodes
fi

log "checking ssh on every node (cloud-init can take a minute)"
for try in 1 2 3 4 5 6; do
  bad=0
  while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
    nssh "$ip" true >/dev/null 2>&1 </dev/null || { bad=$((bad + 1)); }
  done 3< "$NODES_FILE"
  [ "$bad" = 0 ] && break
  log "$bad nodes not reachable yet (try $try), waiting 15 s"; sleep 15
done
[ "$bad" = 0 ] || log "WARNING: $bad nodes still unreachable over ssh; provision.sh will retry them"
log "done. Next: ./provision.sh"