d6c69bcb0c
infra/cloud-devnet: hcloud (doctl variant) create, builder-VM provision from a git-archive source tarball, systemd units for igneumd --devnet-suffix with a sparse --addpeer mesh and a CPU trickle miner per node, stdlib wRPC client, experiments (latency, partition, hop, collect, observer hookup), README with the command sequence and the Hetzner API prices of 3 Oct 2026. infra/gpu-bench: RunPod image recipes (CUDA 12.8, ROCm), bundle, run.sh (vectors gate, 10-min raw, sweep, inline shortcut ratio, nvcc/NVRTC/OpenCL recompile timings, results row, intake upload), bench-log template. infra/seed-nodes: create-seed (persistent IPv4, firewall), provision on the VM, health check, addPeer from the Mac over grpcurl, seeds.txt; igneum-seed-1 created at 188.245.5.161 (Hetzner cx23, fsn1). docs/plans/cloud-devnet.md and docs/plans/seed-nodes.md. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
147 lines
8.1 KiB
Bash
Executable file
147 lines
8.1 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Create the cloud devnet VMs: N nodes, regions round-robin, one firewall, one SSH key. Writes nodes.tsv.
|
|
# Hetzner Cloud through `hcloud` (primary). DigitalOcean through `doctl` with PROVIDER=digitalocean.
|
|
# Spends money from the moment the servers exist (hourly billing on both providers). It prints the plan and
|
|
# the provider's live price and asks for "yes" first (YES=1 skips the question).
|
|
#
|
|
# Before the first run: `hcloud context create igneum` (paste the project's API token; the project is created by
|
|
# the project lead in the Hetzner console, not by this script) or `doctl auth init`.
|
|
# usage: ./create.sh create N nodes
|
|
# ./create.sh builder create only the builder VM (provision.sh does this itself when it needs one)
|
|
|
|
. "$(dirname "$0")/lib/common.sh"
|
|
|
|
what="${1:-nodes}"
|
|
mkdir -p "$BUILD_DIR"
|
|
|
|
# ---- SSH key -------------------------------------------------------------------------------------------------
|
|
if [ ! -f "$SSH_KEY_FILE" ]; then
|
|
log "no key at $SSH_KEY_FILE, generating an ed25519 pair (no passphrase; it only opens these test VMs)"
|
|
ssh-keygen -q -t ed25519 -N "" -C "igneum-devnet" -f "$SSH_KEY_FILE"
|
|
fi
|
|
PUBKEY_FILE="$SSH_KEY_FILE.pub"
|
|
[ -f "$PUBKEY_FILE" ] || die "missing $PUBKEY_FILE"
|
|
|
|
# ---- Hetzner ------------------------------------------------------------------------------------------------------
|
|
hetzner_prepare() {
|
|
need hcloud "brew install hcloud"
|
|
hcloud context active >/dev/null 2>&1 || die "no active hcloud context: hcloud context create igneum"
|
|
if ! hcloud ssh-key describe "$SSH_KEY_NAME" >/dev/null 2>&1; then
|
|
hcloud ssh-key create --name "$SSH_KEY_NAME" --public-key-from-file "$PUBKEY_FILE" >/dev/null
|
|
log "uploaded ssh key $SSH_KEY_NAME"
|
|
fi
|
|
if ! hcloud firewall describe "$PREFIX-devnet" >/dev/null 2>&1; then
|
|
hcloud firewall create --name "$PREFIX-devnet" --label igneum=devnet >/dev/null
|
|
hcloud firewall add-rule "$PREFIX-devnet" --direction in --protocol tcp --port 22 --source-ips 0.0.0.0/0 --source-ips ::/0 --description ssh >/dev/null
|
|
hcloud firewall add-rule "$PREFIX-devnet" --direction in --protocol tcp --port "$P2P_PORT" --source-ips 0.0.0.0/0 --source-ips ::/0 --description igneum-p2p >/dev/null
|
|
hcloud firewall add-rule "$PREFIX-devnet" --direction in --protocol icmp --source-ips 0.0.0.0/0 --source-ips ::/0 --description ping >/dev/null
|
|
log "created firewall $PREFIX-devnet (in: 22, $P2P_PORT, icmp; RPC never leaves loopback)"
|
|
fi
|
|
}
|
|
|
|
hetzner_price() {
|
|
log "live price list for $1 (per location, EUR, excl. VAT):"
|
|
hcloud server-type describe "$1" 2>/dev/null | sed -n '/Pricings/,$p' | head -40 || true
|
|
}
|
|
|
|
hetzner_create_one() { # name type location
|
|
local name="$1" type="$2" loc="$3"
|
|
if hcloud server describe "$name" >/dev/null 2>&1; then log "$name exists, keeping it"; return; fi
|
|
hcloud server create --name "$name" --type "$type" --image "$IMAGE" --location "$loc" \
|
|
--ssh-key "$SSH_KEY_NAME" --firewall "$PREFIX-devnet" --label igneum=devnet --label role="${4:-node}" >/dev/null
|
|
log "created $name ($type, $loc)"
|
|
}
|
|
|
|
hetzner_ip() { hcloud server ip "$1"; }
|
|
|
|
# ---- DigitalOcean --------------------------------------------------------------------------------------------------
|
|
do_prepare() {
|
|
need doctl "brew install doctl"
|
|
doctl account get >/dev/null 2>&1 || die "doctl is not authenticated: doctl auth init"
|
|
DO_KEY_ID=$(doctl compute ssh-key list --format ID,Name --no-header | awk -v n="$SSH_KEY_NAME" '$2 == n { print $1 }')
|
|
if [ -z "$DO_KEY_ID" ]; then
|
|
DO_KEY_ID=$(doctl compute ssh-key import "$SSH_KEY_NAME" --public-key-file "$PUBKEY_FILE" --format ID --no-header)
|
|
log "imported ssh key $SSH_KEY_NAME ($DO_KEY_ID)"
|
|
fi
|
|
DO_FW_ID=$(doctl compute firewall list --format ID,Name --no-header | awk -v n="$PREFIX-devnet" '$2 == n { print $1 }')
|
|
if [ -z "$DO_FW_ID" ]; then
|
|
DO_FW_ID=$(doctl compute firewall create --name "$PREFIX-devnet" --tag-names "$PREFIX-devnet" \
|
|
--inbound-rules "protocol:tcp,ports:22,address:0.0.0.0/0,address:::/0 protocol:tcp,ports:$P2P_PORT,address:0.0.0.0/0,address:::/0 protocol:icmp,address:0.0.0.0/0,address:::/0" \
|
|
--outbound-rules "protocol:tcp,ports:all,address:0.0.0.0/0,address:::/0 protocol:udp,ports:all,address:0.0.0.0/0,address:::/0 protocol:icmp,address:0.0.0.0/0,address:::/0" \
|
|
--format ID --no-header)
|
|
log "created firewall $PREFIX-devnet ($DO_FW_ID), applied by tag"
|
|
fi
|
|
}
|
|
|
|
do_price() { log "live price list for $1 (USD):"; doctl compute size list --format Slug,Memory,VCPUs,Disk,PriceMonthly,PriceHourly | grep -E "^Slug|^$1 " || true; }
|
|
|
|
do_create_one() { # name size region
|
|
local name="$1" size="$2" reg="$3"
|
|
if doctl compute droplet get "$name" >/dev/null 2>&1; then log "$name exists, keeping it"; return; fi
|
|
doctl compute droplet create "$name" --size "$size" --image "$DO_IMAGE" --region "$reg" --ssh-keys "$DO_KEY_ID" \
|
|
--tag-names "$PREFIX-devnet,role:${4:-node}" --wait >/dev/null
|
|
log "created $name ($size, $reg)"
|
|
}
|
|
|
|
do_ip() { doctl compute droplet get "$1" --format PublicIPv4 --no-header; }
|
|
|
|
# ---- plan and confirm --------------------------------------------------------------------------------------------
|
|
if [ "$PROVIDER" = digitalocean ]; then
|
|
do_prepare; TYPE="$DO_SIZE"; BTYPE="$DO_BUILDER_SIZE"
|
|
else
|
|
hetzner_prepare; TYPE="${SERVER_TYPE:-by-location}"; BTYPE="$BUILDER_TYPE"
|
|
fi
|
|
|
|
if [ "$what" = builder ]; then
|
|
log "plan: 1 builder VM $BTYPE in $(region_of 1) on $PROVIDER (deleted by provision.sh after the build unless KEEP_BUILDER=1)"
|
|
if [ "$PROVIDER" = digitalocean ]; then do_price "$BTYPE"; else hetzner_price "$BTYPE"; fi
|
|
confirm "create the builder now (hourly billing starts)?"
|
|
if [ "$PROVIDER" = digitalocean ]; then do_create_one "$PREFIX-builder" "$BTYPE" "$(region_of 1)" builder; ip=$(do_ip "$PREFIX-builder")
|
|
else hetzner_create_one "$PREFIX-builder" "$BTYPE" "$(region_of 1)" builder; ip=$(hetzner_ip "$PREFIX-builder"); fi
|
|
printf '%s\n' "$ip" > "$BUILD_DIR/builder.ip"
|
|
log "builder $ip (saved to build/builder.ip)"
|
|
exit 0
|
|
fi
|
|
|
|
log "plan: $N nodes ($IMAGE) on $PROVIDER, regions round-robin:"
|
|
for i in $(seq 1 "$N"); do
|
|
reg=$(region_of "$i"); t="$TYPE"; [ "$PROVIDER" = digitalocean ] || t=$(type_for_location "$reg")
|
|
printf ' %s %s %s\n' "$(node_name "$i")" "$reg" "$t"
|
|
done
|
|
if [ "$PROVIDER" = digitalocean ]; then do_price "$TYPE"; else
|
|
for t in $(for i in $(seq 1 "$N"); do type_for_location "$(region_of "$i")"; done | sort -u); do hetzner_price "$t"; done
|
|
fi
|
|
log "cost at the Hetzner API prices of 3 Oct 2026 (net EUR per month: cx23 6.49 EU, cpx22 30.99 sin, cpx21 37.49 ash/hil):"
|
|
log " 20 nodes as 4 per location = 8 x 6.49 + 4 x 30.99 + 8 x 37.49 = EUR 476/mo, EUR 0.76/h; an evening of 6 h about EUR 5 plus the builder (about EUR 0.03/h)"
|
|
log " EU-heavy alternative REGIONS=hel1,fsn1,nbg1,hel1,fsn1,nbg1,hel1,ash,hil,sin (14 EU, 2 each elsewhere): about EUR 303/mo, EUR 0.47/h"
|
|
log " DigitalOcean s-2vcpu-4gb: USD 24 per node per month (USD 0.036/h, DO pricing page): 20 nodes USD 480/mo, USD 0.71/h"
|
|
confirm "create $N servers now (hourly billing starts)?"
|
|
|
|
: > "$NODES_FILE.tmp"
|
|
for i in $(seq 1 "$N"); do
|
|
name=$(node_name "$i"); reg=$(region_of "$i")
|
|
if [ "$PROVIDER" = digitalocean ]; then do_create_one "$name" "$TYPE" "$reg"; else hetzner_create_one "$name" "$(type_for_location "$reg")" "$reg"; fi
|
|
done
|
|
log "waiting 20 s for the servers to boot, then collecting addresses"
|
|
sleep 20
|
|
for i in $(seq 1 "$N"); do
|
|
name=$(node_name "$i"); reg=$(region_of "$i")
|
|
if [ "$PROVIDER" = digitalocean ]; then ip=$(do_ip "$name"); else ip=$(hetzner_ip "$name"); fi
|
|
printf '%s\t%s\t%s\t%s\n' "$name" "$i" "$reg" "$ip" >> "$NODES_FILE.tmp"
|
|
done
|
|
mv "$NODES_FILE.tmp" "$NODES_FILE"
|
|
cp "$NODES_FILE" "$BUILD_DIR/nodes-$(date -u +%Y%m%d-%H%M%S).tsv"
|
|
log "wrote $NODES_FILE:"
|
|
cat "$NODES_FILE"
|
|
|
|
log "checking ssh on every node (cloud-init can take a minute)"
|
|
for try in 1 2 3 4 5 6; do
|
|
bad=0
|
|
while IFS=$'\t' read -r name idx reg ip; do
|
|
nssh "$ip" true >/dev/null 2>&1 || { bad=$((bad + 1)); }
|
|
done < "$NODES_FILE"
|
|
[ "$bad" = 0 ] && break
|
|
log "$bad nodes not reachable yet (try $try), waiting 15 s"; sleep 15
|
|
done
|
|
[ "$bad" = 0 ] || log "WARNING: $bad nodes still unreachable over ssh; provision.sh will retry them"
|
|
log "done. Next: ./provision.sh"
|