4574890602
A new Hetzner account is capped at 10 primary IPs (IPv4 and IPv6 both count), 20 shared vCPUs, 8 dedicated vCPUs and no Arm, and a network cannot span zones. So: one private network per zone (10.20.<zone>.0/24), the lowest-index node of each zone keeps a public IPv4 and is its gateway (NAT, MSS clamp, one DNAT port 27000+index per private node, persisted as igneum-nat.service), every other node has no public address. nodes.tsv gains access, pub and port columns; lib resolves same-zone vs cross-zone dial addresses and jumps ssh through the gateway for private nodes. All nodes.tsv loops read on fd 3 (a backgrounded ssh drained the file). TYPE_BY_INDEX puts nodes 8 to 11 on ccx13; node 12 is the last shared one the account allows. create.sh prints the plan's cost from the live API. provision.sh install takes node names and skips binaries whose sha256 matches. Binaries copied from the seed's staged v4 build (same sources), no vCPU for a builder. Results 2026-10-04: RTT matrix (hel1-fsn1 35 ms, ash-sin 289 ms) and a 10-minute propagation window of 644 blocks: p50 343 ms, p90 497 ms, p99 666 ms across 12 nodes in 5 locations. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
192 lines
10 KiB
Bash
192 lines
10 KiB
Bash
# Shared helpers for infra/cloud-devnet. Source this; it sources config.sh.
|
|
# Needs bash 3.2 or newer (the Mac's /bin/bash is fine), ssh, scp, python3.
|
|
|
|
set -euo pipefail
|
|
|
|
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
REPO="$(cd "$HERE/../.." && pwd)"
|
|
export REPO
|
|
# shellcheck source=../config.sh
|
|
. "$HERE/config.sh"
|
|
|
|
# nodes.tsv, one line per node, written by create.sh (7 tab-separated columns):
|
|
# 1 name 2 index 3 region 4 ip 5 access 6 pub 7 port
|
|
# ip = the node's private IP (NET_MODE=private) or its public IPv4 (NET_MODE=public): the address same-zone
|
|
# peers and every script use for the node; ssh resolves it (ssh_target) to a direct or a jumped connection
|
|
# access = public | private
|
|
# pub = the public IPv4 that reaches this node from outside its zone: its own (public) or its zone gateway's (private)
|
|
# port = the TCP port on pub that reaches this node's p2p: 26611 (public) or FWD_PORT_BASE + index (private)
|
|
# Readers must take all seven fields: `while IFS=$'\t' read -r name idx reg ip access pub port`.
|
|
NODES_FILE="${NODES_FILE:-$HERE/nodes.tsv}"
|
|
BUILD_DIR="$HERE/build"
|
|
BIN_DIR="$BUILD_DIR/bin"
|
|
RESULTS_DIR="$HERE/results"
|
|
|
|
# Hetzner API token: the only line of ~/.config/igneum/hetzner-token (never printed, never in the repo).
|
|
if [ -z "${HCLOUD_TOKEN:-}" ] && [ -s "${HETZNER_TOKEN_FILE:-$HOME/.config/igneum/hetzner-token}" ]; then
|
|
HCLOUD_TOKEN="$(head -1 "${HETZNER_TOKEN_FILE:-$HOME/.config/igneum/hetzner-token}" | tr -d '[:space:]')"; export HCLOUD_TOKEN
|
|
fi
|
|
|
|
log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; }
|
|
die() { log "ERROR: $*" >&2; exit 1; }
|
|
need() { command -v "$1" >/dev/null 2>&1 || die "$1 is not installed ($2)"; }
|
|
|
|
# Hetzner location name for node index i (1-based) and the DigitalOcean one.
|
|
region_of() {
|
|
local i="$1" list
|
|
if [ "$PROVIDER" = digitalocean ]; then list="$DO_REGIONS"; else list="$REGIONS"; fi
|
|
local n; n=$(printf '%s' "$list" | tr ',' '\n' | grep -c .)
|
|
printf '%s' "$list" | tr ',' '\n' | sed -n "$(( (i - 1) % n + 1 ))p"
|
|
}
|
|
|
|
node_name() { printf '%s-%02d' "$PREFIX" "$1"; }
|
|
|
|
# Server type for a Hetzner location: SERVER_TYPE if set, else the TYPE_BY_LOCATION map, else cx23.
|
|
type_for_location() {
|
|
if [ -n "$SERVER_TYPE" ]; then printf '%s' "$SERVER_TYPE"; return; fi
|
|
local t; t=$(printf '%s' "$TYPE_BY_LOCATION" | tr ',' '\n' | awk -F= -v l="$1" '$1 == l { print $2 }')
|
|
printf '%s' "${t:-cx23}"
|
|
}
|
|
|
|
# Server type for node index i in region r: a TYPE_BY_INDEX range that contains i, else type_for_location.
|
|
type_for_index() {
|
|
local t; t=$(printf '%s' "${TYPE_BY_INDEX:-}" | tr ',' '\n' | awk -F'[-:]' -v i="$1" 'NF == 3 && i >= $1 && i <= $2 { print $3; exit }')
|
|
if [ -n "$t" ]; then printf '%s' "$t"; else type_for_location "$2"; fi
|
|
}
|
|
|
|
# ---- zones (Hetzner network zones; a private network lives in exactly one) ----------------------------------------
|
|
# zone_of_region hel1 -> eu-central; zone_index eu-central -> 1; zone_net eu-central -> 10.20.1.0/24
|
|
zone_of_region() {
|
|
if [ "$NET_MODE" != private ]; then printf 'public'; return; fi
|
|
printf '%s' "$ZONES" | tr ';' '\n' | awk -F: -v r="$1" '{ n = split($3, l, " "); for (i = 1; i <= n; i++) if (l[i] == r) { print $1; exit } }'
|
|
}
|
|
zone_index() { printf '%s' "$ZONES" | tr ';' '\n' | awk -F: -v z="$1" '$1 == z { print $2; exit }'; }
|
|
zone_net() { printf '%s.%s.0/24' "$NET_PREFIX" "$(zone_index "$1")"; }
|
|
zone_hetzner_gw() { printf '%s.%s.1' "$NET_PREFIX" "$(zone_index "$1")"; } # Hetzner's own router in the subnet
|
|
network_name() { printf '%s-net-%s' "$PREFIX" "$1"; }
|
|
zones_in_plan() { local i; for i in $(seq 1 "$N"); do zone_of_region "$(region_of "$i")"; done | sort -u; }
|
|
# the gateway of a zone in the plan: the lowest node index whose region is in that zone
|
|
gateway_index_for_zone() { local i; for i in $(seq 1 "$N"); do [ "$(zone_of_region "$(region_of "$i")")" = "$1" ] && { printf '%s' "$i"; return; }; done; }
|
|
zone_of_ip() { printf '%s' "$ZONES" | tr ';' '\n' | awk -F: -v p="$NET_PREFIX" -v ip="$1" '{ if (index(ip, p "." $2 ".") == 1) { print $1; exit } }'; }
|
|
|
|
# ---- nodes.tsv accessors -----------------------------------------------------------------------------------------
|
|
require_nodes() { [ -s "$NODES_FILE" ] || die "no $NODES_FILE: run create.sh first"; }
|
|
node_count() { grep -c . "$NODES_FILE"; }
|
|
node_field() { awk -F'\t' -v n="$1" -v c="$2" '$1 == n || $2 == n { print $c; exit }' "$NODES_FILE"; } # name-or-index, column
|
|
node_ip() { node_field "$1" 4; }
|
|
node_region() { node_field "$1" 3; }
|
|
node_access() { node_field "$1" 5; }
|
|
node_pub() { node_field "$1" 6; }
|
|
node_port() { node_field "$1" 7; }
|
|
node_zone() { zone_of_region "$(node_region "$1")"; }
|
|
node_names() { cut -f1 "$NODES_FILE"; }
|
|
nodes_in_region() { awk -F'\t' -v r="$1" '$3 == r { print $1 }' "$NODES_FILE"; }
|
|
regions_present() { cut -f3 "$NODES_FILE" | sort -u; }
|
|
public_nodes() { awk -F'\t' '$5 == "public" { print $1 }' "$NODES_FILE"; }
|
|
private_nodes() { awk -F'\t' '$5 == "private" { print $1 }' "$NODES_FILE"; }
|
|
# the gateway (public node) of a zone, from nodes.tsv; its public IP
|
|
gateway_of_zone() { local n; for n in $(public_nodes); do [ "$(node_zone "$n")" = "$1" ] && { printf '%s' "$n"; return; }; done; }
|
|
gateway_pub_of_zone() { local g; g=$(gateway_of_zone "$1"); [ -n "$g" ] && node_pub "$g"; }
|
|
|
|
# The address node `from` uses to reach node `to` (name or index): the private IP inside one zone, the public
|
|
# address (own or gateway) across zones. reach_addr has no port; dial_addr has the p2p port.
|
|
reach_addr() {
|
|
if [ "$NET_MODE" != private ] || [ "$(node_zone "$1")" = "$(node_zone "$2")" ]; then node_ip "$2"; else node_pub "$2"; fi
|
|
}
|
|
dial_addr() {
|
|
if [ "$NET_MODE" != private ] || [ "$(node_zone "$1")" = "$(node_zone "$2")" ]; then printf '%s:%s' "$(node_ip "$2")" "$P2P_PORT"
|
|
else printf '%s:%s' "$(node_pub "$2")" "$(node_port "$2")"; fi
|
|
}
|
|
# what the node claims as its own address (--externalip): public nodes their IP, private nodes gateway:port
|
|
external_addr() {
|
|
if [ "$(node_access "$1")" = private ]; then printf '%s:%s' "$(node_pub "$1")" "$(node_port "$1")"; else node_pub "$1"; fi
|
|
}
|
|
|
|
# Outbound peers of node index i: the ring neighbour i+1 and chords at i + k*N/MESH_OUT (1-based, wrapping),
|
|
# each as the address i can dial. With MESH_OUT=2 that is i+1 and i+N/2; inbound links double it, about 4 peers each.
|
|
peers_of() {
|
|
local i="$1" n; n=$(node_count)
|
|
local k step idx out=""
|
|
for k in $(seq 1 "$MESH_OUT"); do
|
|
if [ "$k" = 1 ]; then step=1; else step=$(( (k - 1) * n / MESH_OUT )); fi
|
|
idx=$(( (i - 1 + step) % n + 1 ))
|
|
[ "$idx" = "$i" ] && continue
|
|
out="$out $(dial_addr "$i" "$idx")"
|
|
done
|
|
printf '%s' "$out" | sed 's/^ //' | tr ' ' ','
|
|
}
|
|
|
|
# ---- ssh -----------------------------------------------------------------------------------------------------------
|
|
SSH_OPTS=(-i "$SSH_KEY_FILE" -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$HERE/build/known_hosts" -o ConnectTimeout=15 -o BatchMode=yes -o ServerAliveInterval=30)
|
|
|
|
# ssh_target <ip> prints "<host>" or "<host>\t<jump public ip>": a public node's ip (either column) connects
|
|
# directly to its public IPv4; a private address (the node's or the builder's) jumps through its zone's gateway.
|
|
ssh_target() {
|
|
local ip="$1" pub z gw
|
|
if [ "$NET_MODE" != private ] || [ ! -s "$NODES_FILE" ]; then printf '%s' "$ip"; return; fi
|
|
pub=$(awk -F'\t' -v ip="$ip" '$5 == "public" && ($4 == ip || $6 == ip) { print $6; exit }' "$NODES_FILE")
|
|
if [ -n "$pub" ]; then printf '%s' "$pub"; return; fi
|
|
case "$ip" in
|
|
"$NET_PREFIX".*) z=$(zone_of_ip "$ip"); gw=$(gateway_pub_of_zone "$z")
|
|
[ -n "$gw" ] || die "no gateway in nodes.tsv for zone $z (ip $ip)"
|
|
printf '%s\t%s' "$ip" "$gw" ;;
|
|
*) printf '%s' "$ip" ;;
|
|
esac
|
|
}
|
|
# ssh option list for a target: adds a ProxyCommand through the jump host when ssh_target gave one. The jump
|
|
# connection gets the same key and known_hosts (a plain -J would not pass -i along).
|
|
ssh_opts_for() {
|
|
local t tab; tab=$' '; t=$(ssh_target "$1"); local host="${t%%"$tab"*}" jump="${t#*"$tab"}"
|
|
SSH_HOST="$host"
|
|
if [ "$jump" != "$t" ] && [ -n "$jump" ]; then
|
|
SSH_EXTRA=(-o "ProxyCommand=ssh -i $SSH_KEY_FILE -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$HERE/build/known_hosts -o ConnectTimeout=15 -o BatchMode=yes -W %h:%p $SSH_USER@$jump")
|
|
else
|
|
SSH_EXTRA=()
|
|
fi
|
|
}
|
|
# One retry when the connection itself failed (ssh exit 255: a dropped jump, a node still booting); the command's
|
|
# own exit code is passed through untouched.
|
|
nssh() {
|
|
local ip="$1" rc; shift; ssh_opts_for "$ip"
|
|
ssh "${SSH_OPTS[@]}" ${SSH_EXTRA[@]+"${SSH_EXTRA[@]}"} "$SSH_USER@$SSH_HOST" "$@" && return 0; rc=$?
|
|
[ "$rc" = 255 ] || return "$rc"
|
|
sleep 3; ssh "${SSH_OPTS[@]}" ${SSH_EXTRA[@]+"${SSH_EXTRA[@]}"} "$SSH_USER@$SSH_HOST" "$@"
|
|
}
|
|
# scp with the same resolution: the remote side is whichever argument looks like user@host:path (one host per call)
|
|
nscp() {
|
|
local a ip="" args=()
|
|
for a in "$@"; do
|
|
case "$a" in "$SSH_USER@"*:*) ip="${a#"$SSH_USER"@}"; ip="${ip%%:*}" ;; esac
|
|
done
|
|
[ -n "$ip" ] || die "nscp: no $SSH_USER@host:path argument"
|
|
ssh_opts_for "$ip"
|
|
for a in "$@"; do
|
|
case "$a" in "$SSH_USER@$ip:"*) args+=("$SSH_USER@$SSH_HOST:${a#"$SSH_USER@$ip:"}") ;; *) args+=("$a") ;; esac
|
|
done
|
|
scp -q "${SSH_OPTS[@]}" ${SSH_EXTRA[@]+"${SSH_EXTRA[@]}"} "${args[@]}" && return 0
|
|
sleep 3; scp -q "${SSH_OPTS[@]}" ${SSH_EXTRA[@]+"${SSH_EXTRA[@]}"} "${args[@]}"
|
|
}
|
|
|
|
# Run a command on every node in parallel (PAR at a time), prefixing output with the node name.
|
|
# usage: on_all '<shell command>' (the command runs on the node through ssh)
|
|
on_all() {
|
|
local cmd="$1" par="${PAR:-10}" name idx reg ip access pub port
|
|
mkdir -p "$BUILD_DIR"
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
|
|
( nssh "$ip" "$cmd" 2>&1 | sed "s/^/[$name] /" ) &
|
|
while [ "$(jobs -r | wc -l)" -ge "$par" ]; do sleep 0.2; done
|
|
done 3< "$NODES_FILE"
|
|
wait
|
|
}
|
|
|
|
# One RPC call on a node through the loopback wRPC JSON endpoint (node/wrpc.py is installed by provision.sh).
|
|
rpc() { local node="$1" method="$2" params="${3:-{\}}"; nssh "$(node_ip "$node")" "python3 /opt/igneum/bin/wrpc.py call $method '$params'"; }
|
|
|
|
results_dir_for() { local d="$RESULTS_DIR/${1:-$(date -u +%Y-%m-%d)}"; mkdir -p "$d"; printf '%s' "$d"; }
|
|
|
|
genesis_bits_decimal() { printf '%d' "$GENESIS_BITS"; }
|
|
|
|
confirm() {
|
|
if [ "${YES:-0}" = 1 ]; then return 0; fi
|
|
printf '%s [type yes]: ' "$1"; read -r a; [ "$a" = yes ] || die "not confirmed"
|
|
}
|