# Shared helpers for infra/cloud-devnet. Source this; it sources config.sh. # Needs bash 3.2 or newer (the Mac's /bin/bash is fine), ssh, scp, python3. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" REPO="$(cd "$HERE/../.." && pwd)" export REPO # shellcheck source=../config.sh . "$HERE/config.sh" # nodes.tsv, one line per node, written by create.sh (7 tab-separated columns): # 1 name 2 index 3 region 4 ip 5 access 6 pub 7 port # ip = the node's private IP (NET_MODE=private) or its public IPv4 (NET_MODE=public): the address same-zone # peers and every script use for the node; ssh resolves it (ssh_target) to a direct or a jumped connection # access = public | private # pub = the public IPv4 that reaches this node from outside its zone: its own (public) or its zone gateway's (private) # port = the TCP port on pub that reaches this node's p2p: 26611 (public) or FWD_PORT_BASE + index (private) # Readers must take all seven fields: `while IFS=$'\t' read -r name idx reg ip access pub port`. NODES_FILE="${NODES_FILE:-$HERE/nodes.tsv}" BUILD_DIR="$HERE/build" BIN_DIR="$BUILD_DIR/bin" RESULTS_DIR="$HERE/results" # Hetzner API token: the only line of ~/.config/igneum/hetzner-token (never printed, never in the repo). if [ -z "${HCLOUD_TOKEN:-}" ] && [ -s "${HETZNER_TOKEN_FILE:-$HOME/.config/igneum/hetzner-token}" ]; then HCLOUD_TOKEN="$(head -1 "${HETZNER_TOKEN_FILE:-$HOME/.config/igneum/hetzner-token}" | tr -d '[:space:]')"; export HCLOUD_TOKEN fi log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; } die() { log "ERROR: $*" >&2; exit 1; } need() { command -v "$1" >/dev/null 2>&1 || die "$1 is not installed ($2)"; } # Hetzner location name for node index i (1-based) and the DigitalOcean one. region_of() { local i="$1" list if [ "$PROVIDER" = digitalocean ]; then list="$DO_REGIONS"; else list="$REGIONS"; fi local n; n=$(printf '%s' "$list" | tr ',' '\n' | grep -c .) printf '%s' "$list" | tr ',' '\n' | sed -n "$(( (i - 1) % n + 1 ))p" } node_name() { printf '%s-%02d' "$PREFIX" "$1"; } # Server type for a Hetzner location: SERVER_TYPE if set, else the TYPE_BY_LOCATION map, else cx23. type_for_location() { if [ -n "$SERVER_TYPE" ]; then printf '%s' "$SERVER_TYPE"; return; fi local t; t=$(printf '%s' "$TYPE_BY_LOCATION" | tr ',' '\n' | awk -F= -v l="$1" '$1 == l { print $2 }') printf '%s' "${t:-cx23}" } # Server type for node index i in region r: a TYPE_BY_INDEX range that contains i, else type_for_location. type_for_index() { local t; t=$(printf '%s' "${TYPE_BY_INDEX:-}" | tr ',' '\n' | awk -F'[-:]' -v i="$1" 'NF == 3 && i >= $1 && i <= $2 { print $3; exit }') if [ -n "$t" ]; then printf '%s' "$t"; else type_for_location "$2"; fi } # ---- zones (Hetzner network zones; a private network lives in exactly one) ---------------------------------------- # zone_of_region hel1 -> eu-central; zone_index eu-central -> 1; zone_net eu-central -> 10.20.1.0/24 zone_of_region() { if [ "$NET_MODE" != private ]; then printf 'public'; return; fi printf '%s' "$ZONES" | tr ';' '\n' | awk -F: -v r="$1" '{ n = split($3, l, " "); for (i = 1; i <= n; i++) if (l[i] == r) { print $1; exit } }' } zone_index() { printf '%s' "$ZONES" | tr ';' '\n' | awk -F: -v z="$1" '$1 == z { print $2; exit }'; } zone_net() { printf '%s.%s.0/24' "$NET_PREFIX" "$(zone_index "$1")"; } zone_hetzner_gw() { printf '%s.%s.1' "$NET_PREFIX" "$(zone_index "$1")"; } # Hetzner's own router in the subnet network_name() { printf '%s-net-%s' "$PREFIX" "$1"; } zones_in_plan() { local i; for i in $(seq 1 "$N"); do zone_of_region "$(region_of "$i")"; done | sort -u; } # the gateway of a zone in the plan: the lowest node index whose region is in that zone gateway_index_for_zone() { local i; for i in $(seq 1 "$N"); do [ "$(zone_of_region "$(region_of "$i")")" = "$1" ] && { printf '%s' "$i"; return; }; done; } zone_of_ip() { printf '%s' "$ZONES" | tr ';' '\n' | awk -F: -v p="$NET_PREFIX" -v ip="$1" '{ if (index(ip, p "." $2 ".") == 1) { print $1; exit } }'; } # ---- nodes.tsv accessors ----------------------------------------------------------------------------------------- require_nodes() { [ -s "$NODES_FILE" ] || die "no $NODES_FILE: run create.sh first"; } node_count() { grep -c . "$NODES_FILE"; } node_field() { awk -F'\t' -v n="$1" -v c="$2" '$1 == n || $2 == n { print $c; exit }' "$NODES_FILE"; } # name-or-index, column node_ip() { node_field "$1" 4; } node_region() { node_field "$1" 3; } node_access() { node_field "$1" 5; } node_pub() { node_field "$1" 6; } node_port() { node_field "$1" 7; } node_zone() { zone_of_region "$(node_region "$1")"; } node_names() { cut -f1 "$NODES_FILE"; } nodes_in_region() { awk -F'\t' -v r="$1" '$3 == r { print $1 }' "$NODES_FILE"; } regions_present() { cut -f3 "$NODES_FILE" | sort -u; } public_nodes() { awk -F'\t' '$5 == "public" { print $1 }' "$NODES_FILE"; } private_nodes() { awk -F'\t' '$5 == "private" { print $1 }' "$NODES_FILE"; } # the gateway (public node) of a zone, from nodes.tsv; its public IP gateway_of_zone() { local n; for n in $(public_nodes); do [ "$(node_zone "$n")" = "$1" ] && { printf '%s' "$n"; return; }; done; } gateway_pub_of_zone() { local g; g=$(gateway_of_zone "$1"); [ -n "$g" ] && node_pub "$g"; } # The address node `from` uses to reach node `to` (name or index): the private IP inside one zone, the public # address (own or gateway) across zones. reach_addr has no port; dial_addr has the p2p port. reach_addr() { if [ "$NET_MODE" != private ] || [ "$(node_zone "$1")" = "$(node_zone "$2")" ]; then node_ip "$2"; else node_pub "$2"; fi } dial_addr() { if [ "$NET_MODE" != private ] || [ "$(node_zone "$1")" = "$(node_zone "$2")" ]; then printf '%s:%s' "$(node_ip "$2")" "$P2P_PORT" else printf '%s:%s' "$(node_pub "$2")" "$(node_port "$2")"; fi } # what the node claims as its own address (--externalip): public nodes their IP, private nodes gateway:port external_addr() { if [ "$(node_access "$1")" = private ]; then printf '%s:%s' "$(node_pub "$1")" "$(node_port "$1")"; else node_pub "$1"; fi } # Outbound peers of node index i: the ring neighbour i+1 and chords at i + k*N/MESH_OUT (1-based, wrapping), # each as the address i can dial. With MESH_OUT=2 that is i+1 and i+N/2; inbound links double it, about 4 peers each. peers_of() { local i="$1" n; n=$(node_count) local k step idx out="" for k in $(seq 1 "$MESH_OUT"); do if [ "$k" = 1 ]; then step=1; else step=$(( (k - 1) * n / MESH_OUT )); fi idx=$(( (i - 1 + step) % n + 1 )) [ "$idx" = "$i" ] && continue out="$out $(dial_addr "$i" "$idx")" done printf '%s' "$out" | sed 's/^ //' | tr ' ' ',' } # ---- ssh ----------------------------------------------------------------------------------------------------------- SSH_OPTS=(-i "$SSH_KEY_FILE" -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="$HERE/build/known_hosts" -o ConnectTimeout=15 -o BatchMode=yes -o ServerAliveInterval=30) # ssh_target prints "" or "\t": a public node's ip (either column) connects # directly to its public IPv4; a private address (the node's or the builder's) jumps through its zone's gateway. ssh_target() { local ip="$1" pub z gw if [ "$NET_MODE" != private ] || [ ! -s "$NODES_FILE" ]; then printf '%s' "$ip"; return; fi pub=$(awk -F'\t' -v ip="$ip" '$5 == "public" && ($4 == ip || $6 == ip) { print $6; exit }' "$NODES_FILE") if [ -n "$pub" ]; then printf '%s' "$pub"; return; fi case "$ip" in "$NET_PREFIX".*) z=$(zone_of_ip "$ip"); gw=$(gateway_pub_of_zone "$z") [ -n "$gw" ] || die "no gateway in nodes.tsv for zone $z (ip $ip)" printf '%s\t%s' "$ip" "$gw" ;; *) printf '%s' "$ip" ;; esac } # ssh option list for a target: adds a ProxyCommand through the jump host when ssh_target gave one. The jump # connection gets the same key and known_hosts (a plain -J would not pass -i along). ssh_opts_for() { local t tab; tab=$' '; t=$(ssh_target "$1"); local host="${t%%"$tab"*}" jump="${t#*"$tab"}" SSH_HOST="$host" if [ "$jump" != "$t" ] && [ -n "$jump" ]; then SSH_EXTRA=(-o "ProxyCommand=ssh -i $SSH_KEY_FILE -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=$HERE/build/known_hosts -o ConnectTimeout=15 -o BatchMode=yes -W %h:%p $SSH_USER@$jump") else SSH_EXTRA=() fi } # One retry when the connection itself failed (ssh exit 255: a dropped jump, a node still booting); the command's # own exit code is passed through untouched. nssh() { local ip="$1" rc; shift; ssh_opts_for "$ip" ssh "${SSH_OPTS[@]}" ${SSH_EXTRA[@]+"${SSH_EXTRA[@]}"} "$SSH_USER@$SSH_HOST" "$@" && return 0; rc=$? [ "$rc" = 255 ] || return "$rc" sleep 3; ssh "${SSH_OPTS[@]}" ${SSH_EXTRA[@]+"${SSH_EXTRA[@]}"} "$SSH_USER@$SSH_HOST" "$@" } # scp with the same resolution: the remote side is whichever argument looks like user@host:path (one host per call) nscp() { local a ip="" args=() for a in "$@"; do case "$a" in "$SSH_USER@"*:*) ip="${a#"$SSH_USER"@}"; ip="${ip%%:*}" ;; esac done [ -n "$ip" ] || die "nscp: no $SSH_USER@host:path argument" ssh_opts_for "$ip" for a in "$@"; do case "$a" in "$SSH_USER@$ip:"*) args+=("$SSH_USER@$SSH_HOST:${a#"$SSH_USER@$ip:"}") ;; *) args+=("$a") ;; esac done scp -q "${SSH_OPTS[@]}" ${SSH_EXTRA[@]+"${SSH_EXTRA[@]}"} "${args[@]}" && return 0 sleep 3; scp -q "${SSH_OPTS[@]}" ${SSH_EXTRA[@]+"${SSH_EXTRA[@]}"} "${args[@]}" } # Run a command on every node in parallel (PAR at a time), prefixing output with the node name. # usage: on_all '' (the command runs on the node through ssh) on_all() { local cmd="$1" par="${PAR:-10}" name idx reg ip access pub port mkdir -p "$BUILD_DIR" while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do ( nssh "$ip" "$cmd" 2>&1 | sed "s/^/[$name] /" ) & while [ "$(jobs -r | wc -l)" -ge "$par" ]; do sleep 0.2; done done 3< "$NODES_FILE" wait } # One RPC call on a node through the loopback wRPC JSON endpoint (node/wrpc.py is installed by provision.sh). rpc() { local node="$1" method="$2" params="${3:-{\}}"; nssh "$(node_ip "$node")" "python3 /opt/igneum/bin/wrpc.py call $method '$params'"; } results_dir_for() { local d="$RESULTS_DIR/${1:-$(date -u +%Y-%m-%d)}"; mkdir -p "$d"; printf '%s' "$d"; } genesis_bits_decimal() { printf '%d' "$GENESIS_BITS"; } confirm() { if [ "${YES:-0}" = 1 ]; then return 0; fi printf '%s [type yes]: ' "$1"; read -r a; [ "$a" = yes ] || die "not confirmed" }