4574890602
A new Hetzner account is capped at 10 primary IPs (IPv4 and IPv6 both count), 20 shared vCPUs, 8 dedicated vCPUs and no Arm, and a network cannot span zones. So: one private network per zone (10.20.<zone>.0/24), the lowest-index node of each zone keeps a public IPv4 and is its gateway (NAT, MSS clamp, one DNAT port 27000+index per private node, persisted as igneum-nat.service), every other node has no public address. nodes.tsv gains access, pub and port columns; lib resolves same-zone vs cross-zone dial addresses and jumps ssh through the gateway for private nodes. All nodes.tsv loops read on fd 3 (a backgrounded ssh drained the file). TYPE_BY_INDEX puts nodes 8 to 11 on ccx13; node 12 is the last shared one the account allows. create.sh prints the plan's cost from the live API. provision.sh install takes node names and skips binaries whose sha256 matches. Binaries copied from the seed's staged v4 build (same sources), no vCPU for a builder. Results 2026-10-04: RTT matrix (hel1-fsn1 35 ms, ash-sin 289 ms) and a 10-minute propagation window of 644 blocks: p50 343 ms, p90 497 ms, p99 666 ms across 12 nodes in 5 locations. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
77 lines
4.4 KiB
Bash
Executable file
77 lines
4.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Runs ON a zone gateway VM (Debian 12, Hetzner) as root. Makes it the zone's NAT router and p2p port forwarder:
|
|
# gateway.sh <private-range> <subnet-router> <public-ip> [<port>:<private-ip> ...]
|
|
# <subnet-router> is Hetzner's router in the zone subnet (10.20.<zone>.1), the next hop of the DHCP-pushed default.
|
|
# - net.ipv4.ip_forward=1 (persisted)
|
|
# - iptables: MASQUERADE for <private-range> out of the public interface, FORWARD accepted both ways, TCP MSS clamped
|
|
# to the path MTU (the private interface is MTU 1450), one DNAT rule per <port>:<private-ip> pair to that node's
|
|
# p2p port; all in IGNEUM-* chains that this script flushes and refills, so it is idempotent
|
|
# - /etc/igneum/nat.sh plus igneum-nat.service replay the rules after a reboot
|
|
# - Hetzner pushes the network's 0.0.0.0/0 route by DHCP (option 121) to every member, the gateway included; a
|
|
# dhclient exit hook drops that default route on the private interface here, so the gateway keeps its own uplink.
|
|
set -euo pipefail
|
|
range="$1"; router="$2"; pubip="$3"; shift 3
|
|
P2P_PORT="${P2P_PORT:-26611}"
|
|
pubif=$(ip -4 route show default | awk '/dev/ { for (i = 1; i <= NF; i++) if ($i == "dev") { print $(i + 1); exit } }' | head -1)
|
|
[ -n "$pubif" ] || pubif=eth0
|
|
privif=$(ip -4 -o addr show | awk -v p="${range%%.*}." '$4 ~ "^" p { print $2; exit }')
|
|
[ -n "$privif" ] || { echo "no interface in $range yet"; exit 1; }
|
|
|
|
command -v iptables >/dev/null 2>&1 || { export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -y -qq iptables >/dev/null; }
|
|
mkdir -p /etc/igneum
|
|
printf 'net.ipv4.ip_forward = 1\n' > /etc/sysctl.d/90-igneum-gateway.conf
|
|
sysctl -q -p /etc/sysctl.d/90-igneum-gateway.conf
|
|
|
|
{
|
|
printf '#!/bin/sh\n# generated by infra/cloud-devnet/net/gateway.sh; replayed by igneum-nat.service\nset -e\n'
|
|
printf 'sysctl -q net.ipv4.ip_forward=1\n'
|
|
for t in nat:PREROUTING:IGNEUM-DNAT nat:POSTROUTING:IGNEUM-SNAT filter:FORWARD:IGNEUM-FWD mangle:FORWARD:IGNEUM-MSS; do
|
|
IFS=: read -r table chain mine <<< "$t"
|
|
printf 'iptables -t %s -N %s 2>/dev/null || true\niptables -t %s -F %s\n' "$table" "$mine" "$table" "$mine"
|
|
printf 'iptables -t %s -C %s -j %s 2>/dev/null || iptables -t %s -I %s -j %s\n' "$table" "$chain" "$mine" "$table" "$chain" "$mine"
|
|
done
|
|
printf 'iptables -t nat -A IGNEUM-SNAT -s %s -o %s -j MASQUERADE\n' "$range" "$pubif"
|
|
printf 'iptables -A IGNEUM-FWD -i %s -s %s -j ACCEPT\n' "$privif" "$range"
|
|
printf 'iptables -A IGNEUM-FWD -o %s -d %s -m conntrack --ctstate RELATED,ESTABLISHED,DNAT -j ACCEPT\n' "$privif" "$range"
|
|
printf 'iptables -t mangle -A IGNEUM-MSS -p tcp --tcp-flags SYN,RST SYN -j TCPMSS --clamp-mss-to-pmtu\n'
|
|
for pair in "$@"; do
|
|
port="${pair%%:*}"; ip="${pair#*:}"
|
|
printf 'iptables -t nat -A IGNEUM-DNAT -d %s -p tcp --dport %s -j DNAT --to-destination %s:%s\n' "$pubip" "$port" "$ip" "$P2P_PORT"
|
|
done
|
|
# the DHCP-pushed default route on the private interface must not win over the uplink
|
|
printf 'while ip -4 route show default dev %s | grep -q .; do ip route del default dev %s; done\n' "$privif" "$privif"
|
|
} > /etc/igneum/nat.sh
|
|
chmod +x /etc/igneum/nat.sh
|
|
|
|
cat > /etc/systemd/system/igneum-nat.service <<UNIT
|
|
[Unit]
|
|
Description=Igneum devnet zone gateway (NAT and p2p port forwarding)
|
|
After=network-online.target
|
|
Wants=network-online.target
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/etc/igneum/nat.sh
|
|
RemainAfterExit=yes
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
UNIT
|
|
systemctl daemon-reload
|
|
systemctl enable igneum-nat >/dev/null 2>&1
|
|
|
|
cat > /etc/dhcp/dhclient-exit-hooks.d/igneum-gateway <<HOOK
|
|
# gateway of the Igneum devnet zone: never take the network's 0.0.0.0/0 route (it points at this machine)
|
|
if [ "\$interface" = "$privif" ]; then
|
|
case "\$reason" in BOUND|RENEW|REBIND|REBOOT) while ip -4 route show default dev $privif | grep -q .; do ip route del default dev $privif; done ;; esac
|
|
fi
|
|
HOOK
|
|
# the gateway is also the ssh jump host of its zone: the scripts open many jumped sessions at once and sshd's
|
|
# default MaxStartups (10:30:100) drops the surplus silently
|
|
printf 'MaxStartups 100:30:200\nMaxSessions 64\n' > /etc/ssh/sshd_config.d/igneum-jump.conf
|
|
systemctl reload ssh 2>/dev/null || systemctl reload sshd 2>/dev/null || true
|
|
/etc/igneum/nat.sh
|
|
systemctl start igneum-nat
|
|
echo "gateway: forward on, $range via $pubif (masquerade), $# port forward(s) on $pubip, private interface $privif (router $router)"
|
|
iptables -t nat -S IGNEUM-DNAT | grep -c DNAT || true
|
|
ip -4 route show default
|