6b5bd92b84
A new Hetzner account is capped at 10 primary IPs (IPv4 and IPv6 both count), 20 shared vCPUs, 8 dedicated vCPUs and no Arm, and a network cannot span zones. So: one private network per zone (10.20.<zone>.0/24), the lowest-index node of each zone keeps a public IPv4 and is its gateway (NAT, MSS clamp, one DNAT port 27000+index per private node, persisted as igneum-nat.service), every other node has no public address. nodes.tsv gains access, pub and port columns; lib resolves same-zone vs cross-zone dial addresses and jumps ssh through the gateway for private nodes. All nodes.tsv loops read on fd 3 (a backgrounded ssh drained the file). TYPE_BY_INDEX puts nodes 8 to 11 on ccx13; node 12 is the last shared one the account allows. create.sh prints the plan's cost from the live API. provision.sh install takes node names and skips binaries whose sha256 matches. Binaries copied from the seed's staged v4 build (same sources), no vCPU for a builder. Results 2026-10-04: RTT matrix (hel1-fsn1 35 ms, ash-sin 289 ms) and a 10-minute propagation window of 644 blocks: p50 343 ms, p90 497 ms, p99 666 ms across 12 nodes in 5 locations. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
98 lines
6 KiB
Bash
Executable file
98 lines
6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Build the node once and install it on every node.
|
|
#
|
|
# ./provision.sh source tarball -> builder VM -> build/bin/{igneumd,igneum-miner} -> every node (systemd units)
|
|
# ./provision.sh build only the build (creates the builder VM if build/builder.ip is missing, deletes it afterwards)
|
|
# ./provision.sh install only the install on the nodes (needs build/bin/igneumd)
|
|
# ./provision.sh install igneum-09 ... the install on the named nodes only (the mesh is still computed from all of nodes.tsv)
|
|
# The binaries are copied only when the node's sha256 differs from build/bin (a re-install moves no 47 MB twice).
|
|
# BIN_SOURCE=mac ./provision.sh [build|all] no builder VM: cross-compile on this Mac (infra/cross/build-linux.sh,
|
|
# cargo-zigbuild, x86_64-unknown-linux-gnu glibc 2.36) and take build/bin from its output
|
|
#
|
|
# Two ways to get the binaries. BIN_SOURCE=builder (the default, 3 Oct 2026): a builder VM compiles the source
|
|
# tarball (10 to 25 minutes on 8 vCPU, about EUR 0.05). BIN_SOURCE=mac (4 Oct 2026): cargo-zigbuild on this Mac
|
|
# cross-compiles for x86_64 Linux glibc 2.36 (zig is the C/C++ toolchain and linker; rocksdb, blst, secp256k1 and the
|
|
# execution layer link; infra/cross/build-linux.sh), no VM at all. The binaries are cached in build/bin; REBUILD=1
|
|
# forces a new build.
|
|
# NET_MODE=private (4 Oct 2026): the builder has no public address either; it sits behind the zone gateway of
|
|
# region 1 and lib's nssh/nscp jump through that gateway (create.sh builder does the uplink check).
|
|
|
|
. "$(dirname "$0")/lib/common.sh"
|
|
|
|
what="${1:-all}"
|
|
mkdir -p "$BUILD_DIR" "$BIN_DIR"
|
|
|
|
# ---- build ---------------------------------------------------------------------------------------------------------
|
|
do_build() {
|
|
if [ -x "$BIN_DIR/igneumd" ] && [ "${REBUILD:-0}" != 1 ]; then
|
|
log "build/bin/igneumd exists ($(cat "$BIN_DIR/version.txt" 2>/dev/null | tr '\n' ' ')); REBUILD=1 to rebuild"
|
|
return
|
|
fi
|
|
if [ "${BIN_SOURCE:-builder}" = mac ]; then
|
|
log "BIN_SOURCE=mac: cross-compiling on this Mac (no builder VM) from $NODE_SRC"
|
|
NODE_SRC="$NODE_SRC" OUT_DIR="$BIN_DIR" "$HERE/../cross/build-linux.sh"
|
|
printf 'exported %s\nigneum-node (%s): HEAD %s (%s), cross-compiled on the Mac\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$NODE_SRC" "$(git -C "$NODE_SRC" rev-parse --short HEAD)" "$(git -C "$NODE_SRC" rev-parse --abbrev-ref HEAD)" > "$BIN_DIR/src.stamp"
|
|
log "binaries in $BIN_DIR: $(cat "$BIN_DIR/version.txt" | tr '\n' ' ')"
|
|
return
|
|
fi
|
|
"$HERE/make-source.sh"
|
|
if [ ! -s "$BUILD_DIR/builder.ip" ]; then
|
|
YES="${YES:-0}" "$HERE/create.sh" builder
|
|
fi
|
|
bip=$(cat "$BUILD_DIR/builder.ip")
|
|
log "builder at $bip; waiting for ssh"
|
|
for try in $(seq 1 12); do nssh "$bip" true >/dev/null 2>&1 && break; sleep 10; done
|
|
nssh "$bip" true || die "builder $bip not reachable over ssh"
|
|
log "uploading source ($(du -h "$BUILD_DIR/src.tar.gz" | cut -f1)) and the build script"
|
|
nscp "$BUILD_DIR/src.tar.gz" "$HERE/builder/build-on-builder.sh" "$SSH_USER@$bip:/root/"
|
|
nssh "$bip" "bash /root/build-on-builder.sh" | tee "$BUILD_DIR/build.log"
|
|
nscp "$SSH_USER@$bip:/root/out/igneumd" "$SSH_USER@$bip:/root/out/igneum-miner" "$SSH_USER@$bip:/root/out/version.txt" "$BIN_DIR/"
|
|
chmod +x "$BIN_DIR/igneumd" "$BIN_DIR/igneum-miner"
|
|
cp "$BUILD_DIR/src.stamp" "$BIN_DIR/src.stamp"
|
|
log "binaries in $BIN_DIR: $(cat "$BIN_DIR/version.txt" | tr '\n' ' ')"
|
|
if [ "${KEEP_BUILDER:-0}" = 1 ]; then
|
|
log "KEEP_BUILDER=1: builder $bip stays up (it bills by the hour)"
|
|
else
|
|
log "deleting the builder VM"
|
|
if [ "$PROVIDER" = digitalocean ]; then doctl compute droplet delete -f "$PREFIX-builder"; else hcloud server delete "$PREFIX-builder" >/dev/null; fi
|
|
rm -f "$BUILD_DIR/builder.ip"
|
|
fi
|
|
}
|
|
|
|
# ---- install ------------------------------------------------------------------------------------------------------
|
|
do_install() { # [node names]
|
|
require_nodes
|
|
[ -x "$BIN_DIR/igneumd" ] || die "no $BIN_DIR/igneumd: run ./provision.sh build"
|
|
local n; n=$(node_count); local bits; bits=$(genesis_bits_decimal); local only="$*"
|
|
local sha_d sha_m; sha_d=$(shasum -a 256 "$BIN_DIR/igneumd" | cut -c1-64); sha_m=$(shasum -a 256 "$BIN_DIR/igneum-miner" | cut -c1-64)
|
|
log "installing on ${only:-all $n} nodes (devnet suffix $DEVNET_SUFFIX, genesis bits $GENESIS_BITS = $bits, $MESH_OUT outbound peers each, $MINER_THREADS miner thread)"
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
|
|
if [ -n "$only" ] && ! printf ' %s ' $only | grep -q " $name "; then continue; fi
|
|
peers=$(peers_of "$idx")
|
|
(
|
|
for try in 1 2 3; do nssh "$ip" "mkdir -p /opt/igneum/bin /etc/igneum" && break; sleep 10; done
|
|
# the binaries only when they differ (sha256 on the node), the small files always
|
|
have=$(nssh "$ip" "cd /opt/igneum/bin 2>/dev/null && sha256sum igneumd igneum-miner 2>/dev/null | cut -c1-64 | tr '\n' ' '" </dev/null || true)
|
|
case "$have" in
|
|
"$sha_d $sha_m ") ;;
|
|
*) nscp "$BIN_DIR/igneumd" "$BIN_DIR/igneum-miner" "$SSH_USER@$ip:/opt/igneum/bin/" ;;
|
|
esac
|
|
nscp "$HERE/node/wrpc.py" "$HERE/node/run-igneumd.sh" "$HERE/node/blocklog.sh" "$SSH_USER@$ip:/opt/igneum/bin/"
|
|
nscp "$HERE/node/install-node.sh" "$HERE/node/igneumd.service" "$HERE/node/igneum-miner.service" "$HERE/node/igneum-blocklog.service" "$SSH_USER@$ip:/root/"
|
|
nssh "$ip" "bash /root/install-node.sh '$name' '$idx' '$DEVNET_SUFFIX' '$bits' '$peers' '$(external_addr "$idx")' '$MINER_THREADS' '$MINER_STATUS_SECS'" 2>&1 | sed "s/^/[$name] /"
|
|
) &
|
|
# at most 10 installs at once
|
|
while [ "$(jobs -r | wc -l)" -ge 10 ]; do sleep 1; done
|
|
done 3< "$NODES_FILE"
|
|
wait
|
|
log "installed. Peer map:"
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do printf ' %s (%s, %s, %s) -> %s\n' "$name" "$reg" "$ip" "$(external_addr "$idx")" "$(peers_of "$idx")"; done 3< "$NODES_FILE"
|
|
log "next: ./start.sh"
|
|
}
|
|
|
|
case "$what" in
|
|
all) do_build; do_install ;;
|
|
build) do_build ;;
|
|
install) shift; do_install "$@" ;;
|
|
*) die "usage: provision.sh [all|build|install]" ;;
|
|
esac
|