0b713429ef
A new Hetzner account is capped at 10 primary IPs (IPv4 and IPv6 both count), 20 shared vCPUs, 8 dedicated vCPUs and no Arm, and a network cannot span zones. So: one private network per zone (10.20.<zone>.0/24), the lowest-index node of each zone keeps a public IPv4 and is its gateway (NAT, MSS clamp, one DNAT port 27000+index per private node, persisted as igneum-nat.service), every other node has no public address. nodes.tsv gains access, pub and port columns; lib resolves same-zone vs cross-zone dial addresses and jumps ssh through the gateway for private nodes. All nodes.tsv loops read on fd 3 (a backgrounded ssh drained the file). TYPE_BY_INDEX puts nodes 8 to 11 on ccx13; node 12 is the last shared one the account allows. create.sh prints the plan's cost from the live API. provision.sh install takes node names and skips binaries whose sha256 matches. Binaries copied from the seed's staged v4 build (same sources), no vCPU for a builder. Results 2026-10-04: RTT matrix (hel1-fsn1 35 ms, ash-sin 289 ms) and a 10-minute propagation window of 644 blocks: p50 343 ms, p90 497 ms, p99 666 ms across 12 nodes in 5 locations. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
39 lines
2.6 KiB
Bash
Executable file
39 lines
2.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Experiment 1a: inter-region RTT and block propagation delay.
|
|
# ./experiments/latency.sh [minutes] default 10
|
|
# 1. RTT: every node pings every other node (5 pings, 0.2 s apart) in parallel; results/<date>/latency/rtt.tsv and a
|
|
# region-pair table.
|
|
# 2. Propagation: the block logs run anyway (igneum-blocklog); this waits `minutes`, pulls blocks.tsv from every node
|
|
# and joins on the block hash: first arrival anywhere = t0 of that block, per node delay = arrival - t0.
|
|
# Clocks: chrony on every VM; the per-node offset is recorded (chronyc tracking) and the join is only as good as it.
|
|
. "$(dirname "$0")/../lib/common.sh"
|
|
require_nodes
|
|
minutes="${1:-10}"
|
|
out="$(results_dir_for)/latency"; mkdir -p "$out/nodes"
|
|
t_start=$(date +%s)000
|
|
|
|
# Each node pings the address it can reach for every other node (reach_addr: the private IP inside its zone, the
|
|
# public IP, or the zone gateway's, across zones); the row names the target by its nodes.tsv ip so analyze.py joins.
|
|
log "RTT matrix: $(node_count) nodes x $(( $(node_count) - 1 )) targets, 5 pings each"
|
|
: > "$out/rtt.tsv"
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
|
|
pairs=$(while IFS=$'\t' read -r n2 i2 r2 ip2 a2 p2 po2; do [ "$n2" = "$name" ] && continue; printf '%s=%s ' "$ip2" "$(reach_addr "$idx" "$i2")"; done < "$NODES_FILE")
|
|
(
|
|
nssh "$ip" "for pr in $pairs; do id=\${pr%%=*}; t=\${pr##*=}; r=\$(ping -c 5 -i 0.2 -W 2 \$t 2>/dev/null | awk -F'/' '/rtt|round-trip/ { print \$4\"\t\"\$5\"\t\"\$6 }'); printf '%s\t%s\t%s\n' '$name' \"\$id\" \"\${r:-NA\tNA\tNA}\"; done" 2>/dev/null
|
|
) >> "$out/rtt.tsv" &
|
|
done 3< "$NODES_FILE"
|
|
wait
|
|
log "rtt.tsv: $(grep -c . "$out/rtt.tsv") pairs. Region-pair medians (avg RTT ms):"
|
|
python3 "$HERE/experiments/analyze.py" rtt "$out" "$NODES_FILE" | tee "$out/rtt-by-region.md"
|
|
|
|
log "propagation window: $minutes min from $(date -u +%H:%M:%S) (the miners must be running: ./start.sh miners)"
|
|
sleep $(( minutes * 60 ))
|
|
t_end=$(date +%s)000
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
|
|
mkdir -p "$out/nodes/$name"
|
|
nscp "$SSH_USER@$ip:/var/log/igneum/blocks.tsv" "$out/nodes/$name/blocks.tsv" 2>/dev/null </dev/null || log "no blocks.tsv on $name"
|
|
nssh "$ip" "chronyc tracking 2>/dev/null | grep -E 'System time|RMS offset'" > "$out/nodes/$name/chrony.txt" 2>/dev/null </dev/null || true
|
|
done 3< "$NODES_FILE"
|
|
python3 "$HERE/experiments/analyze.py" propagation "$out" "$NODES_FILE" "$t_start" "$t_end" | tee "$out/propagation.md"
|
|
printf '%s\tlatency %s min, window %s..%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$minutes" "$t_start" "$t_end" >> "$(results_dir_for)/events.log"
|
|
log "written: $out/rtt.tsv, rtt-by-region.md, propagation.md"
|