4574890602
A new Hetzner account is capped at 10 primary IPs (IPv4 and IPv6 both count), 20 shared vCPUs, 8 dedicated vCPUs and no Arm, and a network cannot span zones. So: one private network per zone (10.20.<zone>.0/24), the lowest-index node of each zone keeps a public IPv4 and is its gateway (NAT, MSS clamp, one DNAT port 27000+index per private node, persisted as igneum-nat.service), every other node has no public address. nodes.tsv gains access, pub and port columns; lib resolves same-zone vs cross-zone dial addresses and jumps ssh through the gateway for private nodes. All nodes.tsv loops read on fd 3 (a backgrounded ssh drained the file). TYPE_BY_INDEX puts nodes 8 to 11 on ccx13; node 12 is the last shared one the account allows. create.sh prints the plan's cost from the live API. provision.sh install takes node names and skips binaries whose sha256 matches. Binaries copied from the seed's staged v4 build (same sources), no vCPU for a builder. Results 2026-10-04: RTT matrix (hel1-fsn1 35 ms, ash-sin 289 ms) and a 10-minute propagation window of 644 blocks: p50 343 ms, p90 497 ms, p99 666 ms across 12 nodes in 5 locations. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
115 lines
7.9 KiB
Bash
Executable file
115 lines
7.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Experiment 1b: cut one region off, heal it, measure the reorg depth and the heal time.
|
|
# ./experiments/partition.sh <region> <minutes> e.g. ./experiments/partition.sh sin 10
|
|
# ./experiments/partition.sh heal remove every partition rule (if a run was interrupted)
|
|
#
|
|
# Cut: on every node of the region, iptables DROP of p2p traffic (port 26611, both directions, both roles) to and
|
|
# from every node outside the region, tagged with a comment so heal finds exactly these rules. The minority keeps
|
|
# mining on its own tips, the majority on theirs (both sides have --enable-unsynced-mining).
|
|
# Heal: the rules are deleted; the nodes reconnect through their --addpeer retries (backoff up to 8 minutes in
|
|
# components/connectionmanager, so a reconnect kick restarts igneumd on the minority side when RECONNECT_KICK=1,
|
|
# default on: a restart reconnects at once and the chain state is on disk).
|
|
# Measured, through the loopback RPC:
|
|
# reorg depth = removedChainBlockHashes of getVirtualChainFromBlock(start = the node's own sink at heal time),
|
|
# per node, after convergence; plus the largest single virtualChainChanged removal in chain.tsv
|
|
# heal time = seconds from heal until all nodes report at most 2 distinct sinks for 3 consecutive 5 s polls
|
|
# locks = getFinalityCheckpoints on both sides at heal: any index locked with two different hashes is a
|
|
# conflicting lock (the gate 3 question; the rule's floor predicts none)
|
|
. "$(dirname "$0")/../lib/common.sh"
|
|
require_nodes
|
|
|
|
heal_all() {
|
|
log "removing partition rules everywhere"
|
|
on_all "iptables -S 2>/dev/null | grep -- '--comment igneum-partition' | sed 's/^-A/-D/' | while read -r r; do iptables \$r; done; true" >/dev/null
|
|
}
|
|
|
|
region="${1:-}"; minutes="${2:-10}"
|
|
[ -n "$region" ] || die "usage: partition.sh <region> <minutes> | partition.sh heal"
|
|
if [ "$region" = heal ]; then heal_all; exit 0; fi
|
|
minority=$(nodes_in_region "$region"); [ -n "$minority" ] || die "no nodes in region $region (present: $(regions_present | tr '\n' ' '))"
|
|
majority=$(node_names | grep -vxF -f <(printf '%s\n' "$minority"))
|
|
# the addresses a minority node exchanges p2p traffic with: private IPs inside its zone, public (gateway) IPs across
|
|
# zones (NET_MODE=private: cross-zone packets carry the gateways' public addresses, never the far private IP)
|
|
outside_ips_for() { local m; for m in $majority; do reach_addr "$1" "$m"; done | sort -u | tr '\n' ' '; }
|
|
stamp=$(date -u +%Y%m%d-%H%M%S)
|
|
out="$(results_dir_for)/partition-$region-$stamp"; mkdir -p "$out"
|
|
log "partition: region $region ($(printf '%s\n' "$minority" | wc -l | tr -d ' ') nodes) cut off for $minutes min; majority $(printf '%s\n' "$majority" | wc -l | tr -d ' ') nodes"
|
|
|
|
snapshot() { # file: one sample line per node, prefixed with the node name
|
|
: > "$1"
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
|
|
( printf '%s\t%s\n' "$name" "$(nssh "$ip" 'python3 /opt/igneum/bin/wrpc.py sample' 2>/dev/null)" >> "$1" ) &
|
|
done 3< "$NODES_FILE"; wait
|
|
}
|
|
checkpoints() { # node file
|
|
nssh "$(node_ip "$1")" "python3 /opt/igneum/bin/wrpc.py call getFinalityCheckpoints '{\"last\": 100}'" > "$2" 2>/dev/null || echo '{}' > "$2"
|
|
}
|
|
|
|
snapshot "$out/before.tsv"
|
|
t0=$(date +%s)
|
|
for n in $minority; do
|
|
rules=""
|
|
for ip in $(outside_ips_for "$n"); do
|
|
rules="$rules iptables -I INPUT -s $ip -p tcp --dport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I INPUT -s $ip -p tcp --sport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I OUTPUT -d $ip -p tcp --dport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I OUTPUT -d $ip -p tcp --sport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
done
|
|
nssh "$(node_ip "$n")" "$rules true" && log "cut $n"
|
|
done
|
|
printf '%s\tpartition %s cut\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$region" >> "$(results_dir_for)/events.log"
|
|
trap 'log "interrupted: healing"; heal_all' INT TERM
|
|
|
|
: > "$out/during.tsv"
|
|
for i in $(seq 1 $(( minutes * 2 ))); do
|
|
sleep 30
|
|
snapshot "$out/during-$i.tsv"; cat "$out/during-$i.tsv" >> "$out/during.tsv"; rm -f "$out/during-$i.tsv"
|
|
m_sinks=$(for n in $minority; do awk -F'\t' -v n="$n" '$1 == n { print $3 }' "$out/during.tsv" | tail -1; done | sort -u | wc -l | tr -d ' ')
|
|
M_sinks=$(for n in $majority; do awk -F'\t' -v n="$n" '$1 == n { print $3 }' "$out/during.tsv" | tail -1; done | sort -u | wc -l | tr -d ' ')
|
|
log "t+$(( i * 30 )) s: minority distinct sinks $m_sinks, majority distinct sinks $M_sinks"
|
|
done
|
|
|
|
log "heal: snapshots and checkpoints on both sides, then rules off"
|
|
snapshot "$out/at-heal.tsv"
|
|
for n in $minority; do checkpoints "$n" "$out/checkpoints-$n-at-heal.json"; done
|
|
first_major=$(printf '%s\n' "$majority" | head -1); checkpoints "$first_major" "$out/checkpoints-$first_major-at-heal.json"
|
|
heal_all
|
|
t_heal=$(date +%s)
|
|
trap - INT TERM
|
|
if [ "${RECONNECT_KICK:-1}" = 1 ]; then
|
|
for n in $minority; do nssh "$(node_ip "$n")" 'systemctl restart igneumd igneum-blocklog' && log "kicked $n (restart, reconnects at once)"; done
|
|
fi
|
|
printf '%s\tpartition %s healed\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$region" >> "$(results_dir_for)/events.log"
|
|
|
|
log "waiting for convergence (at most 2 distinct sinks for 3 polls, 5 s apart; up to 20 min)"
|
|
ok=0; converged_at=""
|
|
for i in $(seq 1 240); do
|
|
sleep 5
|
|
snapshot "$out/poll.tsv"
|
|
d=$(cut -f3 "$out/poll.tsv" | grep -c . ); u=$(cut -f3 "$out/poll.tsv" | sort -u | grep -c .)
|
|
if [ "$u" -le 2 ] && [ "$d" = "$(node_count)" ]; then ok=$((ok + 1)); else ok=0; fi
|
|
[ $(( i % 6 )) = 0 ] && log "t+$(( i * 5 )) s after heal: $u distinct sinks"
|
|
if [ "$ok" -ge 3 ]; then converged_at=$(( $(date +%s) - t_heal - 10 )); break; fi
|
|
done
|
|
[ -n "$converged_at" ] && log "converged $converged_at s after heal" || log "no convergence within 20 min (see poll.tsv)"
|
|
|
|
log "reorg depth from each node's own sink at heal (getVirtualChainFromBlock)"
|
|
: > "$out/reorg.tsv"
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
|
|
sink=$(awk -F'\t' -v n="$name" '$1 == n { print $3 }' "$out/at-heal.tsv")
|
|
side=majority; printf '%s\n' "$minority" | grep -qx "$name" && side=minority
|
|
r=$(nssh "$ip" "python3 /opt/igneum/bin/wrpc.py call getVirtualChainFromBlock '{\"startHash\": \"$sink\", \"includeAcceptedTransactionIds\": false}'" 2>/dev/null </dev/null \
|
|
| python3 -c 'import json,sys; j=json.load(sys.stdin); print(len(j.get("removedChainBlockHashes",[])), len(j.get("addedChainBlockHashes",[])))' 2>/dev/null || echo "NA NA")
|
|
chainmax=$(nssh "$ip" "awk -F'\t' -v t=$(( t_heal * 1000 )) '\$1 >= t && \$3 > m { m = \$3 } END { print m + 0 }' /var/log/igneum/chain.tsv" 2>/dev/null </dev/null || echo NA)
|
|
printf '%s\t%s\t%s\t%s\t%s\n' "$name" "$side" "$sink" "$r" "$chainmax" | tr ' ' '\t' >> "$out/reorg.tsv"
|
|
done 3< "$NODES_FILE"
|
|
{
|
|
printf '# Partition %s, %s min, %s\n\n' "$region" "$minutes" "$stamp"
|
|
printf 'cut at %s, healed at %s, converged %s s after heal (criterion: at most 2 distinct sinks for 3 polls)\n\n' "$(date -u -r "$t0" +%H:%M:%S 2>/dev/null || date -u -d @"$t0" +%H:%M:%S)" "$(date -u -r "$t_heal" +%H:%M:%S 2>/dev/null || date -u -d @"$t_heal" +%H:%M:%S)" "${converged_at:-none}"
|
|
printf '| node | side | sink at heal | removed (reorg depth) | added | max single removal after heal |\n|---|---|---|---|---|---|\n'
|
|
awk -F'\t' '{ printf "| %s | %s | %s | %s | %s | %s |\n", $1, $2, substr($3, 1, 12), $4, $5, $6 }' "$out/reorg.tsv"
|
|
printf '\nMinority reorg depth, max: %s. Majority, max: %s.\n' "$(awk -F'\t' '$2 == "minority" && $4 != "NA" && $4 > m { m = $4 } END { print m + 0 }' "$out/reorg.tsv")" "$(awk -F'\t' '$2 == "majority" && $4 != "NA" && $4 > m { m = $4 } END { print m + 0 }' "$out/reorg.tsv")"
|
|
printf '\nConflicting locks at heal (same checkpoint index, different hash, both locked):\n'
|
|
python3 "$HERE/experiments/analyze.py" locks "$out"
|
|
} | tee "$out/partition.md"
|
|
log "written: $out/partition.md (plus before/during/at-heal/poll/reorg tsv and the checkpoint dumps)"
|