53cb492de3
partition.sh adapted to private-network mode: the cut sits on the region's gateway (INPUT, OUTPUT and FORWARD against the far gateways' public IPs over 26611 and the DNAT ports 27001:27099), since a per-node port-26611 rule leaves the DNAT links up. It now records the locks per side at cut, during, at heal and after convergence, the first lock after the heal from the journals, and the heal time as each minority node's first chain removal of 5+ blocks (the sink-count criterion is tip churn on a healthy network). hop.sh and partition.sh hold the Mac awake with caffeinate; analyze.py gains a 10-s hop series (difficulty, block count, 1- and 2-min rates, threads) and an overshoot table; collect.sh writes hop-series.tsv and hop.md and gzips the journals. Results 2026-10-04: partition 1 (window still filling) reorg 431/496 on the minority, 2 on the majority, healed in 10 and 14 s; partition 2 (locks active): minority locked nothing during the cut, majority locked every interval at 66.8% to 84.5% of total, 0 conflicting locks over 107 indices, healed in 11 and 15 s, first lock after heal 13 s. Hash-rate steps x1.42, x0.70, x0.75, x1.32: difficulty overshoots x1.67, x0.66, x0.53, x1.60, settle 751 s, never in 900 s, 241 s, 646 s. Failures stated in summary.md: the Mac hibernated during the hop (phase 2 ran 94 min), the first partition could not see locks, the script's heal and first-lock figures were artefacts (fixed), and another agent's v2 rollout restarted every node during the second heal. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
232 lines
17 KiB
Bash
Executable file
232 lines
17 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Experiment 1b: cut one region off, heal it, measure the reorg depth and the heal time.
|
|
# ./experiments/partition.sh <region> <minutes> e.g. ./experiments/partition.sh sin 10
|
|
# ./experiments/partition.sh heal remove every partition rule (if a run was interrupted)
|
|
#
|
|
# Cut, NET_MODE=public: on every node of the region, iptables DROP of p2p traffic (port 26611, both directions, both
|
|
# roles) to and from every node outside the region, tagged with a comment so heal finds exactly these rules.
|
|
# Cut, NET_MODE=private (4 Oct 2026): every cross-zone packet of the region passes its zone gateway and carries the far
|
|
# gateways' public IPs, on 26611 (public peers) or a DNAT port 27000+index (private peers; after PREROUTING the
|
|
# forwarded packet is addressed to the private node's 26611). So the rules go on the region's gateway only, in INPUT
|
|
# and OUTPUT (its own igneumd) and in FORWARD (the private nodes behind it, both directions), against each far
|
|
# gateway's public IP over the whole p2p port set 26611 and 27001:27099 as source or destination port. ssh (22),
|
|
# chrony and apt are untouched. A rule per node and port 26611 alone (the public-mode cut) leaves the DNAT links up.
|
|
# The minority keeps mining on its own tips, the majority on theirs (both sides have --enable-unsynced-mining).
|
|
# Heal: the rules are deleted; the nodes reconnect through their --addpeer retries (backoff up to 8 minutes in
|
|
# components/connectionmanager, so a reconnect kick restarts igneumd on the minority side when RECONNECT_KICK=1,
|
|
# default on: a restart reconnects at once and the chain state is on disk).
|
|
# Measured, through the loopback RPC:
|
|
# reorg depth = removedChainBlockHashes of getVirtualChainFromBlock(start = the node's own sink at heal time),
|
|
# per node, after convergence; plus the largest single virtualChainChanged removal in chain.tsv
|
|
# heal time = seconds from heal until all nodes report at most 2 distinct sinks for 3 consecutive 5 s polls
|
|
# locks = getFinalityCheckpoints on both sides at cut, at heal and after convergence: the highest locked
|
|
# index per side, any index locked with two different hashes (a conflicting lock: the gate 3
|
|
# question; the rule's floor predicts none), and the first lock after the heal (LOCK_WAIT seconds,
|
|
# default 600, skipped while the weight window is still filling: no lock is possible before it)
|
|
. "$(dirname "$0")/../lib/common.sh"
|
|
require_nodes
|
|
# A schedule of sleeps pauses when the Mac sleeps (4 Oct 2026: the hop's 15-min 4-thread phase ran 94 min because the
|
|
# laptop hibernated on a flat battery). On macOS re-run under caffeinate -i, which holds the machine awake while this runs.
|
|
if [ "$(uname)" = Darwin ] && [ -z "${IGNEUM_CAFFEINATED:-}" ] && command -v caffeinate >/dev/null 2>&1; then
|
|
IGNEUM_CAFFEINATED=1 exec caffeinate -i "$0" "$@"
|
|
fi
|
|
|
|
heal_all() {
|
|
log "removing partition rules everywhere"
|
|
on_all "iptables -S 2>/dev/null | grep -- '--comment igneum-partition' | sed 's/^-A/-D/' | while read -r r; do iptables \$r; done; true" >/dev/null
|
|
}
|
|
|
|
region="${1:-}"; minutes="${2:-10}"
|
|
[ -n "$region" ] || die "usage: partition.sh <region> <minutes> | partition.sh heal"
|
|
if [ "$region" = heal ]; then heal_all; exit 0; fi
|
|
minority=$(nodes_in_region "$region"); [ -n "$minority" ] || die "no nodes in region $region (present: $(regions_present | tr '\n' ' '))"
|
|
majority=$(node_names | grep -vxF -f <(printf '%s\n' "$minority"))
|
|
# the addresses a minority node exchanges p2p traffic with: private IPs inside its zone, public (gateway) IPs across
|
|
# zones (NET_MODE=private: cross-zone packets carry the gateways' public addresses, never the far private IP)
|
|
outside_ips_for() { local m; for m in $majority; do reach_addr "$1" "$m"; done | sort -u | tr '\n' ' '; }
|
|
stamp=$(date -u +%Y%m%d-%H%M%S)
|
|
out="$(results_dir_for)/partition-$region-$stamp"; mkdir -p "$out"
|
|
log "partition: region $region ($(printf '%s\n' "$minority" | wc -l | tr -d ' ') nodes) cut off for $minutes min; majority $(printf '%s\n' "$majority" | wc -l | tr -d ' ') nodes"
|
|
|
|
snapshot() { # file: one sample line per node, prefixed with the node name
|
|
: > "$1"
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
|
|
( printf '%s\t%s\n' "$name" "$(nssh "$ip" 'python3 /opt/igneum/bin/wrpc.py sample' 2>/dev/null)" >> "$1" ) &
|
|
done 3< "$NODES_FILE"; wait
|
|
}
|
|
checkpoints() { # node file
|
|
nssh "$(node_ip "$1")" "python3 /opt/igneum/bin/wrpc.py call getFinalityCheckpoints '{\"last\": 100}'" > "$2" 2>/dev/null || echo '{}' > "$2"
|
|
}
|
|
|
|
# the highest locked checkpoint index and the window state on one node: "<max locked index or none> <daa> <window filled>"
|
|
lock_state() { # node
|
|
nssh "$(node_ip "$1")" "python3 /opt/igneum/bin/wrpc.py call getFinalityCheckpoints '{\"last\": 400}'" 2>/dev/null </dev/null \
|
|
| python3 -c 'import json,sys
|
|
j=json.load(sys.stdin); cps=j.get("checkpoints") or []
|
|
locked=[int(c["index"]) for c in cps if str(c.get("state","")).lower()=="locked"]
|
|
daa=max([int(c.get("daaScore",0)) for c in cps] or [0])
|
|
print(max(locked) if locked else "none", daa, len(locked))' 2>/dev/null || echo "NA NA NA"
|
|
}
|
|
first_minor=$(printf '%s\n' "$minority" | head -1)
|
|
first_major=$(printf '%s\n' "$majority" | head -1)
|
|
locks_line() { # label: one line per side into locks.tsv
|
|
local m M; m=$(lock_state "$first_minor"); M=$(lock_state "$first_major")
|
|
printf '%s\t%s\tminority\t%s\t%s\n%s\t%s\tmajority\t%s\t%s\n' "$(date -u +%H:%M:%S)" "$1" "$first_minor" "$m" "$(date -u +%H:%M:%S)" "$1" "$first_major" "$M" | tr ' ' '\t' >> "$out/locks.tsv"
|
|
log "locks ($1): minority $first_minor [max locked index, daa, locked count] $m; majority $first_major $M"
|
|
}
|
|
|
|
snapshot "$out/before.tsv"
|
|
printf 'time\tmoment\tside\tnode\tmax_locked_index\tdaa\tlocked_count\n' > "$out/locks.tsv"
|
|
locks_line at-cut
|
|
for n in $minority; do checkpoints "$n" "$out/checkpoints-$n-at-cut.json"; done
|
|
checkpoints "$first_major" "$out/checkpoints-$first_major-at-cut.json"
|
|
t0=$(date +%s)
|
|
P2P_PORTS="$P2P_PORT,$(( FWD_PORT_BASE + 1 )):$(( FWD_PORT_BASE + 99 ))"
|
|
if [ "$NET_MODE" = private ]; then
|
|
# one rule set per zone gateway of the region: INPUT/OUTPUT for the gateway's own node, FORWARD for the nodes behind it
|
|
for z in $(for n in $minority; do node_zone "$n"; done | sort -u); do
|
|
gw=$(gateway_of_zone "$z"); [ -n "$gw" ] || die "no gateway in nodes.tsv for zone $z"
|
|
printf '%s\n' "$minority" | grep -qx "$gw" || die "gateway $gw of zone $z is not in region $region: the cut would also split that zone"
|
|
rules=""
|
|
for ip in $(outside_ips_for "$gw"); do
|
|
for chain in INPUT FORWARD; do
|
|
rules="$rules iptables -I $chain -s $ip -p tcp -m multiport --dports $P2P_PORTS -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I $chain -s $ip -p tcp -m multiport --sports $P2P_PORTS -m comment --comment igneum-partition -j DROP;"
|
|
done
|
|
for chain in OUTPUT FORWARD; do
|
|
rules="$rules iptables -I $chain -d $ip -p tcp -m multiport --dports $P2P_PORTS -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I $chain -d $ip -p tcp -m multiport --sports $P2P_PORTS -m comment --comment igneum-partition -j DROP;"
|
|
done
|
|
done
|
|
nssh "$(node_ip "$gw")" "$rules true" && log "cut zone $z at its gateway $gw (INPUT, OUTPUT, FORWARD; far gateways $(outside_ips_for "$gw"); ports $P2P_PORTS)"
|
|
done
|
|
else
|
|
for n in $minority; do
|
|
rules=""
|
|
for ip in $(outside_ips_for "$n"); do
|
|
rules="$rules iptables -I INPUT -s $ip -p tcp --dport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I INPUT -s $ip -p tcp --sport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I OUTPUT -d $ip -p tcp --dport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I OUTPUT -d $ip -p tcp --sport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
done
|
|
nssh "$(node_ip "$n")" "$rules true" && log "cut $n"
|
|
done
|
|
fi
|
|
printf '%s\tpartition %s cut\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$region" >> "$(results_dir_for)/events.log"
|
|
trap 'log "interrupted: healing"; heal_all' INT TERM
|
|
|
|
: > "$out/during.tsv"
|
|
for i in $(seq 1 $(( minutes * 2 ))); do
|
|
sleep 30
|
|
snapshot "$out/during-$i.tsv"; cat "$out/during-$i.tsv" >> "$out/during.tsv"; rm -f "$out/during-$i.tsv"
|
|
m_sinks=$(for n in $minority; do awk -F'\t' -v n="$n" '$1 == n { print $3 }' "$out/during.tsv" | tail -1; done | sort -u | wc -l | tr -d ' ')
|
|
M_sinks=$(for n in $majority; do awk -F'\t' -v n="$n" '$1 == n { print $3 }' "$out/during.tsv" | tail -1; done | sort -u | wc -l | tr -d ' ')
|
|
log "t+$(( i * 30 )) s: minority distinct sinks $m_sinks, majority distinct sinks $M_sinks"
|
|
[ $(( i % 4 )) = 0 ] && locks_line "cut+$(( i * 30 ))s"
|
|
done
|
|
|
|
log "heal: snapshots and checkpoints on both sides, then rules off"
|
|
snapshot "$out/at-heal.tsv"
|
|
for n in $minority; do checkpoints "$n" "$out/checkpoints-$n-at-heal.json"; done
|
|
checkpoints "$first_major" "$out/checkpoints-$first_major-at-heal.json"
|
|
locks_line at-heal
|
|
heal_all
|
|
t_heal=$(date +%s)
|
|
trap - INT TERM
|
|
if [ "${RECONNECT_KICK:-1}" = 1 ]; then
|
|
for n in $minority; do nssh "$(node_ip "$n")" 'systemctl restart igneumd igneum-blocklog' && log "kicked $n (restart, reconnects at once)"; done
|
|
fi
|
|
printf '%s\tpartition %s healed\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$region" >> "$(results_dir_for)/events.log"
|
|
|
|
log "waiting for convergence (at most 2 distinct sinks for 3 polls, 5 s apart; up to 20 min)"
|
|
ok=0; converged_at=""
|
|
for i in $(seq 1 240); do
|
|
sleep 5
|
|
snapshot "$out/poll.tsv"
|
|
d=$(cut -f3 "$out/poll.tsv" | grep -c . ); u=$(cut -f3 "$out/poll.tsv" | sort -u | grep -c .)
|
|
if [ "$u" -le 2 ] && [ "$d" = "$(node_count)" ]; then ok=$((ok + 1)); else ok=0; fi
|
|
[ $(( i % 6 )) = 0 ] && log "t+$(( i * 5 )) s after heal: $u distinct sinks"
|
|
if [ "$ok" -ge 3 ]; then converged_at=$(( $(date +%s) - t_heal - 10 )); break; fi
|
|
done
|
|
[ -n "$converged_at" ] && log "converged $converged_at s after heal" || log "no convergence within 20 min (see poll.tsv)"
|
|
locks_line converged
|
|
for n in $minority; do checkpoints "$n" "$out/checkpoints-$n-converged.json"; done
|
|
checkpoints "$first_major" "$out/checkpoints-$first_major-converged.json"
|
|
|
|
# the first lock after the heal: poll the majority node until a locked index above the one it held at heal appears
|
|
heal_max=$(awk -F'\t' '$2 == "at-heal" && $3 == "majority" { print $5 }' "$out/locks.tsv" | head -1)
|
|
heal_daa=$(awk -F'\t' '$2 == "at-heal" && $3 == "majority" { print $6 }' "$out/locks.tsv" | head -1)
|
|
first_lock="none within LOCK_WAIT"
|
|
if [ "${heal_daa:-0}" != NA ] && [ "${heal_daa:-0}" -lt "${WEIGHT_WINDOW_DAA:-7200}" ] && [ "${heal_max:-none}" = none ]; then
|
|
first_lock="not possible: weight window still filling (daa $heal_daa of ${WEIGHT_WINDOW_DAA:-7200} at heal)"
|
|
log "first lock after heal: $first_lock"
|
|
else
|
|
log "waiting up to ${LOCK_WAIT:-600} s for the first lock after the heal (majority node $first_major, above index ${heal_max:-none})"
|
|
for i in $(seq 1 $(( ${LOCK_WAIT:-600} / 15 ))); do
|
|
sleep 15
|
|
st=$(lock_state "$first_major"); mx=${st%% *}
|
|
if [ "$mx" != none ] && [ "$mx" != NA ] && { [ "${heal_max:-none}" = none ] || [ "$mx" -gt "$heal_max" ]; }; then
|
|
first_lock="index $mx, $(( $(date +%s) - t_heal )) s after heal (majority $first_major); minority $first_minor: $(lock_state "$first_minor")"
|
|
locks_line first-lock-after-heal; break
|
|
fi
|
|
done
|
|
log "first lock after heal: $first_lock"
|
|
fi
|
|
# the same from the journals, which do not depend on when the poll above started: the first LOCKED line after t_heal
|
|
# on one node per side (seconds after the rules came off), plus certificates the minority adopted from the majority
|
|
for n in $first_major $first_minor; do
|
|
nssh "$(node_ip "$n")" "journalctl -u igneumd --no-pager -o short-iso --since '@$(( t_heal - 2 ))' --until '@$(( t_heal + ${LOCK_WAIT:-600} ))' | grep -E 'Finality: (checkpoint [0-9]+ LOCKED|certificate at index)' | head -60" > "$out/locks-journal-$n.txt" 2>/dev/null </dev/null || true
|
|
l=$(grep -m1 ' LOCKED: ' "$out/locks-journal-$n.txt" | sed -E 's/^([0-9T:-]+)\+0000 .*checkpoint ([0-9]+) LOCKED.*signed ([0-9]+ = [0-9.]+% of active, [0-9.]+% of total).*/\2\t\1\t\3/')
|
|
a=$(grep -c 'certificate at index' "$out/locks-journal-$n.txt" 2>/dev/null || echo 0)
|
|
if [ -n "$l" ]; then
|
|
ts=$(printf '%s' "$l" | cut -f2); secs=$(( $(date -u -j -f %Y-%m-%dT%H:%M:%S "$ts" +%s 2>/dev/null || date -u -d "$ts" +%s) - t_heal ))
|
|
printf '%s\t%s\t%s\t%s\t%s\n' "$n" "$(printf '%s' "$l" | cut -f1)" "$secs" "$(printf '%s' "$l" | cut -f3)" "$a" >> "$out/first-lock.tsv"
|
|
else
|
|
printf '%s\tnone\t-\t-\t%s\n' "$n" "$a" >> "$out/first-lock.tsv"
|
|
fi
|
|
done
|
|
|
|
# the moment each minority node adopted the majority chain: its first virtualChainChanged removal of 5 or more blocks
|
|
# after the rules came off (chain.tsv), with the reconnect and IBD lines from its journal; the sink-count criterion
|
|
# above is noisy (a healthy 12-node network shows 2 to 4 distinct sinks at any instant), this is the heal time
|
|
log "heal time from the minority nodes' chain.tsv and journal"
|
|
: > "$out/heal.tsv"
|
|
for n in $minority; do
|
|
nssh "$(node_ip "$n")" "awk -F'\t' -v t=$(( t_heal * 1000 - 20000 )) '\$1 >= t && \$3 >= 5' /var/log/igneum/chain.tsv; echo; journalctl -u igneumd --no-pager -o short-iso --since '@$(( t_heal - 10 ))' --until '@$(( t_heal + 300 ))' | grep -iE 'connected to|IBD|reorg' | head -40" > "$out/heal-$n.txt" 2>/dev/null </dev/null || true
|
|
first=$(awk -F'\t' -v t=$(( t_heal * 1000 )) 'NF >= 3 && $1 >= t && $3 >= 5 { print $1, $3; exit }' "$out/heal-$n.txt")
|
|
if [ -n "$first" ]; then
|
|
printf '%s\t%s\t%s\n' "$n" "$(( (${first%% *} - t_heal * 1000) / 1000 ))" "${first#* }" >> "$out/heal.tsv"
|
|
else
|
|
printf '%s\tnone\tnone\n' "$n" >> "$out/heal.tsv"
|
|
fi
|
|
done
|
|
|
|
log "reorg depth from each node's own sink at heal (getVirtualChainFromBlock)"
|
|
: > "$out/reorg.tsv"
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
|
|
sink=$(awk -F'\t' -v n="$name" '$1 == n { print $3 }' "$out/at-heal.tsv")
|
|
side=majority; printf '%s\n' "$minority" | grep -qx "$name" && side=minority
|
|
r=$(nssh "$ip" "python3 /opt/igneum/bin/wrpc.py call getVirtualChainFromBlock '{\"startHash\": \"$sink\", \"includeAcceptedTransactionIds\": false}'" 2>/dev/null </dev/null \
|
|
| python3 -c 'import json,sys; j=json.load(sys.stdin); print(len(j.get("removedChainBlockHashes",[])), len(j.get("addedChainBlockHashes",[])))' 2>/dev/null || echo "NA NA")
|
|
chainmax=$(nssh "$ip" "awk -F'\t' -v t=$(( t_heal * 1000 )) '\$1 >= t && \$3 > m { m = \$3 } END { print m + 0 }' /var/log/igneum/chain.tsv" 2>/dev/null </dev/null || echo NA)
|
|
printf '%s\t%s\t%s\t%s\t%s\n' "$name" "$side" "$sink" "$r" "$chainmax" | tr ' ' '\t' >> "$out/reorg.tsv"
|
|
done 3< "$NODES_FILE"
|
|
{
|
|
printf '# Partition %s, %s min, %s\n\n' "$region" "$minutes" "$stamp"
|
|
printf 'cut at %s, healed at %s, converged %s s after heal (criterion: at most 2 distinct sinks for 3 polls)\n\n' "$(date -u -r "$t0" +%H:%M:%S 2>/dev/null || date -u -d @"$t0" +%H:%M:%S)" "$(date -u -r "$t_heal" +%H:%M:%S 2>/dev/null || date -u -d @"$t_heal" +%H:%M:%S)" "${converged_at:-none}"
|
|
printf '| node | side | sink at heal | removed (reorg depth) | added | max single removal after heal |\n|---|---|---|---|---|---|\n'
|
|
awk -F'\t' '{ printf "| %s | %s | %s | %s | %s | %s |\n", $1, $2, substr($3, 1, 12), $4, $5, $6 }' "$out/reorg.tsv"
|
|
printf '\nHeal time (seconds from the rules coming off until the node'"'"'s first chain removal of 5 or more blocks, chain.tsv; journal in heal-<node>.txt):\n\n| node | adopted the majority chain after s | blocks removed |\n|---|---|---|\n'
|
|
awk -F'\t' '{ printf "| %s | %s | %s |\n", $1, $2, $3 }' "$out/heal.tsv"
|
|
printf '\nMinority reorg depth, max: %s. Majority, max: %s.\n' "$(awk -F'\t' '$2 == "minority" && $4 != "NA" && $4 > m { m = $4 } END { print m + 0 }' "$out/reorg.tsv")" "$(awk -F'\t' '$2 == "majority" && $4 != "NA" && $4 > m { m = $4 } END { print m + 0 }' "$out/reorg.tsv")"
|
|
printf '\nConflicting locks at heal (same checkpoint index, different hash, both locked):\n'
|
|
python3 "$HERE/experiments/analyze.py" locks "$out"
|
|
printf '\nLocks per side (highest locked checkpoint index, DAA, locked count on one node per side):\n\n| time | moment | side | node | max locked index | daa | locked count |\n|---|---|---|---|---|---|---|\n'
|
|
awk -F'\t' 'NR > 1 { printf "| %s | %s | %s | %s | %s | %s | %s |\n", $1, $2, $3, $4, $5, $6, $7 }' "$out/locks.tsv"
|
|
printf '\nFirst lock after the heal: %s\n' "$first_lock"
|
|
if [ -s "$out/first-lock.tsv" ]; then
|
|
printf '\nFirst lock after the heal from the journals (locks-journal-<node>.txt):\n\n| node | first locked index after heal | s after heal | signed | certificates adopted from the other side |\n|---|---|---|---|---|\n'
|
|
awk -F'\t' '{ printf "| %s | %s | %s | %s | %s |\n", $1, $2, $3, $4, $5 }' "$out/first-lock.tsv"
|
|
fi
|
|
} | tee "$out/partition.md"
|
|
log "written: $out/partition.md (plus before/during/at-heal/poll/reorg tsv and the checkpoint dumps)"
|