d8dc8c6ba3
infra/cloud-devnet: hcloud (doctl variant) create, builder-VM provision from a git-archive source tarball, systemd units for igneumd --devnet-suffix with a sparse --addpeer mesh and a CPU trickle miner per node, stdlib wRPC client, experiments (latency, partition, hop, collect, observer hookup), README with the command sequence and the Hetzner API prices of 3 Oct 2026. infra/gpu-bench: RunPod image recipes (CUDA 12.8, ROCm), bundle, run.sh (vectors gate, 10-min raw, sweep, inline shortcut ratio, nvcc/NVRTC/OpenCL recompile timings, results row, intake upload), bench-log template. infra/seed-nodes: create-seed (persistent IPv4, firewall), provision on the VM, health check, addPeer from the Mac over grpcurl, seeds.txt; igneum-seed-1 created at 188.245.5.161 (Hetzner cx23, fsn1). docs/plans/cloud-devnet.md and docs/plans/seed-nodes.md. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
113 lines
7.6 KiB
Bash
Executable file
113 lines
7.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Experiment 1b: cut one region off, heal it, measure the reorg depth and the heal time.
|
|
# ./experiments/partition.sh <region> <minutes> e.g. ./experiments/partition.sh sin 10
|
|
# ./experiments/partition.sh heal remove every partition rule (if a run was interrupted)
|
|
#
|
|
# Cut: on every node of the region, iptables DROP of p2p traffic (port 26611, both directions, both roles) to and
|
|
# from every node outside the region, tagged with a comment so heal finds exactly these rules. The minority keeps
|
|
# mining on its own tips, the majority on theirs (both sides have --enable-unsynced-mining).
|
|
# Heal: the rules are deleted; the nodes reconnect through their --addpeer retries (backoff up to 8 minutes in
|
|
# components/connectionmanager, so a reconnect kick restarts igneumd on the minority side when RECONNECT_KICK=1,
|
|
# default on: a restart reconnects at once and the chain state is on disk).
|
|
# Measured, through the loopback RPC:
|
|
# reorg depth = removedChainBlockHashes of getVirtualChainFromBlock(start = the node's own sink at heal time),
|
|
# per node, after convergence; plus the largest single virtualChainChanged removal in chain.tsv
|
|
# heal time = seconds from heal until all nodes report at most 2 distinct sinks for 3 consecutive 5 s polls
|
|
# locks = getFinalityCheckpoints on both sides at heal: any index locked with two different hashes is a
|
|
# conflicting lock (the gate 3 question; the rule's floor predicts none)
|
|
. "$(dirname "$0")/../lib/common.sh"
|
|
require_nodes
|
|
|
|
heal_all() {
|
|
log "removing partition rules everywhere"
|
|
on_all "iptables -S 2>/dev/null | grep -- '--comment igneum-partition' | sed 's/^-A/-D/' | while read -r r; do iptables \$r; done; true" >/dev/null
|
|
}
|
|
|
|
region="${1:-}"; minutes="${2:-10}"
|
|
[ -n "$region" ] || die "usage: partition.sh <region> <minutes> | partition.sh heal"
|
|
if [ "$region" = heal ]; then heal_all; exit 0; fi
|
|
minority=$(nodes_in_region "$region"); [ -n "$minority" ] || die "no nodes in region $region (present: $(regions_present | tr '\n' ' '))"
|
|
majority=$(node_names | grep -vxF -f <(printf '%s\n' "$minority"))
|
|
outside_ips=$(for n in $majority; do node_ip "$n"; done | tr '\n' ' ')
|
|
stamp=$(date -u +%Y%m%d-%H%M%S)
|
|
out="$(results_dir_for)/partition-$region-$stamp"; mkdir -p "$out"
|
|
log "partition: region $region ($(printf '%s\n' "$minority" | wc -l | tr -d ' ') nodes) cut off for $minutes min; majority $(printf '%s\n' "$majority" | wc -l | tr -d ' ') nodes"
|
|
|
|
snapshot() { # file: one sample line per node, prefixed with the node name
|
|
: > "$1"
|
|
while IFS=$'\t' read -r name idx reg ip; do
|
|
( printf '%s\t%s\n' "$name" "$(nssh "$ip" 'python3 /opt/igneum/bin/wrpc.py sample' 2>/dev/null)" >> "$1" ) &
|
|
done < "$NODES_FILE"; wait
|
|
}
|
|
checkpoints() { # node file
|
|
nssh "$(node_ip "$1")" "python3 /opt/igneum/bin/wrpc.py call getFinalityCheckpoints '{\"last\": 100}'" > "$2" 2>/dev/null || echo '{}' > "$2"
|
|
}
|
|
|
|
snapshot "$out/before.tsv"
|
|
t0=$(date +%s)
|
|
for n in $minority; do
|
|
rules=""
|
|
for ip in $outside_ips; do
|
|
rules="$rules iptables -I INPUT -s $ip -p tcp --dport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I INPUT -s $ip -p tcp --sport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I OUTPUT -d $ip -p tcp --dport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
rules="$rules iptables -I OUTPUT -d $ip -p tcp --sport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
|
|
done
|
|
nssh "$(node_ip "$n")" "$rules true" && log "cut $n"
|
|
done
|
|
printf '%s\tpartition %s cut\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$region" >> "$(results_dir_for)/events.log"
|
|
trap 'log "interrupted: healing"; heal_all' INT TERM
|
|
|
|
: > "$out/during.tsv"
|
|
for i in $(seq 1 $(( minutes * 2 ))); do
|
|
sleep 30
|
|
snapshot "$out/during-$i.tsv"; cat "$out/during-$i.tsv" >> "$out/during.tsv"; rm -f "$out/during-$i.tsv"
|
|
m_sinks=$(for n in $minority; do awk -F'\t' -v n="$n" '$1 == n { print $3 }' "$out/during.tsv" | tail -1; done | sort -u | wc -l | tr -d ' ')
|
|
M_sinks=$(for n in $majority; do awk -F'\t' -v n="$n" '$1 == n { print $3 }' "$out/during.tsv" | tail -1; done | sort -u | wc -l | tr -d ' ')
|
|
log "t+$(( i * 30 )) s: minority distinct sinks $m_sinks, majority distinct sinks $M_sinks"
|
|
done
|
|
|
|
log "heal: snapshots and checkpoints on both sides, then rules off"
|
|
snapshot "$out/at-heal.tsv"
|
|
for n in $minority; do checkpoints "$n" "$out/checkpoints-$n-at-heal.json"; done
|
|
first_major=$(printf '%s\n' "$majority" | head -1); checkpoints "$first_major" "$out/checkpoints-$first_major-at-heal.json"
|
|
heal_all
|
|
t_heal=$(date +%s)
|
|
trap - INT TERM
|
|
if [ "${RECONNECT_KICK:-1}" = 1 ]; then
|
|
for n in $minority; do nssh "$(node_ip "$n")" 'systemctl restart igneumd igneum-blocklog' && log "kicked $n (restart, reconnects at once)"; done
|
|
fi
|
|
printf '%s\tpartition %s healed\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$region" >> "$(results_dir_for)/events.log"
|
|
|
|
log "waiting for convergence (at most 2 distinct sinks for 3 polls, 5 s apart; up to 20 min)"
|
|
ok=0; converged_at=""
|
|
for i in $(seq 1 240); do
|
|
sleep 5
|
|
snapshot "$out/poll.tsv"
|
|
d=$(cut -f3 "$out/poll.tsv" | grep -c . ); u=$(cut -f3 "$out/poll.tsv" | sort -u | grep -c .)
|
|
if [ "$u" -le 2 ] && [ "$d" = "$(node_count)" ]; then ok=$((ok + 1)); else ok=0; fi
|
|
[ $(( i % 6 )) = 0 ] && log "t+$(( i * 5 )) s after heal: $u distinct sinks"
|
|
if [ "$ok" -ge 3 ]; then converged_at=$(( $(date +%s) - t_heal - 10 )); break; fi
|
|
done
|
|
[ -n "$converged_at" ] && log "converged $converged_at s after heal" || log "no convergence within 20 min (see poll.tsv)"
|
|
|
|
log "reorg depth from each node's own sink at heal (getVirtualChainFromBlock)"
|
|
: > "$out/reorg.tsv"
|
|
while IFS=$'\t' read -r name idx reg ip; do
|
|
sink=$(awk -F'\t' -v n="$name" '$1 == n { print $3 }' "$out/at-heal.tsv")
|
|
side=majority; printf '%s\n' "$minority" | grep -qx "$name" && side=minority
|
|
r=$(nssh "$ip" "python3 /opt/igneum/bin/wrpc.py call getVirtualChainFromBlock '{\"startHash\": \"$sink\", \"includeAcceptedTransactionIds\": false}'" 2>/dev/null \
|
|
| python3 -c 'import json,sys; j=json.load(sys.stdin); print(len(j.get("removedChainBlockHashes",[])), len(j.get("addedChainBlockHashes",[])))' 2>/dev/null || echo "NA NA")
|
|
chainmax=$(nssh "$ip" "awk -F'\t' -v t=$(( t_heal * 1000 )) '\$1 >= t && \$3 > m { m = \$3 } END { print m + 0 }' /var/log/igneum/chain.tsv" 2>/dev/null || echo NA)
|
|
printf '%s\t%s\t%s\t%s\t%s\n' "$name" "$side" "$sink" "$r" "$chainmax" | tr ' ' '\t' >> "$out/reorg.tsv"
|
|
done < "$NODES_FILE"
|
|
{
|
|
printf '# Partition %s, %s min, %s\n\n' "$region" "$minutes" "$stamp"
|
|
printf 'cut at %s, healed at %s, converged %s s after heal (criterion: at most 2 distinct sinks for 3 polls)\n\n' "$(date -u -r "$t0" +%H:%M:%S 2>/dev/null || date -u -d @"$t0" +%H:%M:%S)" "$(date -u -r "$t_heal" +%H:%M:%S 2>/dev/null || date -u -d @"$t_heal" +%H:%M:%S)" "${converged_at:-none}"
|
|
printf '| node | side | sink at heal | removed (reorg depth) | added | max single removal after heal |\n|---|---|---|---|---|---|\n'
|
|
awk -F'\t' '{ printf "| %s | %s | %s | %s | %s | %s |\n", $1, $2, substr($3, 1, 12), $4, $5, $6 }' "$out/reorg.tsv"
|
|
printf '\nMinority reorg depth, max: %s. Majority, max: %s.\n' "$(awk -F'\t' '$2 == "minority" && $4 != "NA" && $4 > m { m = $4 } END { print m + 0 }' "$out/reorg.tsv")" "$(awk -F'\t' '$2 == "majority" && $4 != "NA" && $4 > m { m = $4 } END { print m + 0 }' "$out/reorg.tsv")"
|
|
printf '\nConflicting locks at heal (same checkpoint index, different hash, both locked):\n'
|
|
python3 "$HERE/experiments/analyze.py" locks "$out"
|
|
} | tee "$out/partition.md"
|
|
log "written: $out/partition.md (plus before/during/at-heal/poll/reorg tsv and the checkpoint dumps)"
|