#!/usr/bin/env bash
# Experiment 1b: cut one region off, heal it, measure the reorg depth and the heal time.
#   ./experiments/partition.sh <region> <minutes>      e.g. ./experiments/partition.sh sin 10
#   ./experiments/partition.sh heal                    remove every partition rule (if a run was interrupted)
#
# Cut, NET_MODE=public: on every node of the region, iptables DROP of p2p traffic (port 26611, both directions, both
# roles) to and from every node outside the region, tagged with a comment so heal finds exactly these rules.
# Cut, NET_MODE=private (4 Oct 2026): every cross-zone packet of the region passes its zone gateway and carries the far
# gateways' public IPs, on 26611 (public peers) or a DNAT port 27000+index (private peers; after PREROUTING the
# forwarded packet is addressed to the private node's 26611). So the rules go on the region's gateway only, in INPUT
# and OUTPUT (its own igneumd) and in FORWARD (the private nodes behind it, both directions), against each far
# gateway's public IP over the whole p2p port set 26611 and 27001:27099 as source or destination port. ssh (22),
# chrony and apt are untouched. A rule per node and port 26611 alone (the public-mode cut) leaves the DNAT links up.
# The minority keeps mining on its own tips, the majority on theirs (both sides have --enable-unsynced-mining).
# Heal: the rules are deleted; the nodes reconnect through their --addpeer retries (backoff up to 8 minutes in
# components/connectionmanager, so a reconnect kick restarts igneumd on the minority side when RECONNECT_KICK=1,
# default on: a restart reconnects at once and the chain state is on disk).
# Measured, through the loopback RPC:
#   reorg depth  = removedChainBlockHashes of getVirtualChainFromBlock(start = the node's own sink at heal time),
#                  per node, after convergence; plus the largest single virtualChainChanged removal in chain.tsv
#   heal time    = seconds from heal until all nodes report at most 2 distinct sinks for 3 consecutive 5 s polls
#   locks        = getFinalityCheckpoints on both sides at cut, at heal and after convergence: the highest locked
#                  index per side, any index locked with two different hashes (a conflicting lock: the gate 3
#                  question; the rule's floor predicts none), and the first lock after the heal (LOCK_WAIT seconds,
#                  default 600, skipped while the weight window is still filling: no lock is possible before it)
. "$(dirname "$0")/../lib/common.sh"
require_nodes
# A schedule of sleeps pauses when the Mac sleeps (4 Oct 2026: the hop's 15-min 4-thread phase ran 94 min because the
# laptop hibernated on a flat battery). On macOS re-run under caffeinate -i, which holds the machine awake while this runs.
if [ "$(uname)" = Darwin ] && [ -z "${IGNEUM_CAFFEINATED:-}" ] && command -v caffeinate >/dev/null 2>&1; then
  IGNEUM_CAFFEINATED=1 exec caffeinate -i "$0" "$@"
fi

heal_all() {
  log "removing partition rules everywhere"
  on_all "iptables -S 2>/dev/null | grep -- '--comment igneum-partition' | sed 's/^-A/-D/' | while read -r r; do iptables \$r; done; true" >/dev/null
}

region="${1:-}"; minutes="${2:-10}"
[ -n "$region" ] || die "usage: partition.sh <region> <minutes> | partition.sh heal"
if [ "$region" = heal ]; then heal_all; exit 0; fi
minority=$(nodes_in_region "$region"); [ -n "$minority" ] || die "no nodes in region $region (present: $(regions_present | tr '\n' ' '))"
majority=$(node_names | grep -vxF -f <(printf '%s\n' "$minority"))
# the addresses a minority node exchanges p2p traffic with: private IPs inside its zone, public (gateway) IPs across
# zones (NET_MODE=private: cross-zone packets carry the gateways' public addresses, never the far private IP)
outside_ips_for() { local m; for m in $majority; do reach_addr "$1" "$m"; done | sort -u | tr '\n' ' '; }
stamp=$(date -u +%Y%m%d-%H%M%S)
out="$(results_dir_for)/partition-$region-$stamp"; mkdir -p "$out"
log "partition: region $region ($(printf '%s\n' "$minority" | wc -l | tr -d ' ') nodes) cut off for $minutes min; majority $(printf '%s\n' "$majority" | wc -l | tr -d ' ') nodes"

snapshot() {   # file: one sample line per node, prefixed with the node name
  : > "$1"
  while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
    ( printf '%s\t%s\n' "$name" "$(nssh "$ip" 'python3 /opt/igneum/bin/wrpc.py sample' 2>/dev/null)" >> "$1" ) &
  done 3< "$NODES_FILE"; wait
}
checkpoints() {   # node file
  nssh "$(node_ip "$1")" "python3 /opt/igneum/bin/wrpc.py call getFinalityCheckpoints '{\"last\": 100}'" > "$2" 2>/dev/null || echo '{}' > "$2"
}

# the highest locked checkpoint index and the window state on one node: "<max locked index or none> <daa> <window filled>"
lock_state() {   # node
  nssh "$(node_ip "$1")" "python3 /opt/igneum/bin/wrpc.py call getFinalityCheckpoints '{\"last\": 400}'" 2>/dev/null </dev/null \
    | python3 -c 'import json,sys
j=json.load(sys.stdin); cps=j.get("checkpoints") or []
locked=[int(c["index"]) for c in cps if str(c.get("state","")).lower()=="locked"]
daa=max([int(c.get("daaScore",0)) for c in cps] or [0])
print(max(locked) if locked else "none", daa, len(locked))' 2>/dev/null || echo "NA NA NA"
}
first_minor=$(printf '%s\n' "$minority" | head -1)
first_major=$(printf '%s\n' "$majority" | head -1)
locks_line() {   # label: one line per side into locks.tsv
  local m M; m=$(lock_state "$first_minor"); M=$(lock_state "$first_major")
  printf '%s\t%s\tminority\t%s\t%s\n%s\t%s\tmajority\t%s\t%s\n' "$(date -u +%H:%M:%S)" "$1" "$first_minor" "$m" "$(date -u +%H:%M:%S)" "$1" "$first_major" "$M" | tr ' ' '\t' >> "$out/locks.tsv"
  log "locks ($1): minority $first_minor [max locked index, daa, locked count] $m; majority $first_major $M"
}

snapshot "$out/before.tsv"
printf 'time\tmoment\tside\tnode\tmax_locked_index\tdaa\tlocked_count\n' > "$out/locks.tsv"
locks_line at-cut
for n in $minority; do checkpoints "$n" "$out/checkpoints-$n-at-cut.json"; done
checkpoints "$first_major" "$out/checkpoints-$first_major-at-cut.json"
t0=$(date +%s)
P2P_PORTS="$P2P_PORT,$(( FWD_PORT_BASE + 1 )):$(( FWD_PORT_BASE + 99 ))"
if [ "$NET_MODE" = private ]; then
  # one rule set per zone gateway of the region: INPUT/OUTPUT for the gateway's own node, FORWARD for the nodes behind it
  for z in $(for n in $minority; do node_zone "$n"; done | sort -u); do
    gw=$(gateway_of_zone "$z"); [ -n "$gw" ] || die "no gateway in nodes.tsv for zone $z"
    printf '%s\n' "$minority" | grep -qx "$gw" || die "gateway $gw of zone $z is not in region $region: the cut would also split that zone"
    rules=""
    for ip in $(outside_ips_for "$gw"); do
      for chain in INPUT FORWARD; do
        rules="$rules iptables -I $chain -s $ip -p tcp -m multiport --dports $P2P_PORTS -m comment --comment igneum-partition -j DROP;"
        rules="$rules iptables -I $chain -s $ip -p tcp -m multiport --sports $P2P_PORTS -m comment --comment igneum-partition -j DROP;"
      done
      for chain in OUTPUT FORWARD; do
        rules="$rules iptables -I $chain -d $ip -p tcp -m multiport --dports $P2P_PORTS -m comment --comment igneum-partition -j DROP;"
        rules="$rules iptables -I $chain -d $ip -p tcp -m multiport --sports $P2P_PORTS -m comment --comment igneum-partition -j DROP;"
      done
    done
    nssh "$(node_ip "$gw")" "$rules true" && log "cut zone $z at its gateway $gw (INPUT, OUTPUT, FORWARD; far gateways $(outside_ips_for "$gw"); ports $P2P_PORTS)"
  done
else
  for n in $minority; do
    rules=""
    for ip in $(outside_ips_for "$n"); do
      rules="$rules iptables -I INPUT -s $ip -p tcp --dport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
      rules="$rules iptables -I INPUT -s $ip -p tcp --sport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
      rules="$rules iptables -I OUTPUT -d $ip -p tcp --dport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
      rules="$rules iptables -I OUTPUT -d $ip -p tcp --sport $P2P_PORT -m comment --comment igneum-partition -j DROP;"
    done
    nssh "$(node_ip "$n")" "$rules true" && log "cut $n"
  done
fi
printf '%s\tpartition %s cut\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$region" >> "$(results_dir_for)/events.log"
trap 'log "interrupted: healing"; heal_all' INT TERM

: > "$out/during.tsv"
for i in $(seq 1 $(( minutes * 2 ))); do
  sleep 30
  snapshot "$out/during-$i.tsv"; cat "$out/during-$i.tsv" >> "$out/during.tsv"; rm -f "$out/during-$i.tsv"
  m_sinks=$(for n in $minority; do awk -F'\t' -v n="$n" '$1 == n { print $3 }' "$out/during.tsv" | tail -1; done | sort -u | wc -l | tr -d ' ')
  M_sinks=$(for n in $majority; do awk -F'\t' -v n="$n" '$1 == n { print $3 }' "$out/during.tsv" | tail -1; done | sort -u | wc -l | tr -d ' ')
  log "t+$(( i * 30 )) s: minority distinct sinks $m_sinks, majority distinct sinks $M_sinks"
  [ $(( i % 4 )) = 0 ] && locks_line "cut+$(( i * 30 ))s"
done

log "heal: snapshots and checkpoints on both sides, then rules off"
snapshot "$out/at-heal.tsv"
for n in $minority; do checkpoints "$n" "$out/checkpoints-$n-at-heal.json"; done
checkpoints "$first_major" "$out/checkpoints-$first_major-at-heal.json"
locks_line at-heal
heal_all
t_heal=$(date +%s)
trap - INT TERM
if [ "${RECONNECT_KICK:-1}" = 1 ]; then
  for n in $minority; do nssh "$(node_ip "$n")" 'systemctl restart igneumd igneum-blocklog' && log "kicked $n (restart, reconnects at once)"; done
fi
printf '%s\tpartition %s healed\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$region" >> "$(results_dir_for)/events.log"

log "waiting for convergence (at most 2 distinct sinks for 3 polls, 5 s apart; up to 20 min)"
ok=0; converged_at=""
for i in $(seq 1 240); do
  sleep 5
  snapshot "$out/poll.tsv"
  d=$(cut -f3 "$out/poll.tsv" | grep -c . ); u=$(cut -f3 "$out/poll.tsv" | sort -u | grep -c .)
  if [ "$u" -le 2 ] && [ "$d" = "$(node_count)" ]; then ok=$((ok + 1)); else ok=0; fi
  [ $(( i % 6 )) = 0 ] && log "t+$(( i * 5 )) s after heal: $u distinct sinks"
  if [ "$ok" -ge 3 ]; then converged_at=$(( $(date +%s) - t_heal - 10 )); break; fi
done
[ -n "$converged_at" ] && log "converged $converged_at s after heal" || log "no convergence within 20 min (see poll.tsv)"
locks_line converged
for n in $minority; do checkpoints "$n" "$out/checkpoints-$n-converged.json"; done
checkpoints "$first_major" "$out/checkpoints-$first_major-converged.json"

# the first lock after the heal: poll the majority node until a locked index above the one it held at heal appears
heal_max=$(awk -F'\t' '$2 == "at-heal" && $3 == "majority" { print $5 }' "$out/locks.tsv" | head -1)
heal_daa=$(awk -F'\t' '$2 == "at-heal" && $3 == "majority" { print $6 }' "$out/locks.tsv" | head -1)
first_lock="none within LOCK_WAIT"
if [ "${heal_daa:-0}" != NA ] && [ "${heal_daa:-0}" -lt "${WEIGHT_WINDOW_DAA:-7200}" ] && [ "${heal_max:-none}" = none ]; then
  first_lock="not possible: weight window still filling (daa $heal_daa of ${WEIGHT_WINDOW_DAA:-7200} at heal)"
  log "first lock after heal: $first_lock"
else
  log "waiting up to ${LOCK_WAIT:-600} s for the first lock after the heal (majority node $first_major, above index ${heal_max:-none})"
  for i in $(seq 1 $(( ${LOCK_WAIT:-600} / 15 ))); do
    sleep 15
    st=$(lock_state "$first_major"); mx=${st%% *}
    if [ "$mx" != none ] && [ "$mx" != NA ] && { [ "${heal_max:-none}" = none ] || [ "$mx" -gt "$heal_max" ]; }; then
      first_lock="index $mx, $(( $(date +%s) - t_heal )) s after heal (majority $first_major); minority $first_minor: $(lock_state "$first_minor")"
      locks_line first-lock-after-heal; break
    fi
  done
  log "first lock after heal: $first_lock"
fi
# the same from the journals, which do not depend on when the poll above started: the first LOCKED line after t_heal
# on one node per side (seconds after the rules came off), plus certificates the minority adopted from the majority
for n in $first_major $first_minor; do
  nssh "$(node_ip "$n")" "journalctl -u igneumd --no-pager -o short-iso --since '@$(( t_heal - 2 ))' --until '@$(( t_heal + ${LOCK_WAIT:-600} ))' | grep -E 'Finality: (checkpoint [0-9]+ LOCKED|certificate at index)' | head -60" > "$out/locks-journal-$n.txt" 2>/dev/null </dev/null || true
  l=$(grep -m1 ' LOCKED: ' "$out/locks-journal-$n.txt" | sed -E 's/^([0-9T:-]+)\+0000 .*checkpoint ([0-9]+) LOCKED.*signed ([0-9]+ = [0-9.]+% of active, [0-9.]+% of total).*/\2\t\1\t\3/')
  a=$(grep -c 'certificate at index' "$out/locks-journal-$n.txt" 2>/dev/null || echo 0)
  if [ -n "$l" ]; then
    ts=$(printf '%s' "$l" | cut -f2); secs=$(( $(date -u -j -f %Y-%m-%dT%H:%M:%S "$ts" +%s 2>/dev/null || date -u -d "$ts" +%s) - t_heal ))
    printf '%s\t%s\t%s\t%s\t%s\n' "$n" "$(printf '%s' "$l" | cut -f1)" "$secs" "$(printf '%s' "$l" | cut -f3)" "$a" >> "$out/first-lock.tsv"
  else
    printf '%s\tnone\t-\t-\t%s\n' "$n" "$a" >> "$out/first-lock.tsv"
  fi
done

# the moment each minority node adopted the majority chain: its first virtualChainChanged removal of 5 or more blocks
# after the rules came off (chain.tsv), with the reconnect and IBD lines from its journal; the sink-count criterion
# above is noisy (a healthy 12-node network shows 2 to 4 distinct sinks at any instant), this is the heal time
log "heal time from the minority nodes' chain.tsv and journal"
: > "$out/heal.tsv"
for n in $minority; do
  nssh "$(node_ip "$n")" "awk -F'\t' -v t=$(( t_heal * 1000 - 20000 )) '\$1 >= t && \$3 >= 5' /var/log/igneum/chain.tsv; echo; journalctl -u igneumd --no-pager -o short-iso --since '@$(( t_heal - 10 ))' --until '@$(( t_heal + 300 ))' | grep -iE 'connected to|IBD|reorg' | head -40" > "$out/heal-$n.txt" 2>/dev/null </dev/null || true
  first=$(awk -F'\t' -v t=$(( t_heal * 1000 )) 'NF >= 3 && $1 >= t && $3 >= 5 { print $1, $3; exit }' "$out/heal-$n.txt")
  if [ -n "$first" ]; then
    printf '%s\t%s\t%s\n' "$n" "$(( (${first%% *} - t_heal * 1000) / 1000 ))" "${first#* }" >> "$out/heal.tsv"
  else
    printf '%s\tnone\tnone\n' "$n" >> "$out/heal.tsv"
  fi
done

log "reorg depth from each node's own sink at heal (getVirtualChainFromBlock)"
: > "$out/reorg.tsv"
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
  sink=$(awk -F'\t' -v n="$name" '$1 == n { print $3 }' "$out/at-heal.tsv")
  side=majority; printf '%s\n' "$minority" | grep -qx "$name" && side=minority
  r=$(nssh "$ip" "python3 /opt/igneum/bin/wrpc.py call getVirtualChainFromBlock '{\"startHash\": \"$sink\", \"includeAcceptedTransactionIds\": false}'" 2>/dev/null </dev/null \
      | python3 -c 'import json,sys; j=json.load(sys.stdin); print(len(j.get("removedChainBlockHashes",[])), len(j.get("addedChainBlockHashes",[])))' 2>/dev/null || echo "NA NA")
  chainmax=$(nssh "$ip" "awk -F'\t' -v t=$(( t_heal * 1000 )) '\$1 >= t && \$3 > m { m = \$3 } END { print m + 0 }' /var/log/igneum/chain.tsv" 2>/dev/null </dev/null || echo NA)
  printf '%s\t%s\t%s\t%s\t%s\n' "$name" "$side" "$sink" "$r" "$chainmax" | tr ' ' '\t' >> "$out/reorg.tsv"
done 3< "$NODES_FILE"
{
  printf '# Partition %s, %s min, %s\n\n' "$region" "$minutes" "$stamp"
  printf 'cut at %s, healed at %s, converged %s s after heal (criterion: at most 2 distinct sinks for 3 polls)\n\n' "$(date -u -r "$t0" +%H:%M:%S 2>/dev/null || date -u -d @"$t0" +%H:%M:%S)" "$(date -u -r "$t_heal" +%H:%M:%S 2>/dev/null || date -u -d @"$t_heal" +%H:%M:%S)" "${converged_at:-none}"
  printf '| node | side | sink at heal | removed (reorg depth) | added | max single removal after heal |\n|---|---|---|---|---|---|\n'
  awk -F'\t' '{ printf "| %s | %s | %s | %s | %s | %s |\n", $1, $2, substr($3, 1, 12), $4, $5, $6 }' "$out/reorg.tsv"
  printf '\nHeal time (seconds from the rules coming off until the node'"'"'s first chain removal of 5 or more blocks, chain.tsv; journal in heal-<node>.txt):\n\n| node | adopted the majority chain after s | blocks removed |\n|---|---|---|\n'
  awk -F'\t' '{ printf "| %s | %s | %s |\n", $1, $2, $3 }' "$out/heal.tsv"
  printf '\nMinority reorg depth, max: %s. Majority, max: %s.\n' "$(awk -F'\t' '$2 == "minority" && $4 != "NA" && $4 > m { m = $4 } END { print m + 0 }' "$out/reorg.tsv")" "$(awk -F'\t' '$2 == "majority" && $4 != "NA" && $4 > m { m = $4 } END { print m + 0 }' "$out/reorg.tsv")"
  printf '\nConflicting locks at heal (same checkpoint index, different hash, both locked):\n'
  python3 "$HERE/experiments/analyze.py" locks "$out"
  printf '\nLocks per side (highest locked checkpoint index, DAA, locked count on one node per side):\n\n| time | moment | side | node | max locked index | daa | locked count |\n|---|---|---|---|---|---|---|\n'
  awk -F'\t' 'NR > 1 { printf "| %s | %s | %s | %s | %s | %s | %s |\n", $1, $2, $3, $4, $5, $6, $7 }' "$out/locks.tsv"
  printf '\nFirst lock after the heal: %s\n' "$first_lock"
  if [ -s "$out/first-lock.tsv" ]; then
    printf '\nFirst lock after the heal from the journals (locks-journal-<node>.txt):\n\n| node | first locked index after heal | s after heal | signed | certificates adopted from the other side |\n|---|---|---|---|---|\n'
    awk -F'\t' '{ printf "| %s | %s | %s | %s | %s |\n", $1, $2, $3, $4, $5 }' "$out/first-lock.tsv"
  fi
} | tee "$out/partition.md"
log "written: $out/partition.md (plus before/during/at-heal/poll/reorg tsv and the checkpoint dumps)"