#!/usr/bin/env bash
# Roll a new igneumd onto the cloud devnet one node at a time, with the difficulty v2 activation height in every
# node's override file (4 October 2026, the Hetzner rehearsal of docs/analysis/difficulty-2026-10-04-oscillation.md
# section 8). The chain keeps running: each node is stopped, its binary replaced, the override file rewritten, the
# node and its miner started again, and the next node is not touched until this one is synced with peers and its
# DAA score is within a few blocks of a reference node.
#
#   ./rollout-v2.sh roll <igneumd> <activation-daa> [node ...]    the roll (all nodes in nodes.tsv order, or the named ones)
#   ./rollout-v2.sh check [reference-node]                         every node's DAA, difficulty, v2 line in its journal, and
#                                                                  whether the reference node's sink is a chain block on it
#   ./rollout-v2.sh watch <activation-daa> [minutes]               one check line per minute until activation + 600 has passed
#   ./rollout-v2.sh back [node ...]                                put /opt/igneum/bin/igneumd.prev back and drop the field
#
# What is NOT done here: provision.sh install (it rewrites the override file to genesis_bits alone and node.env to one
# miner thread), and the miner (it follows templates; the binary stays). The previous binary is kept as igneumd.prev.
. "$(dirname "$0")/lib/common.sh"
require_nodes
cmd="${1:-}"; shift || true

override_json() { printf '{"genesis_bits": %s, "difficulty_v2_activation_daa": %s}' "$(genesis_bits_decimal)" "$1"; }

# one sample line: ts sink blue daa tips peers difficulty headers blocks
sample() { nssh "$(node_ip "$1")" "python3 /opt/igneum/bin/wrpc.py sample 2>/dev/null" </dev/null 2>/dev/null | tr -d '\r'; }
sample_field() { printf '%s' "$1" | cut -f"$2"; }

roll_one() {   # name binary activation sha
  local name="$1" bin="$2" act="$3" sha="$4" ip; ip=$(node_ip "$name")
  local have; have=$(nssh "$ip" "sha256sum /opt/igneum/bin/igneumd.new 2>/dev/null | cut -c1-64" </dev/null || true)
  if [ "$have" = "$sha" ]; then
    log "[$name] igneumd.new already staged (sha256 ${sha:0:12})"
  else
    log "[$name] uploading igneumd ($(stat -f %z "$bin") bytes, sha256 ${sha:0:12})"
    nscp "$bin" "$SSH_USER@$ip:/opt/igneum/bin/igneumd.new"
    have=$(nssh "$ip" "sha256sum /opt/igneum/bin/igneumd.new | cut -c1-64" </dev/null)
    [ "$have" = "$sha" ] || die "[$name] upload sha256 mismatch: $have"
  fi
  nssh "$ip" "set -e
    chmod +x /opt/igneum/bin/igneumd.new
    [ -f /opt/igneum/bin/igneumd.prev ] || cp -p /opt/igneum/bin/igneumd /opt/igneum/bin/igneumd.prev
    printf '%s\n' '$(override_json "$act")' > /etc/igneum/override-params.json
    systemctl stop igneumd
    mv /opt/igneum/bin/igneumd.new /opt/igneum/bin/igneumd
    systemctl start igneumd
    sleep 2
    systemctl start igneum-miner igneum-blocklog     # both stop with the node (Requires=) and do not come back by themselves
    echo \"override: \$(cat /etc/igneum/override-params.json)\"
    echo \"igneumd: \$(/opt/igneum/bin/igneumd --version | head -1), unit \$(systemctl is-active igneumd), miner \$(systemctl is-active igneum-miner)\"" </dev/null 2>&1 | sed "s/^/[$name] /"
  # rejoin: the v2 line in the journal, then synced with peers and a DAA score near the reference
  local i line peers daa ref refdaa sink
  for i in $(seq 1 24); do
    sleep 5
    line=$(sample "$name"); peers=$(sample_field "$line" 6); daa=$(sample_field "$line" 4); sink=$(sample_field "$line" 2)
    ref=$(sample "$REF"); refdaa=$(sample_field "$ref" 4)
    if [ -n "$daa" ] && [ -n "$refdaa" ] && [ "${peers:-0}" -ge 1 ] && [ $(( refdaa - daa )) -le 3 ] && [ $(( daa - refdaa )) -le 3 ]; then
      log "[$name] rejoined after $(( i * 5 )) s: daa=$daa (ref $REF $refdaa) peers=$peers difficulty=$(sample_field "$line" 7) sink=${sink:0:12}"
      nssh "$ip" "journalctl -u igneumd --no-pager -n 400 -o cat | grep -m1 'Difficulty rule v2'" </dev/null 2>/dev/null | sed "s/^/[$name] journal: /"
      return 0
    fi
  done
  log "[$name] did NOT rejoin within 120 s: last sample '$line' (ref $REF: '$ref')"
  nssh "$ip" "journalctl -u igneumd --no-pager -n 30 -o cat" </dev/null 2>/dev/null | sed "s/^/[$name] journal: /"
  return 1
}

check() {   # [reference]
  local ref="${1:-$(node_names | head -1)}" refline refsink refdaa
  refline=$(sample "$ref"); refdaa=$(sample_field "$refline" 4)
  # the reference's sink of two minutes ago (its 5-s sample log), deep enough to have reached every node
  refsink=$(nssh "$(node_ip "$ref")" "tail -25 /var/log/igneum/samples.tsv | head -1 | cut -f2" </dev/null 2>/dev/null | tr -d '\r')
  [ -n "$refsink" ] || refsink=$(sample_field "$refline" 2)
  printf '%-10s %8s %12s %5s %4s %-8s %-12s %s\n' node daa difficulty peers tips "ref-sink" sink "v2 line in journal"
  while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
    (
      line=$(sample "$name")
      onchain=$(nssh "$ip" "python3 /opt/igneum/bin/wrpc.py call getBlock '{\"hash\":\"$refsink\",\"includeTransactions\":false}' 2>/dev/null | python3 -c 'import json,sys; j=json.load(sys.stdin); b=j.get(\"block\",j); v=b.get(\"verboseData\",{}); print(\"chain\" if v.get(\"isChainBlock\") else (\"have\" if b.get(\"header\") else \"MISSING\"))' 2>/dev/null || echo unknown" </dev/null 2>/dev/null | tail -1)
      v2=$(nssh "$ip" "journalctl -u igneumd --no-pager -b -o cat 2>/dev/null | grep 'Difficulty rule v2' | tail -1 | sed 's/.*DAA score //'" </dev/null 2>/dev/null)
      printf '%-10s %8s %12.0f %5s %4s %-8s %-12s %s\n' "$name" "$(sample_field "$line" 4)" "$(sample_field "$line" 7)" "$(sample_field "$line" 6)" "$(sample_field "$line" 5)" "${onchain:-?}" "$(sample_field "$line" 2 | cut -c1-12)" "${v2:-none}"
    ) &
  done 3< "$NODES_FILE"
  wait
  printf 'reference %s: daa %s, its sink of 2 min ago %s ("chain" = that block is on the node'"'"'s selected chain, "have" = known but not on its chain, MISSING = not received)\n' "$ref" "$refdaa" "${refsink:0:12}"
}

case "$cmd" in
  stage)   # <igneumd>: igneumd.new onto every node without touching anything running: the Mac -> the 4 gateways in
           # parallel, then each gateway -> its private nodes over the zone network (agent forwarding: ssh-add the ops key first)
    bin="${1:?igneumd binary}"; sha=$(shasum -a 256 "$bin" | cut -c1-64)
    log "staging igneumd (sha256 ${sha:0:12}) as /opt/igneum/bin/igneumd.new: gateways first"
    for g in $(public_nodes); do
      ( have=$(nssh "$(node_ip "$g")" "sha256sum /opt/igneum/bin/igneumd.new 2>/dev/null | cut -c1-64" </dev/null || true)
        if [ "$have" = "$sha" ]; then log "[$g] already staged"; else nscp "$bin" "$SSH_USER@$(node_ip "$g"):/opt/igneum/bin/igneumd.new" && log "[$g] staged"; fi ) &
    done; wait
    for p in $(private_nodes); do
      z=$(node_zone "$p"); g=$(gateway_of_zone "$z")
      ( ssh "${SSH_OPTS[@]}" -A "$SSH_USER@$(node_pub "$g")" "scp -o StrictHostKeyChecking=accept-new -o BatchMode=yes /opt/igneum/bin/igneumd.new $SSH_USER@$(node_ip "$p"):/opt/igneum/bin/igneumd.new" && log "[$p] staged from $g" ) &
    done; wait
    while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
      ( have=$(nssh "$ip" "sha256sum /opt/igneum/bin/igneumd.new 2>/dev/null | cut -c1-64" </dev/null || true); [ "$have" = "$sha" ] && echo "[$name] ok" || echo "[$name] MISSING or wrong ($have)" ) &
    done 3< "$NODES_FILE"; wait ;;
  roll)
    bin="${1:?igneumd binary}"; act="${2:?activation daa}"; shift 2
    [ -f "$bin" ] || die "no binary at $bin"
    file "$bin" | grep -q "ELF 64-bit.*x86-64" || die "$bin is not an x86-64 Linux ELF"
    sha=$(shasum -a 256 "$bin" | cut -c1-64)
    names="${*:-$(node_names | tr '\n' ' ')}"
    REF=$(node_names | head -1)
    log "rolling igneumd (sha256 ${sha:0:12}) with difficulty_v2_activation_daa=$act onto: $names; reference node $REF"
    refline=$(sample "$REF"); log "reference $REF now: daa $(sample_field "$refline" 4), difficulty $(sample_field "$refline" 7), sink $(sample_field "$refline" 2 | cut -c1-12)"
    for n in $names; do
      if [ "$n" = "$REF" ]; then REF=$(node_names | sed -n 2p); fi   # the reference must not be the node being restarted
      roll_one "$n" "$bin" "$act" "$sha" || die "stopped at $n (collect logs: ./experiments/collect.sh)"
      REF=$(node_names | head -1)
    done
    log "rolled $names. Now: ./rollout-v2.sh check, then ./rollout-v2.sh watch $act" ;;
  check) check "${1:-}" ;;
  watch)
    act="${1:?activation daa}"; mins="${2:-0}"; n=0
    while true; do
      n=$((n + 1)); log "check $n"; check
      refdaa=$(sample "$(node_names | head -1)" | cut -f4)
      if [ -n "$refdaa" ] && [ "$refdaa" -ge $(( act + 600 )) ]; then log "DAA $refdaa is past $act + 600"; break; fi
      if [ "$mins" != 0 ] && [ "$n" -ge "$mins" ]; then break; fi
      sleep 55
    done ;;
  back)
    names="${*:-$(node_names | tr '\n' ' ')}"
    for n in $names; do
      nssh "$(node_ip "$n")" "set -e; [ -f /opt/igneum/bin/igneumd.prev ] || { echo 'no igneumd.prev'; exit 1; }
        printf '{\"genesis_bits\": %s}\n' '$(genesis_bits_decimal)' > /etc/igneum/override-params.json
        systemctl stop igneumd; cp -p /opt/igneum/bin/igneumd.prev /opt/igneum/bin/igneumd; systemctl start igneumd; sleep 2; systemctl start igneum-miner igneum-blocklog
        echo \"back to \$(/opt/igneum/bin/igneumd --version | head -1), \$(cat /etc/igneum/override-params.json)\"" </dev/null 2>&1 | sed "s/^/[$n] /"
    done ;;
  *) die "usage: rollout-v2.sh roll <igneumd> <activation-daa> [node ...] | check [ref] | watch <activation-daa> [minutes] | back [node ...]" ;;
esac