igneum/tools/fleet/fleet-puller.sh

#!/usr/bin/env bash
# Igneum fleet puller (main's order 21:5x BST 7 Oct 2026): the box fetches a SIGNED move file from the fleet file host on a
# one-minute timer, verifies the signature against the fleet key this box already trusts (its authorized_keys), downloads the
# named binary pair, checks every sha, and at the named minute (UTC epoch) restarts its Devnet 3 node and miner together from
# the running box-dn3.sh's own environment, then reads the version string and digest back and posts the line to the intake.
# A proxy outage never blocks a digest-moving release again: nothing here needs ssh. Env: BASE (file host prefix), LABEL,
# INTAKE_URL + INTAKE_KEY (the report-only intake key that every miner package carries). State under /root/fleet/move.
set -u
F=/root/fleet; M=$F/move; mkdir -p $M; cd $M
LABEL="${LABEL:?}"; BASE="${BASE:?}"; INTAKE_URL="${INTAKE_URL:-}"; INTAKE_KEY="${INTAKE_KEY:-}"
stamp(){ date -u +%Y-%m-%dT%H:%M:%SZ; }
log(){ echo "$(stamp) $*" >> $M/puller.log; }
post(){ # post "<title>" "<body>"
  echo "$(stamp) $1 | $2" >> $M/readback.log
  [ -n "$INTAKE_URL" ] && [ -n "$INTAKE_KEY" ] && python3 - "$1" "$2" "$LABEL" "$INTAKE_URL" "$INTAKE_KEY" <<'PY' >/dev/null 2>&1
import sys, json, urllib.request
t,b,l,u,k=sys.argv[1:]
req=urllib.request.Request(u, data=json.dumps({"from":"fleet:"+l,"kind":"note","title":t[:300],"body":b[:4000]}).encode(), headers={"Content-Type":"application/json","x-igneum-key":k}, method="POST")
try: urllib.request.urlopen(req, timeout=20).read()
except Exception as e: print(e)
PY
  true; }
awk '{print "igneum-fleet-move " $1, $2}' /root/.ssh/authorized_keys > $M/allowed_signers
jq_(){ python3 -c 'import sys,json; d=json.load(open(sys.argv[1])); v=d
for k in sys.argv[2].split("."): v=v[k] if isinstance(v,dict) else v[int(k)]
print(v if not isinstance(v,(list,dict)) else json.dumps(v))' "$@" 2>/dev/null; }
log "puller start label=$LABEL base=$BASE"
while :; do
  if curl -fsS -m 25 -o $M/move.json.tmp "$BASE/move.json" && curl -fsS -m 25 -o $M/move.json.sig.tmp "$BASE/move.json.sig"; then
    if ssh-keygen -Y verify -f $M/allowed_signers -I igneum-fleet-move -n igneum-fleet-move -s $M/move.json.sig.tmp < $M/move.json.tmp >/dev/null 2>&1; then
      mv -f $M/move.json.tmp $M/move.json; mv -f $M/move.json.sig.tmp $M/move.json.sig
    else log "BAD SIGNATURE on move.json, ignored"; rm -f $M/move.json.tmp $M/move.json.sig.tmp; sleep 60; continue; fi
  else [ -f $M/move.json ] || { sleep 60; continue; }; fi
  id=$(jq_ $M/move.json id); at=$(jq_ $M/move.json at_epoch); [ -n "$id" ] || { log "move.json without id"; sleep 60; continue; }
  # a staggered move (8 Oct 2026, the 0.3.25 re-execution from genesis): "delays": {"<label>": <seconds>} adds to at_epoch for that box; absent = 0
  dly=$(jq_ $M/move.json delays.$LABEL 2>/dev/null); case "$dly" in ''|*[!0-9]*) dly=0;; esac
  if [ "${at:-0}" -gt 0 ] 2>/dev/null; then at=$((at+dly)); fi
  if [ -e $M/applied-$id ]; then sleep 60; continue; fi
  pair=hands; for l in $(jq_ $M/move.json node_lane_labels | tr -d '[]",'); do [ "$l" = "$LABEL" ] && pair=node_lane; done
  url=$(jq_ $M/move.json pairs.$pair.url); tsha=$(jq_ $M/move.json pairs.$pair.sha); dsha=$(jq_ $M/move.json pairs.$pair.igneumd_sha); msha=$(jq_ $M/move.json pairs.$pair.miner_sha)
  if [ ! -e $M/fetched-$id ]; then
    mkdir -p $M/$id && curl -fsS -m 600 -o $M/$id/pair.tgz "$url" || { log "fetch failed $url"; sleep 60; continue; }
    echo "$tsha  $M/$id/pair.tgz" | sha256sum -c - >/dev/null 2>&1 || { log "TARBALL SHA MISMATCH $id"; rm -f $M/$id/pair.tgz; sleep 60; continue; }
    tar -xzf $M/$id/pair.tgz -C $M/$id && [ "$(sha256sum $M/$id/igneumd | cut -c1-64)" = "$dsha" ] && [ "$(sha256sum $M/$id/igneum-miner | cut -c1-64)" = "$msha" ] || { log "BINARY SHA MISMATCH $id"; rm -rf $M/$id; sleep 60; continue; }
    chmod +x $M/$id/igneumd $M/$id/igneum-miner; touch $M/fetched-$id
    # the pack gate's pair list by file: this move's miner sha (16 hex) appended once, so box-dn3.sh accepts the pair at the minute without a hand edit
    grep -qx "${msha:0:16}" $F/in/pair-miners.txt 2>/dev/null || echo "${msha:0:16}" >> $F/in/pair-miners.txt
    post "FLEET MOVE $id FETCHED $LABEL" "pair=$pair igneumd=${dsha:0:16} miner=${msha:0:16} at_epoch=$at"
    log "fetched $id pair=$pair"
  fi
  p0=$(pgrep -of '[b]ash in/box-dn3.sh'); [ -n "$p0" ] && tr '\0' '\n' < /proc/$p0/environ | grep -E '^[A-Z_][A-Z0-9_]*=' | grep -vE '^(PWD|OLDPWD|SHLVL|_|TERM|SHELL|LS_COLORS|HOSTNAME|SSH_[A-Z_]+|LANG|LC_[A-Z_]+|LOGNAME|USER|MAIL|MINER_ONLY)=' > $M/env-last.tmp && [ -s $M/env-last.tmp ] && mv -f $M/env-last.tmp $M/env-last
  now=$(date +%s)
  if [ "${at:-0}" -le 0 ] 2>/dev/null; then sleep 60; continue; fi
  if [ "$now" -lt "$at" ]; then d=$((at-now)); [ $d -gt 60 ] && d=60; sleep $d; continue; fi
  # 12:1xZ 8 Oct 2026 (shipper): a box still re-walking at the minute waits for its own restart until its state reads right. move.json may carry
  # "require_root": {"height":"0x6687","root":"0x3f53a7b9"}: the box's own EVM (EVM_PORT from env-last, default 26790) must answer that root
  # at that height before this box applies; otherwise HELD (posted once every 10 minutes) and re-checked each minute. No EVM answer = held too.
  # 15:0xZ 8 Oct 2026 (node lane, the acaf08b0 move): "require_replay_done": true holds a box whose executor has not reached its sink since
  # the node's last start (the log's last of "replaying from genesis"/"no snapshot to resume" vs "records up to the tip N are continuous")
  if [ "$(jq_ $M/move.json require_replay_done)" = "True" ] || [ "$(jq_ $M/move.json require_replay_done)" = "true" ]; then
    last=$(grep -anE 'replaying from genesis|no snapshot to resume from|records up to the tip [0-9]+ are continuous' $F/dn3-node.log 2>/dev/null | tail -n 1)
    case "$last" in *"are continuous"*) ;; *) hl=$M/held-replay-$id-stamp; if [ ! -f $hl ] || [ $(( $(date +%s) - $(stat -c %Y $hl) )) -ge 900 ]; then post "FLEET MOVE $id HELD $LABEL" "replay not done: $(echo "$last" | cut -c1-120)"; touch $hl; fi; sleep 60; continue;; esac
  fi
  rh=$(jq_ $M/move.json require_root.height); rr=$(jq_ $M/move.json require_root.root)
  if [ -n "$rh" ] && [ -n "$rr" ]; then
    ep=$(grep -oE '^EVM_PORT=.*' $M/env-last 2>/dev/null | head -n 1 | cut -d= -f2 | tr -d "'\""); ep=${ep:-26790}
    blk=$(curl -s -m 8 -X POST -H 'content-type: application/json' --data "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"eth_getBlockByNumber\",\"params\":[\"$rh\",false]}" http://127.0.0.1:$ep 2>/dev/null)
    got=$(echo "$blk" | grep -oE '"stateRoot":"0x[0-9a-f]+"' | cut -d'"' -f4); gh=$(echo "$blk" | grep -oE '"hash":"0x[0-9a-f]+"' | head -n 1 | cut -d'"' -f4); rhash=$(jq_ $M/move.json require_root.hash)
    if [ "${got:0:${#rr}}" != "$rr" ] || { [ -n "$rhash" ] && [ "${gh:2:${#rhash}}" != "$rhash" ]; }; then
      hl=$M/held-$id-stamp; if [ ! -f $hl ] || [ $(( $(date +%s) - $(stat -c %Y $hl) )) -ge 600 ]; then post "FLEET MOVE $id HELD $LABEL" "block $rh reads hash ${gh:2:8} root ${got:-none}, wanted ${rhash:-any}/$rr (re-walk not done); re-checked each minute"; touch $hl; fi
      sleep 60; continue
    fi
  fi
  # THE MINUTE: node and miner together, from the running box-dn3.sh's own environment
  pid=$(cat /root/fleet/pids/loop.pid 2>/dev/null); [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null || pid=$(pgrep -of '[b]ash in/box-dn3.sh')
  # the restart environment: the running box-dn3.sh's (never MINER_ONLY: a loop restarted alone carries it, and it would leave the node down), else env-last (written by every hand start since 21:4xZ 7 Oct 2026)
  if [ -n "$pid" ]; then tr '\0' '\n' < /proc/$pid/environ | grep -E '^[A-Z_][A-Z0-9_]*=' | grep -vE '^(PWD|OLDPWD|SHLVL|_|TERM|SHELL|LS_COLORS|HOSTNAME|SSH_[A-Z_]+|LANG|LC_[A-Z_]+|LOGNAME|USER|MAIL|MINER_ONLY)=' > $M/env-$id; else log "no running box-dn3.sh; using last env"; [ -f $M/env-last ] && grep -vE '^MINER_ONLY=' $M/env-last > $M/env-$id; fi
  [ -s $M/env-$id ] || { post "FLEET MOVE $id FAILED $LABEL" "no box-dn3.sh environment to restart from"; touch $M/applied-$id; sleep 60; continue; }
  cp $M/env-$id $M/env-last
  RPC=$(grep -E '^RPC_PORT=' $M/env-$id | cut -d= -f2); RPC=${RPC:-26610}
  bash $F/in/fleet-stop.sh all >/dev/null 2>&1; sleep 2
  true
  cd $F/in && NB=$(grep -oE '^NODE_BIN=.*' $M/env-last | cut -d= -f2 | tr -d "'\"" | xargs -n1 basename 2>/dev/null); NB=${NB:-igneumd-0322}; MB=$(grep -oE '^MINER_BIN=.*' $M/env-last | cut -d= -f2 | tr -d "'\"" | xargs -n1 basename 2>/dev/null); MB=${MB:-igneum-miner-0322}; mv -f $NB $NB.pre-$id 2>/dev/null; mv -f $MB $MB.pre-$id 2>/dev/null
  cp $M/$id/igneumd $NB && cp $M/$id/igneum-miner $MB && chmod +x $NB $MB; cd $M
  APPDIR_LOG=$(grep -oE '^APPDIR=.*' $M/env-last 2>/dev/null | head -n 1 | cut -d= -f2 | tr -d "'\""); APPDIR_LOG=${APPDIR_LOG:-dn3}   # 17:1xZ 8 Oct: the marker and the read-back follow the env's APPDIR (devnet-4 logs to dn4-node.log; the 16:50Z APPLIED lines read version= digest= empty for this)
  mk="=== fleet move $id $(date -u +%T)"; echo "$mk" >> $F/$APPDIR_LOG-node.log
  # every value quoted before sourcing (09:05Z 8 Oct 2026: a bare NET_ARGS=--devnet --devnet-suffix=3 line ran "--devnet-suffix=3" as a command and nine boxes came up with no node)
  python3 - "$M/env-$id" <<'PY' > $M/env-$id.quoted
import sys, shlex
for line in open(sys.argv[1]):
    line=line.rstrip("\n")
    if "=" not in line or not line.split("=",1)[0].replace("_","").isalnum(): continue
    k,v=line.split("=",1); v=v.strip()
    if len(v)>=2 and v[0]==v[-1] and v[0] in "'\"": v=v[1:-1]
    print(f"{k}={shlex.quote(v)}")
PY
  (cd $F && set -a && . $M/env-$id.quoted && set +a && setsid nohup bash in/box-dn3.sh </dev/null >/dev/null 2>&1 &)
  want_v=$(jq_ $M/move.json want_version); want_d=$(jq_ $M/move.json want_digest); rb=""; APPDIR_LOG=$(grep -oE '^APPDIR=.*' $M/env-last 2>/dev/null | head -n 1 | cut -d= -f2 | tr -d "'\""); APPDIR_LOG=${APPDIR_LOG:-dn3}   # 16:0xZ 8 Oct: devnet-4 logs to dn4-node.log
  for i in $(seq 1 18); do sleep 10
    v=$(awk -v m="$mk" '$0==m{f=1} f' $F/${APPDIR_LOG:-dn3}-node.log | grep -oE 'igneumd/[0-9]+\.[0-9]+\.[0-9]+-[0-9a-f]+' | tail -n 1); d=$(awk -v m="$mk" '$0==m{f=1} f' $F/${APPDIR_LOG:-dn3}-node.log | grep -o 'digest: [0-9a-f]*' | tail -n 1 | cut -c9-24)
    w=$(/opt/igneum/pkg/bin/igneum-miner watch 1 grpc://127.0.0.1:$RPC 2>/dev/null | grep -oE 'blocks=[0-9]+|peers=[0-9]+|synced=[a-z]+' | tr '\n' ' ')
    [ -n "$v" ] && [ -n "$d" ] && case "$w" in *synced=true*) [[ "$w" != *peers=0* ]] && break;; esac
  done
  m=$(pgrep -fc "igneum-miner(-0322)? mine grpc://127.0.0.1:$RPC "); ok=MISMATCH; [ "$v" = "$want_v" ] && [ "$d" = "$want_d" ] && ok=MATCH
  fp=$(grep -oE 'igneum-pow fingerprint [0-9a-f]{16}[^ ]*' $F/dn3-node.log 2>/dev/null | tail -n 1); fpb=$(grep -aoE 'IGNEUM_POW_FINGERPRINT=[0-9a-f]{16}' $M/$id/igneumd 2>/dev/null | head -n 1); fp="${fp:-igneum-pow fingerprint none} binary ${fpb:-IGNEUM_POW_FINGERPRINT=none}"   # 12:5xZ 8 Oct (shipper): the freeze's crate fingerprint from the node's start line; another value is a stop
  post "FLEET MOVE $id APPLIED $LABEL $ok" "version=$v digest=$d $w miner_procs=$m want=$want_v/$want_d rpc=$RPC ${fp:-igneum-pow fingerprint none}"
  touch $M/applied-$id; log "applied $id $ok $v $d $w"
  # 12:2xZ 8 Oct (node lane): bans persist in the node's DB; once this box's state at the reference block reads equal, unban every address it holds
  if [ -n "$rh" ] && [ -n "$rr" ]; then ( cd /root/fleet && EVM_PORT=$(grep -oE '^EVM_PORT=.*' $M/env-last | cut -d= -f2 | tr -d "'\"") RPC_PORT=$RPC setsid nohup bash in/unban-all.sh "$rh" "$rr" "$rhash" </dev/null >> $M/unban.out 2>&1 & ); fi
  sleep 60
done