0e09aa25de
rollout-v2.sh stages the Linux igneumd (gateways from the Mac, private nodes from their gateway), rolls one node at a time with
difficulty_v2_activation_daa in every override file, checks the common chain and watches the height; results/2026-10-04/
rollout-v2*.log and v2/ (the hash-rate step under v2 and the v1 comparison). docs/plans/difficulty-v2-rollout-devnet.md: the
binaries and their sha256, the activation rule (N = DAA at publish + 10,800; baked at the cut as DAA + 14,400), the exact
restart lines for the observer node, the seed and Mac node 1, the OTA path for the two PCs through NODE_OVERRIDE_PARAMS in
packaged-config.sh (the engine side landed in b8b349a), the rehearsal record. Bench-log entry.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
134 lines
9.4 KiB
Bash
Executable file
134 lines
9.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Roll a new igneumd onto the cloud devnet one node at a time, with the difficulty v2 activation height in every
|
|
# node's override file (4 October 2026, the Hetzner rehearsal of docs/analysis/difficulty-2026-10-04-oscillation.md
|
|
# section 8). The chain keeps running: each node is stopped, its binary replaced, the override file rewritten, the
|
|
# node and its miner started again, and the next node is not touched until this one is synced with peers and its
|
|
# DAA score is within a few blocks of a reference node.
|
|
#
|
|
# ./rollout-v2.sh roll <igneumd> <activation-daa> [node ...] the roll (all nodes in nodes.tsv order, or the named ones)
|
|
# ./rollout-v2.sh check [reference-node] every node's DAA, difficulty, v2 line in its journal, and
|
|
# whether the reference node's sink is a chain block on it
|
|
# ./rollout-v2.sh watch <activation-daa> [minutes] one check line per minute until activation + 600 has passed
|
|
# ./rollout-v2.sh back [node ...] put /opt/igneum/bin/igneumd.prev back and drop the field
|
|
#
|
|
# What is NOT done here: provision.sh install (it rewrites the override file to genesis_bits alone and node.env to one
|
|
# miner thread), and the miner (it follows templates; the binary stays). The previous binary is kept as igneumd.prev.
|
|
. "$(dirname "$0")/lib/common.sh"
|
|
require_nodes
|
|
cmd="${1:-}"; shift || true
|
|
|
|
override_json() { printf '{"genesis_bits": %s, "difficulty_v2_activation_daa": %s}' "$(genesis_bits_decimal)" "$1"; }
|
|
|
|
# one sample line: ts sink blue daa tips peers difficulty headers blocks
|
|
sample() { nssh "$(node_ip "$1")" "python3 /opt/igneum/bin/wrpc.py sample 2>/dev/null" </dev/null 2>/dev/null | tr -d '\r'; }
|
|
sample_field() { printf '%s' "$1" | cut -f"$2"; }
|
|
|
|
roll_one() { # name binary activation sha
|
|
local name="$1" bin="$2" act="$3" sha="$4" ip; ip=$(node_ip "$name")
|
|
local have; have=$(nssh "$ip" "sha256sum /opt/igneum/bin/igneumd.new 2>/dev/null | cut -c1-64" </dev/null || true)
|
|
if [ "$have" = "$sha" ]; then
|
|
log "[$name] igneumd.new already staged (sha256 ${sha:0:12})"
|
|
else
|
|
log "[$name] uploading igneumd ($(stat -f %z "$bin") bytes, sha256 ${sha:0:12})"
|
|
nscp "$bin" "$SSH_USER@$ip:/opt/igneum/bin/igneumd.new"
|
|
have=$(nssh "$ip" "sha256sum /opt/igneum/bin/igneumd.new | cut -c1-64" </dev/null)
|
|
[ "$have" = "$sha" ] || die "[$name] upload sha256 mismatch: $have"
|
|
fi
|
|
nssh "$ip" "set -e
|
|
chmod +x /opt/igneum/bin/igneumd.new
|
|
[ -f /opt/igneum/bin/igneumd.prev ] || cp -p /opt/igneum/bin/igneumd /opt/igneum/bin/igneumd.prev
|
|
printf '%s\n' '$(override_json "$act")' > /etc/igneum/override-params.json
|
|
systemctl stop igneumd
|
|
mv /opt/igneum/bin/igneumd.new /opt/igneum/bin/igneumd
|
|
systemctl start igneumd
|
|
sleep 2
|
|
systemctl start igneum-miner igneum-blocklog # both stop with the node (Requires=) and do not come back by themselves
|
|
echo \"override: \$(cat /etc/igneum/override-params.json)\"
|
|
echo \"igneumd: \$(/opt/igneum/bin/igneumd --version | head -1), unit \$(systemctl is-active igneumd), miner \$(systemctl is-active igneum-miner)\"" </dev/null 2>&1 | sed "s/^/[$name] /"
|
|
# rejoin: the v2 line in the journal, then synced with peers and a DAA score near the reference
|
|
local i line peers daa ref refdaa sink
|
|
for i in $(seq 1 24); do
|
|
sleep 5
|
|
line=$(sample "$name"); peers=$(sample_field "$line" 6); daa=$(sample_field "$line" 4); sink=$(sample_field "$line" 2)
|
|
ref=$(sample "$REF"); refdaa=$(sample_field "$ref" 4)
|
|
if [ -n "$daa" ] && [ -n "$refdaa" ] && [ "${peers:-0}" -ge 1 ] && [ $(( refdaa - daa )) -le 3 ] && [ $(( daa - refdaa )) -le 3 ]; then
|
|
log "[$name] rejoined after $(( i * 5 )) s: daa=$daa (ref $REF $refdaa) peers=$peers difficulty=$(sample_field "$line" 7) sink=${sink:0:12}"
|
|
nssh "$ip" "journalctl -u igneumd --no-pager -n 400 -o cat | grep -m1 'Difficulty rule v2'" </dev/null 2>/dev/null | sed "s/^/[$name] journal: /"
|
|
return 0
|
|
fi
|
|
done
|
|
log "[$name] did NOT rejoin within 120 s: last sample '$line' (ref $REF: '$ref')"
|
|
nssh "$ip" "journalctl -u igneumd --no-pager -n 30 -o cat" </dev/null 2>/dev/null | sed "s/^/[$name] journal: /"
|
|
return 1
|
|
}
|
|
|
|
check() { # [reference]
|
|
local ref="${1:-$(node_names | head -1)}" refline refsink refdaa
|
|
refline=$(sample "$ref"); refdaa=$(sample_field "$refline" 4)
|
|
# the reference's sink of two minutes ago (its 5-s sample log), deep enough to have reached every node
|
|
refsink=$(nssh "$(node_ip "$ref")" "tail -25 /var/log/igneum/samples.tsv | head -1 | cut -f2" </dev/null 2>/dev/null | tr -d '\r')
|
|
[ -n "$refsink" ] || refsink=$(sample_field "$refline" 2)
|
|
printf '%-10s %8s %12s %5s %4s %-8s %-12s %s\n' node daa difficulty peers tips "ref-sink" sink "v2 line in journal"
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
|
|
(
|
|
line=$(sample "$name")
|
|
onchain=$(nssh "$ip" "python3 /opt/igneum/bin/wrpc.py call getBlock '{\"hash\":\"$refsink\",\"includeTransactions\":false}' 2>/dev/null | python3 -c 'import json,sys; j=json.load(sys.stdin); b=j.get(\"block\",j); v=b.get(\"verboseData\",{}); print(\"chain\" if v.get(\"isChainBlock\") else (\"have\" if b.get(\"header\") else \"MISSING\"))' 2>/dev/null || echo unknown" </dev/null 2>/dev/null | tail -1)
|
|
v2=$(nssh "$ip" "journalctl -u igneumd --no-pager -b -o cat 2>/dev/null | grep 'Difficulty rule v2' | tail -1 | sed 's/.*DAA score //'" </dev/null 2>/dev/null)
|
|
printf '%-10s %8s %12.0f %5s %4s %-8s %-12s %s\n' "$name" "$(sample_field "$line" 4)" "$(sample_field "$line" 7)" "$(sample_field "$line" 6)" "$(sample_field "$line" 5)" "${onchain:-?}" "$(sample_field "$line" 2 | cut -c1-12)" "${v2:-none}"
|
|
) &
|
|
done 3< "$NODES_FILE"
|
|
wait
|
|
printf 'reference %s: daa %s, its sink of 2 min ago %s ("chain" = that block is on the node'"'"'s selected chain, "have" = known but not on its chain, MISSING = not received)\n' "$ref" "$refdaa" "${refsink:0:12}"
|
|
}
|
|
|
|
case "$cmd" in
|
|
stage) # <igneumd>: igneumd.new onto every node without touching anything running: the Mac -> the 4 gateways in
|
|
# parallel, then each gateway -> its private nodes over the zone network (agent forwarding: ssh-add the ops key first)
|
|
bin="${1:?igneumd binary}"; sha=$(shasum -a 256 "$bin" | cut -c1-64)
|
|
log "staging igneumd (sha256 ${sha:0:12}) as /opt/igneum/bin/igneumd.new: gateways first"
|
|
for g in $(public_nodes); do
|
|
( have=$(nssh "$(node_ip "$g")" "sha256sum /opt/igneum/bin/igneumd.new 2>/dev/null | cut -c1-64" </dev/null || true)
|
|
if [ "$have" = "$sha" ]; then log "[$g] already staged"; else nscp "$bin" "$SSH_USER@$(node_ip "$g"):/opt/igneum/bin/igneumd.new" && log "[$g] staged"; fi ) &
|
|
done; wait
|
|
for p in $(private_nodes); do
|
|
z=$(node_zone "$p"); g=$(gateway_of_zone "$z")
|
|
( ssh "${SSH_OPTS[@]}" -A "$SSH_USER@$(node_pub "$g")" "scp -o StrictHostKeyChecking=accept-new -o BatchMode=yes /opt/igneum/bin/igneumd.new $SSH_USER@$(node_ip "$p"):/opt/igneum/bin/igneumd.new" && log "[$p] staged from $g" ) &
|
|
done; wait
|
|
while IFS=$'\t' read -r -u 3 name idx reg ip access pub port; do
|
|
( have=$(nssh "$ip" "sha256sum /opt/igneum/bin/igneumd.new 2>/dev/null | cut -c1-64" </dev/null || true); [ "$have" = "$sha" ] && echo "[$name] ok" || echo "[$name] MISSING or wrong ($have)" ) &
|
|
done 3< "$NODES_FILE"; wait ;;
|
|
roll)
|
|
bin="${1:?igneumd binary}"; act="${2:?activation daa}"; shift 2
|
|
[ -f "$bin" ] || die "no binary at $bin"
|
|
file "$bin" | grep -q "ELF 64-bit.*x86-64" || die "$bin is not an x86-64 Linux ELF"
|
|
sha=$(shasum -a 256 "$bin" | cut -c1-64)
|
|
names="${*:-$(node_names | tr '\n' ' ')}"
|
|
REF=$(node_names | head -1)
|
|
log "rolling igneumd (sha256 ${sha:0:12}) with difficulty_v2_activation_daa=$act onto: $names; reference node $REF"
|
|
refline=$(sample "$REF"); log "reference $REF now: daa $(sample_field "$refline" 4), difficulty $(sample_field "$refline" 7), sink $(sample_field "$refline" 2 | cut -c1-12)"
|
|
for n in $names; do
|
|
if [ "$n" = "$REF" ]; then REF=$(node_names | sed -n 2p); fi # the reference must not be the node being restarted
|
|
roll_one "$n" "$bin" "$act" "$sha" || die "stopped at $n (collect logs: ./experiments/collect.sh)"
|
|
REF=$(node_names | head -1)
|
|
done
|
|
log "rolled $names. Now: ./rollout-v2.sh check, then ./rollout-v2.sh watch $act" ;;
|
|
check) check "${1:-}" ;;
|
|
watch)
|
|
act="${1:?activation daa}"; mins="${2:-0}"; n=0
|
|
while true; do
|
|
n=$((n + 1)); log "check $n"; check
|
|
refdaa=$(sample "$(node_names | head -1)" | cut -f4)
|
|
if [ -n "$refdaa" ] && [ "$refdaa" -ge $(( act + 600 )) ]; then log "DAA $refdaa is past $act + 600"; break; fi
|
|
if [ "$mins" != 0 ] && [ "$n" -ge "$mins" ]; then break; fi
|
|
sleep 55
|
|
done ;;
|
|
back)
|
|
names="${*:-$(node_names | tr '\n' ' ')}"
|
|
for n in $names; do
|
|
nssh "$(node_ip "$n")" "set -e; [ -f /opt/igneum/bin/igneumd.prev ] || { echo 'no igneumd.prev'; exit 1; }
|
|
printf '{\"genesis_bits\": %s}\n' '$(genesis_bits_decimal)' > /etc/igneum/override-params.json
|
|
systemctl stop igneumd; cp -p /opt/igneum/bin/igneumd.prev /opt/igneum/bin/igneumd; systemctl start igneumd; sleep 2; systemctl start igneum-miner igneum-blocklog
|
|
echo \"back to \$(/opt/igneum/bin/igneumd --version | head -1), \$(cat /etc/igneum/override-params.json)\"" </dev/null 2>&1 | sed "s/^/[$n] /"
|
|
done ;;
|
|
*) die "usage: rollout-v2.sh roll <igneumd> <activation-daa> [node ...] | check [ref] | watch <activation-daa> [minutes] | back [node ...]" ;;
|
|
esac
|