From d00d1dd42cd8d0f6e76dda856d372e7f8869a7ce Mon Sep 17 00:00:00 2001
From: igneum-labs <337424239+igneum-labs@users.noreply.github.com>
Date: Tue, 6 Oct 2026 21:32:55 +0000
Subject: [PATCH] Capacity layer: fixes from the smoke runs

- pow-fuzz: list saved mismatches from the directory, not an ls|node pipe (pipefail ran
  the || echo too, giving invalid JSON [][]); drop the dead inner accumulation line
- sync-fuzz: merge the override with python, not node (node rounds the u64::MAX activation
  fields to a float the Rust parser rejects); retention-period-days 2 (the 2-day minimum);
  grep -c without || echo (pipefail doubled the count)
- clippy-audit: cap_cargo_t (timeout cannot run the cap_cargo shell function); grep -c fix
- all jobs sync the checkout unconditionally and lib.sh defaults the branch vars so a
  standalone job or smoke never trips set -u on CAP_NODE_BRANCH

Smokes on igneum-build-1, all 0 panics: pow-fuzz 98 rounds / 19,600 programs / 78,400
units; sync-fuzz 142,883 requests, node alive, every kind disconnected or answered;
sim-sweeps 5 rows; model-sweeps 7 sections; clippy-audit 69 branches, 1,192 warnings,
1 error (ca2-coord), 0 advisories.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---
 infra/build-server/capacity/jobs/clippy-audit.sh |  6 +++---
 infra/build-server/capacity/jobs/model-sweeps.sh |  2 +-
 infra/build-server/capacity/jobs/pow-fuzz.sh     |  8 +++++---
 infra/build-server/capacity/jobs/sim-sweeps.sh   |  2 +-
 infra/build-server/capacity/jobs/sync-fuzz.sh    | 13 +++++++------
 infra/build-server/capacity/lib.sh               |  9 +++++++++
 6 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/infra/build-server/capacity/jobs/clippy-audit.sh b/infra/build-server/capacity/jobs/clippy-audit.sh
index 236f4289e..acbeb0251 100755
--- a/infra/build-server/capacity/jobs/clippy-audit.sh
+++ b/infra/build-server/capacity/jobs/clippy-audit.sh
@@ -42,9 +42,9 @@ for entry in "${BRANCHES[@]}"; do
   for c in $CRATES; do
     [ -f "$CLIP/$c/Cargo.toml" ] || continue
     [ "$(date +%s)" -lt "$deadline" ] || break
-    if timeout 1200 cap_cargo "$CLIP/$c" clippy --release --all-targets > "$bdir/clippy-$c.log" 2>&1; then :; else be=$((be + 1)); fi
-    bw=$((bw + $(grep -c '^warning: ' "$bdir/clippy-$c.log" 2>/dev/null || echo 0)))
-    if timeout 300 cap_cargo "$CLIP/$c" audit --color never > "$bdir/audit-$c.log" 2>&1; then :; else audit_vulns=$((audit_vulns + $(grep -c '^Crate:' "$bdir/audit-$c.log" 2>/dev/null || echo 0))); fi
+    if cap_cargo_t 1200 "$CLIP/$c" clippy --release --all-targets > "$bdir/clippy-$c.log" 2>&1; then :; else be=$((be + 1)); fi
+    bw=$((bw + $(grep -c '^warning: ' "$bdir/clippy-$c.log" 2>/dev/null) + 0))
+    if cap_cargo_t 300 "$CLIP/$c" audit --color never > "$bdir/audit-$c.log" 2>&1; then :; else audit_vulns=$((audit_vulns + $(grep -c '^Crate:' "$bdir/audit-$c.log" 2>/dev/null) + 0)); fi
   done
   echo -e "$b\t$sha\t$(date -u +%H:%M:%SZ)\twarnings=$bw\terrors=$be" >> "$done_file"
   warn_total=$((warn_total + bw)); err_total=$((err_total + be)); checked=$((checked + 1))
diff --git a/infra/build-server/capacity/jobs/model-sweeps.sh b/infra/build-server/capacity/jobs/model-sweeps.sh
index 02b720fdb..3e1ed5978 100755
--- a/infra/build-server/capacity/jobs/model-sweeps.sh
+++ b/infra/build-server/capacity/jobs/model-sweeps.sh
@@ -17,7 +17,7 @@ if [ "$DRY" = 1 ]; then
   printf '{"state":"idle","summary":%s,"counters":{"sections":0,"ran":0,"cached":0},"dry_run":true}' "$(cap_json_str "dry run: sections $SECTIONS")" | cap_summary model-sweeps
   exit 0
 fi
-[ -f "$CAP_SRC/sim/horizon/algorithm/model.py" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; }
+cap_sync_checkout >/dev/null 2>&1 || true; [ -f "$CAP_SRC/sim/horizon/algorithm/model.py" ] || { cap_say "no checkout"; exit 1; }
 MODEL="$CAP_SRC/sim/horizon/algorithm/model.py"
 OUT=$(cap_out_dir model-sweeps)
 h=$(sha256sum "$MODEL" | cut -c1-16)
diff --git a/infra/build-server/capacity/jobs/pow-fuzz.sh b/infra/build-server/capacity/jobs/pow-fuzz.sh
index f1cc6ef07..46498360d 100755
--- a/infra/build-server/capacity/jobs/pow-fuzz.sh
+++ b/infra/build-server/capacity/jobs/pow-fuzz.sh
@@ -20,7 +20,7 @@ if [ "$DRY" = 1 ]; then
   exit 0
 fi
 
-[ -d "$CAP_SRC/igneum-pow" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; }
+cap_sync_checkout >/dev/null 2>&1 || true; [ -d "$CAP_SRC/igneum-pow" ] || { cap_say "no checkout"; exit 1; }
 OUT=$(cap_out_dir pow-fuzz)
 POW="$CAP_SRC/igneum-pow"
 export CARGO_TARGET_DIR="$CAP_ROOT/target/pow"
@@ -63,13 +63,15 @@ while [ "$(date +%s)" -lt "$deadline" ]; do
   fi
   base=$((base + N_PER)); rounds=$((rounds + 1))
   cap_cursor_set pow-fuzz "$base"
-  total_programs=$((total_programs + 2 * N_PER)); total_vectors=$((total_vectors + slice_vectors - (total_vectors - total_vectors)))
 done
+# cumulative totals: the cursor holds the running total across slices; this slice adds its own
 total_programs=$(cap_cursor_get pow-fuzz-programs 0); total_programs=$((total_programs + slice_programs))
 total_vectors=$(cap_cursor_get pow-fuzz-vectors 0); total_vectors=$((total_vectors + slice_vectors))
 cap_cursor_set pow-fuzz-programs "$total_programs"; cap_cursor_set pow-fuzz-vectors "$total_vectors"; cap_cursor_set pow-fuzz-panics "$panics"
 
-saved_json=$(ls "$OUT"/mismatch-*.log 2>/dev/null | sed 's|.*/||' | "$NODE_BIN" -e 'const l=require("fs").readFileSync(0,"utf8").split("\n").filter(Boolean);process.stdout.write(JSON.stringify(l))' 2>/dev/null || echo '[]')
+# list the saved mismatch logs as a JSON array directly from the directory (no pipe: pipefail would run a `|| echo` too)
+saved_json=$("$NODE_BIN" -e 'const fs=require("fs"),d=process.argv[1];let l=[];try{l=fs.readdirSync(d).filter(f=>f.startsWith("mismatch-")&&f.endsWith(".log"))}catch{}process.stdout.write(JSON.stringify(l))' "$OUT" 2>/dev/null)
+[ -n "$saved_json" ] || saved_json='[]'
 sum="rounds $rounds, this slice $slice_programs programs and $slice_vectors units, total $total_programs programs; $panics mismatch(es); next seed base $base"
 printf '{"state":"ran","summary":%s,"counters":{"programs":%s,"vectors":%s,"panics":%s,"slice_programs":%s,"slice_vectors":%s},"next_seed_base":%s,"out":%s,"saved":%s}' \
   "$(cap_json_str "$sum")" "$total_programs" "$total_vectors" "$panics" "$slice_programs" "$slice_vectors" "$base" "$(cap_json_str "$OUT")" "${saved_json:-[]}" | cap_summary pow-fuzz
diff --git a/infra/build-server/capacity/jobs/sim-sweeps.sh b/infra/build-server/capacity/jobs/sim-sweeps.sh
index 5ef72a1dc..03a3bd260 100755
--- a/infra/build-server/capacity/jobs/sim-sweeps.sh
+++ b/infra/build-server/capacity/jobs/sim-sweeps.sh
@@ -19,7 +19,7 @@ if [ "$DRY" = 1 ]; then
   printf '{"state":"idle","summary":%s,"counters":{"seeds":0,"rows":0,"moved":0},"dry_run":true,"next_seed":%s}' "$(cap_json_str "dry run: next seed $seed")" "$seed" | cap_summary sim-sweeps
   exit 0
 fi
-[ -d "$CAP_SRC/sim" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; }
+cap_sync_checkout >/dev/null 2>&1 || true; [ -d "$CAP_SRC/sim" ] || { cap_say "no checkout"; exit 1; }
 OUT=$(cap_out_dir sim-sweeps); CSV="$OUT/sweeps.csv"; [ -f "$CSV" ] || echo "utc,sim,seed,metric,value" > "$CSV"
 cd "$CAP_SRC"
 PY="python3"
diff --git a/infra/build-server/capacity/jobs/sync-fuzz.sh b/infra/build-server/capacity/jobs/sync-fuzz.sh
index 199e6e7c5..8670648fc 100755
--- a/infra/build-server/capacity/jobs/sync-fuzz.sh
+++ b/infra/build-server/capacity/jobs/sync-fuzz.sh
@@ -28,7 +28,7 @@ if [ "$DRY" = 1 ]; then
   exit 0
 fi
 
-[ -d "$CAP_FORK/.git" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; }
+cap_sync_checkout >/dev/null 2>&1 || true; [ -d "$CAP_FORK/.git" ] || { cap_say "no checkout"; exit 1; }
 OUT=$(cap_out_dir sync-fuzz)
 export CARGO_TARGET_DIR="$CAP_FORK/target-capacity"
 IGNEUMD="$CARGO_TARGET_DIR/release/igneumd"
@@ -42,14 +42,15 @@ if ! cap_cargo "$CAP_FORK" build --release -p kaspad -p igneum-p2p-probe --featu
   exit 1
 fi
 [ -x "$IGNEUMD" ] || IGNEUMD="$CARGO_TARGET_DIR/release/igneumd"
-# a merged override with a tiny retention so the node prunes (pruned-node semantics)
+# a merged override with skip_proof_of_work so no miner is needed. Python (not node): the override carries u64::MAX
+# activation fields (18446744073709551615) that JS numbers round to a float the Rust parser rejects; python keeps the int.
 MERGED="$CAP_ROOT/sync-override.json"
-"$NODE_BIN" -e 'const fs=require("fs");const o=JSON.parse(fs.readFileSync(process.argv[1]));o.skip_proof_of_work=true;fs.writeFileSync(process.argv[2],JSON.stringify(o))' "$OVERRIDE" "$MERGED" 2>/dev/null || cp "$OVERRIDE" "$MERGED"
+python3 -c 'import json,sys;o=json.load(open(sys.argv[1]));o["skip_proof_of_work"]=True;json.dump(o,open(sys.argv[2],"w"))' "$OVERRIDE" "$MERGED" 2>/dev/null || cp "$OVERRIDE" "$MERGED"
 
 rm -rf "$DATADIR"; mkdir -p "$DATADIR"
 cap_say "starting pruned igneumd ($NET) on loopback"
 nice -n 19 ionice -c3 "$IGNEUMD" --devnet --devnet-suffix="$SUFFIX" --nodnsseed --disable-upnp --nologfiles \
-  --enable-unsynced-mining --utxoindex --unsaferpc --archival=false --retention-period-days=0.02 \
+  --enable-unsynced-mining --utxoindex --unsaferpc --retention-period-days=2 \
   --appdir="$DATADIR" --rpclisten=127.0.0.1:$BASE_PORT --rpclisten-json=127.0.0.1:$((BASE_PORT+2)) \
   --listen=127.0.0.1:$((BASE_PORT+1)) --outpeers=0 --override-params-file="$MERGED" --loglevel=info --yes \
   > "$DATADIR/node.log" 2>&1 &
@@ -75,8 +76,8 @@ cap_say "fuzzing for ${SLICE_S}s (net $NET, trace $trace)"
   --seed "$(( $(date +%s) % 100000 ))" --report-every 200 --alive-every 25 | tee "$trace" | tail -1 > "$OUT/.last" || true
 last=$(cat "$OUT/.last" 2>/dev/null)
 reqs=$(echo "$last" | "$NODE_BIN" -e 'try{const o=JSON.parse(require("fs").readFileSync(0,"utf8"));process.stdout.write(String(o.requests||0))}catch{process.stdout.write("0")}' 2>/dev/null || echo 0)
-alive=$(echo "$last" | grep -c '"node_alive":true' || true)
-down=$(grep -c '"sync-fuzz":"node-down"' "$trace" 2>/dev/null || echo 0)
+alive=$(printf "%s" "$last" | grep -c "\"node_alive\":true" 2>/dev/null); alive=${alive:-0}
+down=$(grep -c '"sync-fuzz":"node-down"' "$trace" 2>/dev/null); down=${down:-0}
 
 panics=$(cap_cursor_get sync-fuzz-panics 0)
 if [ "$down" != 0 ] || { [ "$alive" = 0 ] && echo "$last" | grep -q 'node-down'; }; then
diff --git a/infra/build-server/capacity/lib.sh b/infra/build-server/capacity/lib.sh
index e7990eddd..7418d4b2c 100755
--- a/infra/build-server/capacity/lib.sh
+++ b/infra/build-server/capacity/lib.sh
@@ -32,6 +32,8 @@ SUMMARY="$HERE_LIB/summary.mjs"
 NODE_BIN="${NODE_BIN:-/usr/local/bin/node}"
 UTC_DATE() { date -u +%Y-%m-%d; }
 cap_say() { printf '%s capacity/%s: %s\n' "$(date -u +%H:%M:%S)" "${CAP_JOB:-run}" "$*" >&2; }
+# defaults so `set -u` never trips before the first cap_sync_checkout (a standalone job, or a smoke)
+CAP_NODE_BRANCH="${CAP_NODE_BRANCH:-}"; CAP_REPO_SHA="${CAP_REPO_SHA:-}"; CAP_FORK_SHA="${CAP_FORK_SHA:-}"; CAP_REPO_BRANCH_USED="${CAP_REPO_BRANCH_USED:-}"
 
 # ---- the build-slot and measure hold the layer yields to --------------------------------------------------------------
 # a slot or the measure file is "held" when flock -n cannot take it (another process holds the fd). The same probe the
@@ -94,6 +96,13 @@ cap_cargo() {
   ( cd "$1" && shift && CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS="${CAP_CARGO_JOBS:-16}" \
       nice -n 19 ionice -c3 chrt -i 0 "$HOME/.cargo/bin/cargo" "$@" )
 }
+# cap_cargo with a timeout: <secs> <dir> <cargo args...>. `timeout` cannot run the cap_cargo shell function, so the
+# timeout wraps the external cargo directly (same nice/ionice/chrt). Returns 124 on timeout.
+cap_cargo_t() {
+  local secs="$1" dir="$2"; shift 2
+  ( cd "$dir" && CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS="${CAP_CARGO_JOBS:-16}" \
+      timeout "$secs" nice -n 19 ionice -c3 chrt -i 0 "$HOME/.cargo/bin/cargo" "$@" )
+}
 
 cap_out_dir() {   # <job>: make and echo today's out dir
   local d="$CAP_OUT_ROOT/$1/$(UTC_DATE)"; mkdir -p "$d"; echo "$d"