Capacity layer: fixes from the smoke runs
- pow-fuzz: list saved mismatches from the directory, not an ls|node pipe (pipefail ran the || echo too, giving invalid JSON [][]); drop the dead inner accumulation line - sync-fuzz: merge the override with python, not node (node rounds the u64::MAX activation fields to a float the Rust parser rejects); retention-period-days 2 (the 2-day minimum); grep -c without || echo (pipefail doubled the count) - clippy-audit: cap_cargo_t (timeout cannot run the cap_cargo shell function); grep -c fix - all jobs sync the checkout unconditionally and lib.sh defaults the branch vars so a standalone job or smoke never trips set -u on CAP_NODE_BRANCH Smokes on igneum-build-1, all 0 panics: pow-fuzz 98 rounds / 19,600 programs / 78,400 units; sync-fuzz 142,883 requests, node alive, every kind disconnected or answered; sim-sweeps 5 rows; model-sweeps 7 sections; clippy-audit 69 branches, 1,192 warnings, 1 error (ca2-coord), 0 advisories. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
parent
ae9526ed13
commit
d00d1dd42c
6 changed files with 26 additions and 14 deletions
|
|
@ -42,9 +42,9 @@ for entry in "${BRANCHES[@]}"; do
|
|||
for c in $CRATES; do
|
||||
[ -f "$CLIP/$c/Cargo.toml" ] || continue
|
||||
[ "$(date +%s)" -lt "$deadline" ] || break
|
||||
if timeout 1200 cap_cargo "$CLIP/$c" clippy --release --all-targets > "$bdir/clippy-$c.log" 2>&1; then :; else be=$((be + 1)); fi
|
||||
bw=$((bw + $(grep -c '^warning: ' "$bdir/clippy-$c.log" 2>/dev/null || echo 0)))
|
||||
if timeout 300 cap_cargo "$CLIP/$c" audit --color never > "$bdir/audit-$c.log" 2>&1; then :; else audit_vulns=$((audit_vulns + $(grep -c '^Crate:' "$bdir/audit-$c.log" 2>/dev/null || echo 0))); fi
|
||||
if cap_cargo_t 1200 "$CLIP/$c" clippy --release --all-targets > "$bdir/clippy-$c.log" 2>&1; then :; else be=$((be + 1)); fi
|
||||
bw=$((bw + $(grep -c '^warning: ' "$bdir/clippy-$c.log" 2>/dev/null) + 0))
|
||||
if cap_cargo_t 300 "$CLIP/$c" audit --color never > "$bdir/audit-$c.log" 2>&1; then :; else audit_vulns=$((audit_vulns + $(grep -c '^Crate:' "$bdir/audit-$c.log" 2>/dev/null) + 0)); fi
|
||||
done
|
||||
echo -e "$b\t$sha\t$(date -u +%H:%M:%SZ)\twarnings=$bw\terrors=$be" >> "$done_file"
|
||||
warn_total=$((warn_total + bw)); err_total=$((err_total + be)); checked=$((checked + 1))
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ if [ "$DRY" = 1 ]; then
|
|||
printf '{"state":"idle","summary":%s,"counters":{"sections":0,"ran":0,"cached":0},"dry_run":true}' "$(cap_json_str "dry run: sections $SECTIONS")" | cap_summary model-sweeps
|
||||
exit 0
|
||||
fi
|
||||
[ -f "$CAP_SRC/sim/horizon/algorithm/model.py" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; }
|
||||
cap_sync_checkout >/dev/null 2>&1 || true; [ -f "$CAP_SRC/sim/horizon/algorithm/model.py" ] || { cap_say "no checkout"; exit 1; }
|
||||
MODEL="$CAP_SRC/sim/horizon/algorithm/model.py"
|
||||
OUT=$(cap_out_dir model-sweeps)
|
||||
h=$(sha256sum "$MODEL" | cut -c1-16)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ if [ "$DRY" = 1 ]; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
[ -d "$CAP_SRC/igneum-pow" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; }
|
||||
cap_sync_checkout >/dev/null 2>&1 || true; [ -d "$CAP_SRC/igneum-pow" ] || { cap_say "no checkout"; exit 1; }
|
||||
OUT=$(cap_out_dir pow-fuzz)
|
||||
POW="$CAP_SRC/igneum-pow"
|
||||
export CARGO_TARGET_DIR="$CAP_ROOT/target/pow"
|
||||
|
|
@ -63,13 +63,15 @@ while [ "$(date +%s)" -lt "$deadline" ]; do
|
|||
fi
|
||||
base=$((base + N_PER)); rounds=$((rounds + 1))
|
||||
cap_cursor_set pow-fuzz "$base"
|
||||
total_programs=$((total_programs + 2 * N_PER)); total_vectors=$((total_vectors + slice_vectors - (total_vectors - total_vectors)))
|
||||
done
|
||||
# cumulative totals: the cursor holds the running total across slices; this slice adds its own
|
||||
total_programs=$(cap_cursor_get pow-fuzz-programs 0); total_programs=$((total_programs + slice_programs))
|
||||
total_vectors=$(cap_cursor_get pow-fuzz-vectors 0); total_vectors=$((total_vectors + slice_vectors))
|
||||
cap_cursor_set pow-fuzz-programs "$total_programs"; cap_cursor_set pow-fuzz-vectors "$total_vectors"; cap_cursor_set pow-fuzz-panics "$panics"
|
||||
|
||||
saved_json=$(ls "$OUT"/mismatch-*.log 2>/dev/null | sed 's|.*/||' | "$NODE_BIN" -e 'const l=require("fs").readFileSync(0,"utf8").split("\n").filter(Boolean);process.stdout.write(JSON.stringify(l))' 2>/dev/null || echo '[]')
|
||||
# list the saved mismatch logs as a JSON array directly from the directory (no pipe: pipefail would run a `|| echo` too)
|
||||
saved_json=$("$NODE_BIN" -e 'const fs=require("fs"),d=process.argv[1];let l=[];try{l=fs.readdirSync(d).filter(f=>f.startsWith("mismatch-")&&f.endsWith(".log"))}catch{}process.stdout.write(JSON.stringify(l))' "$OUT" 2>/dev/null)
|
||||
[ -n "$saved_json" ] || saved_json='[]'
|
||||
sum="rounds $rounds, this slice $slice_programs programs and $slice_vectors units, total $total_programs programs; $panics mismatch(es); next seed base $base"
|
||||
printf '{"state":"ran","summary":%s,"counters":{"programs":%s,"vectors":%s,"panics":%s,"slice_programs":%s,"slice_vectors":%s},"next_seed_base":%s,"out":%s,"saved":%s}' \
|
||||
"$(cap_json_str "$sum")" "$total_programs" "$total_vectors" "$panics" "$slice_programs" "$slice_vectors" "$base" "$(cap_json_str "$OUT")" "${saved_json:-[]}" | cap_summary pow-fuzz
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ if [ "$DRY" = 1 ]; then
|
|||
printf '{"state":"idle","summary":%s,"counters":{"seeds":0,"rows":0,"moved":0},"dry_run":true,"next_seed":%s}' "$(cap_json_str "dry run: next seed $seed")" "$seed" | cap_summary sim-sweeps
|
||||
exit 0
|
||||
fi
|
||||
[ -d "$CAP_SRC/sim" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; }
|
||||
cap_sync_checkout >/dev/null 2>&1 || true; [ -d "$CAP_SRC/sim" ] || { cap_say "no checkout"; exit 1; }
|
||||
OUT=$(cap_out_dir sim-sweeps); CSV="$OUT/sweeps.csv"; [ -f "$CSV" ] || echo "utc,sim,seed,metric,value" > "$CSV"
|
||||
cd "$CAP_SRC"
|
||||
PY="python3"
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ if [ "$DRY" = 1 ]; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
[ -d "$CAP_FORK/.git" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; }
|
||||
cap_sync_checkout >/dev/null 2>&1 || true; [ -d "$CAP_FORK/.git" ] || { cap_say "no checkout"; exit 1; }
|
||||
OUT=$(cap_out_dir sync-fuzz)
|
||||
export CARGO_TARGET_DIR="$CAP_FORK/target-capacity"
|
||||
IGNEUMD="$CARGO_TARGET_DIR/release/igneumd"
|
||||
|
|
@ -42,14 +42,15 @@ if ! cap_cargo "$CAP_FORK" build --release -p kaspad -p igneum-p2p-probe --featu
|
|||
exit 1
|
||||
fi
|
||||
[ -x "$IGNEUMD" ] || IGNEUMD="$CARGO_TARGET_DIR/release/igneumd"
|
||||
# a merged override with a tiny retention so the node prunes (pruned-node semantics)
|
||||
# a merged override with skip_proof_of_work so no miner is needed. Python (not node): the override carries u64::MAX
|
||||
# activation fields (18446744073709551615) that JS numbers round to a float the Rust parser rejects; python keeps the int.
|
||||
MERGED="$CAP_ROOT/sync-override.json"
|
||||
"$NODE_BIN" -e 'const fs=require("fs");const o=JSON.parse(fs.readFileSync(process.argv[1]));o.skip_proof_of_work=true;fs.writeFileSync(process.argv[2],JSON.stringify(o))' "$OVERRIDE" "$MERGED" 2>/dev/null || cp "$OVERRIDE" "$MERGED"
|
||||
python3 -c 'import json,sys;o=json.load(open(sys.argv[1]));o["skip_proof_of_work"]=True;json.dump(o,open(sys.argv[2],"w"))' "$OVERRIDE" "$MERGED" 2>/dev/null || cp "$OVERRIDE" "$MERGED"
|
||||
|
||||
rm -rf "$DATADIR"; mkdir -p "$DATADIR"
|
||||
cap_say "starting pruned igneumd ($NET) on loopback"
|
||||
nice -n 19 ionice -c3 "$IGNEUMD" --devnet --devnet-suffix="$SUFFIX" --nodnsseed --disable-upnp --nologfiles \
|
||||
--enable-unsynced-mining --utxoindex --unsaferpc --archival=false --retention-period-days=0.02 \
|
||||
--enable-unsynced-mining --utxoindex --unsaferpc --retention-period-days=2 \
|
||||
--appdir="$DATADIR" --rpclisten=127.0.0.1:$BASE_PORT --rpclisten-json=127.0.0.1:$((BASE_PORT+2)) \
|
||||
--listen=127.0.0.1:$((BASE_PORT+1)) --outpeers=0 --override-params-file="$MERGED" --loglevel=info --yes \
|
||||
> "$DATADIR/node.log" 2>&1 &
|
||||
|
|
@ -75,8 +76,8 @@ cap_say "fuzzing for ${SLICE_S}s (net $NET, trace $trace)"
|
|||
--seed "$(( $(date +%s) % 100000 ))" --report-every 200 --alive-every 25 | tee "$trace" | tail -1 > "$OUT/.last" || true
|
||||
last=$(cat "$OUT/.last" 2>/dev/null)
|
||||
reqs=$(echo "$last" | "$NODE_BIN" -e 'try{const o=JSON.parse(require("fs").readFileSync(0,"utf8"));process.stdout.write(String(o.requests||0))}catch{process.stdout.write("0")}' 2>/dev/null || echo 0)
|
||||
alive=$(echo "$last" | grep -c '"node_alive":true' || true)
|
||||
down=$(grep -c '"sync-fuzz":"node-down"' "$trace" 2>/dev/null || echo 0)
|
||||
alive=$(printf "%s" "$last" | grep -c "\"node_alive\":true" 2>/dev/null); alive=${alive:-0}
|
||||
down=$(grep -c '"sync-fuzz":"node-down"' "$trace" 2>/dev/null); down=${down:-0}
|
||||
|
||||
panics=$(cap_cursor_get sync-fuzz-panics 0)
|
||||
if [ "$down" != 0 ] || { [ "$alive" = 0 ] && echo "$last" | grep -q 'node-down'; }; then
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ SUMMARY="$HERE_LIB/summary.mjs"
|
|||
NODE_BIN="${NODE_BIN:-/usr/local/bin/node}"
|
||||
UTC_DATE() { date -u +%Y-%m-%d; }
|
||||
cap_say() { printf '%s capacity/%s: %s\n' "$(date -u +%H:%M:%S)" "${CAP_JOB:-run}" "$*" >&2; }
|
||||
# defaults so `set -u` never trips before the first cap_sync_checkout (a standalone job, or a smoke)
|
||||
CAP_NODE_BRANCH="${CAP_NODE_BRANCH:-}"; CAP_REPO_SHA="${CAP_REPO_SHA:-}"; CAP_FORK_SHA="${CAP_FORK_SHA:-}"; CAP_REPO_BRANCH_USED="${CAP_REPO_BRANCH_USED:-}"
|
||||
|
||||
# ---- the build-slot and measure hold the layer yields to --------------------------------------------------------------
|
||||
# a slot or the measure file is "held" when flock -n cannot take it (another process holds the fd). The same probe the
|
||||
|
|
@ -94,6 +96,13 @@ cap_cargo() {
|
|||
( cd "$1" && shift && CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS="${CAP_CARGO_JOBS:-16}" \
|
||||
nice -n 19 ionice -c3 chrt -i 0 "$HOME/.cargo/bin/cargo" "$@" )
|
||||
}
|
||||
# cap_cargo with a timeout: <secs> <dir> <cargo args...>. `timeout` cannot run the cap_cargo shell function, so the
|
||||
# timeout wraps the external cargo directly (same nice/ionice/chrt). Returns 124 on timeout.
|
||||
cap_cargo_t() {
|
||||
local secs="$1" dir="$2"; shift 2
|
||||
( cd "$dir" && CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS="${CAP_CARGO_JOBS:-16}" \
|
||||
timeout "$secs" nice -n 19 ionice -c3 chrt -i 0 "$HOME/.cargo/bin/cargo" "$@" )
|
||||
}
|
||||
|
||||
cap_out_dir() { # <job>: make and echo today's out dir
|
||||
local d="$CAP_OUT_ROOT/$1/$(UTC_DATE)"; mkdir -p "$d"; echo "$d"
|
||||
|
|
|
|||
Loading…
Reference in a new issue