127 lines
9.6 KiB
Bash
Executable file
127 lines
9.6 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Phase 1 on one rented card: the memory matrix of docs/analysis/prover-floor.md on the real card. Every point is one
|
|
# igneum-prove-host run against a fresh sp1-gpu-server (killed and its socket unlinked around every point), with an
|
|
# nvidia-smi sampler at 1 s (memory.used, power.draw, utilization); peak = max memory.used, base = the reading just
|
|
# before the point (the card's idle, or the miner's resident set when the miner runs beside the prover), own = peak
|
|
# minus base. Every proof is verified by the host's own (unpatched) verifier: the VERIFIED word in the RESULT line.
|
|
# Writes /root/fleet/out/matrix.json (card facts, the miner row, every point) and the raw logs beside it; the last log
|
|
# line is "RESULT matrix_done" or "RESULT matrix_failed <why>".
|
|
set -uo pipefail
|
|
F=/root/fleet; OUT=$F/out; LOG=$OUT/matrix.log; B=/opt/igneum/pkg/bin; FLOOR=/opt/igneum-floor
|
|
for b in igneum-prove-host igneum-prove-export; do for d in $FLOOR/target/release $FLOOR/prove/proving/igneum-prove/target/release /opt/igneum-segal/proving/igneum-prove/target/release; do [ -x $d/$b ] && ln -sfn $d/$b $FLOOR/bin/$b; done; done # the last one that exists wins: the segment host (--mode chain) over the floor host
|
|
HOST=$FLOOR/bin/igneum-prove-host; FIX=$FLOOR/prove/proving/fixtures
|
|
V1=$FIX/fees-v1-shards2.json; EMPTY=$FIX/block-72854-empty-block-first.json
|
|
LABEL="${LABEL:-box}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}"
|
|
MINER_SECS="${MINER_SECS:-150}"; POINT_TIMEOUT="${POINT_TIMEOUT:-1500}"
|
|
mkdir -p $OUT $F/mine/packs
|
|
exec > >(tee -a $LOG) 2>&1
|
|
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
say() { echo "$(stamp) $*"; }
|
|
q() { nvidia-smi --query-gpu="$1" --format=csv,noheader,nounits -i 0 2>/dev/null | head -1 | tr -d ' '; }
|
|
[ -x "$HOST" ] || { echo "RESULT matrix_failed no host"; exit 2; }
|
|
[ -x "$FLOOR/bin/sp1-gpu-server" ] || { echo "RESULT matrix_failed no patched server"; exit 2; }
|
|
NAME="$(q name)"; TOTAL="$(q memory.total)"; DRV="$(q driver_version)"; PLIM="$(q power.limit)"
|
|
echo "RESULT start $(stamp) label=$LABEL card=$NAME total_mib=$TOTAL driver=$DRV power_limit_w=$PLIM"
|
|
ROWS=$OUT/rows.jsonl; : > $ROWS
|
|
|
|
# the sampler: one csv per point, "ts,mem_mib,power_w,util_pct"
|
|
SAMP=""
|
|
# one nvidia-smi call a second in a loop: `nvidia-smi -l 1` buffers its file output in 4 KB chunks and ignored SIGTERM
|
|
# on the 3080 box (12:19Z), which hung the first matrix in sampler_stop
|
|
sampler_start() { ( while :; do nvidia-smi --query-gpu=timestamp,memory.used,power.draw,utilization.gpu --format=csv,noheader,nounits -i 0 2>/dev/null; sleep 1; done ) > "$1" & SAMP=$!; }
|
|
sampler_stop() { [ -n "$SAMP" ] && { pkill -P $SAMP 2>/dev/null; kill $SAMP 2>/dev/null; sleep 1; kill -9 $SAMP 2>/dev/null; }; SAMP=""; }
|
|
peak_of() { awk -F', *' 'NR>0 {if ($2+0 > m) m=$2+0} END {print m+0}' "$1"; }
|
|
mean_col() { awk -F', *' -v c="$2" '{s+=$c; n++} END {if (n) printf "%.1f", s/n; else print 0}' "$1"; }
|
|
kill_server() { pkill -x sp1-gpu-server 2>/dev/null; sleep 2; pkill -9 -x sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock; }
|
|
idle_mib() { sleep 3; q memory.used; }
|
|
|
|
# 1. idle
|
|
kill_server
|
|
sleep 5
|
|
IDLE_MIB="$(q memory.used)"; IDLE_W="$(q power.draw)"
|
|
echo "RESULT idle mem_mib=$IDLE_MIB power_w=$IDLE_W"
|
|
|
|
# 2. the node's state (the miner mines IBD templates when not synced; the rate is the same, the pack may move)
|
|
NODE_LINE="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"
|
|
waited=0; while [[ "$NODE_LINE" != *synced=true* && $waited -lt 900 ]]; do sleep 30; waited=$((waited+30)); NODE_LINE="$($B/igneum-miner watch 1 grpc://127.0.0.1:26610 2>/dev/null | grep -o 'blocks=[0-9]*.*synced=[a-z]*' | tail -1)"; done
|
|
echo "RESULT node $NODE_LINE waited_s=$waited"
|
|
|
|
# 3. the miner alone: rate, watts, working set
|
|
MPID=""
|
|
miner_start() {
|
|
cd $F/mine
|
|
rm -rf packs/devnet; $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet > $OUT/export-pack.log 2>&1 || say "export-pack failed (see export-pack.log)"
|
|
nohup $B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL" --worker $B/igneum-worker-cuda --worker-args "--device 0 --pack packs/devnet" \
|
|
--prepare-packs packs/prepare --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL" --status-secs 10 > $OUT/miner.log 2>&1 &
|
|
MPID=$!; cd $F
|
|
}
|
|
miner_stop() { [ -n "$MPID" ] && { kill $MPID 2>/dev/null; sleep 2; kill -9 $MPID 2>/dev/null; }; pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda' 2>/dev/null; pkill -x igneum-miner 2>/dev/null; MPID=""; sleep 3; }
|
|
miner_rate() { # mean of the STATUS now= values in the last N lines
|
|
grep STATUS $OUT/miner.log | grep -o ' now=[0-9.]*' | tail -n "${1:-10}" | cut -d= -f2 | awk '{s+=$1; n++} END {if (n) printf "%.2f", s/n; else print 0}'
|
|
}
|
|
miner_start
|
|
say "miner warming 60 s"; sleep 60
|
|
sampler_start $OUT/samp-miner.csv; sleep "$MINER_SECS"; sampler_stop
|
|
MINER_MHS="$(miner_rate 12)"; MINER_W="$(mean_col $OUT/samp-miner.csv 3)"; MINER_UTIL="$(mean_col $OUT/samp-miner.csv 4)"; MINER_MIB="$(peak_of $OUT/samp-miner.csv)"
|
|
MINER_ACC="$(grep -o 'accepted=[0-9]*' $OUT/miner.log | tail -1 | cut -d= -f2)"
|
|
MINER_STATUS="$(grep STATUS $OUT/miner.log | tail -1 | cut -c1-200)"
|
|
echo "RESULT miner mhs=$MINER_MHS watts=$MINER_W util=$MINER_UTIL mem_mib=$MINER_MIB own_mib=$((MINER_MIB - IDLE_MIB)) accepted=${MINER_ACC:-?} status=\"$MINER_STATUS\""
|
|
printf '%s\n' "{\"row\":\"miner\",\"mhs\":$MINER_MHS,\"watts\":$MINER_W,\"util\":$MINER_UTIL,\"mem_mib\":$MINER_MIB,\"own_mib\":$((MINER_MIB - IDLE_MIB)),\"accepted\":\"${MINER_ACC:-}\"}" >> $ROWS
|
|
miner_stop
|
|
|
|
# 4. one point: name, home (stock or patched), mode, threshold env, fixture, beside
|
|
point() {
|
|
local name="$1" home="$2" mode="$3" thr="$4" fixture="$5" beside="$6"
|
|
kill_server; sleep 2
|
|
local base; base="$(q memory.used)"
|
|
local env=(HOME="$home" SP1_PROVER=cuda RUST_LOG=off SP1_GPU_FLOOR_LOG=1)
|
|
[ -n "$thr" ] && env+=(SP1_GPU_ELEMENT_THRESHOLD="$thr")
|
|
local log=$OUT/point-$name.log res=$OUT/point-$name.json samp=$OUT/samp-$name.csv
|
|
say "point $name: mode $mode threshold ${thr:-default} fixture $(basename $fixture) beside_miner=$beside base_mib=$base"
|
|
sampler_start "$samp"
|
|
local t0=$(date +%s)
|
|
env "${env[@]}" timeout "$POINT_TIMEOUT" "$HOST" "$fixture" --mode "$mode" --shard 0 --prover "$WALLET" --out "$res" > "$log" 2>&1
|
|
local rc=$?; local wall=$(( $(date +%s) - t0 ))
|
|
sampler_stop; kill_server
|
|
local peak; peak="$(peak_of "$samp")"
|
|
local rline; rline="$(grep -E "^RESULT (core|compressed) shard" "$log" | tail -1)"
|
|
local secs; secs="$(printf '%s' "$rline" | grep -o 'prove [0-9.]* s' | grep -o '[0-9.]*' | head -1)"
|
|
local bytes; bytes="$(printf '%s' "$rline" | grep -o 'proof [0-9]* bytes' | grep -o '[0-9]*' | head -1)"
|
|
local ver="no"; printf '%s' "$rline" | grep -q 'VERIFIED' && ver="yes"; printf '%s' "$rline" | grep -q 'VERIFY FAILED' && ver="FAILED"
|
|
local err; err="$(grep -m1 -E 'Unsupported GPU memory|out of memory|OutOfMemory|CUDA_ERROR|panicked|Error:|error:' "$log" | cut -c1-200 | tr '"' "'")"
|
|
local setup; setup="$(grep -o 'RESULT setup: [0-9.]* s' "$log" | grep -o '[0-9.]* s' | head -1)"
|
|
echo "RESULT point name=$name mode=$mode thr=${thr:-default} beside=$beside rc=$rc wall_s=$wall prove_s=${secs:-} proof_bytes=${bytes:-} verified=$ver peak_mib=$peak base_mib=$base own_mib=$((peak - base)) setup=\"${setup:-}\" err=\"${err:-}\""
|
|
printf '%s\n' "{\"row\":\"point\",\"name\":\"$name\",\"home\":\"$home\",\"mode\":\"$mode\",\"threshold\":\"${thr:-default}\",\"fixture\":\"$(basename $fixture)\",\"beside_miner\":$beside,\"rc\":$rc,\"wall_s\":$wall,\"prove_s\":${secs:-null},\"proof_bytes\":${bytes:-null},\"verified\":\"$ver\",\"peak_mib\":$peak,\"base_mib\":$base,\"own_mib\":$((peak - base)),\"err\":\"${err:-}\",\"result\":\"$(printf '%s' "$rline" | cut -c1-160 | tr '"' "'")\"}" >> $ROWS
|
|
}
|
|
STOCK=/root; PATCHED=$FLOOR/home
|
|
T25=33554432; T26=67108864; T27=134217728; T24=16777216
|
|
SMALL=0; [ "$TOTAL" -lt 11000 ] && SMALL=1
|
|
|
|
# 5. the stock server, alone (the SDK downloads it into /root/.sp1/bin on the first run)
|
|
point stock-comp-v1 $STOCK compressed "" $V1 false
|
|
# 6. the patched server, alone
|
|
point alone-comp-26-v1 $PATCHED compressed $T26 $V1 false
|
|
point alone-comp-27-v1 $PATCHED compressed $T27 $V1 false
|
|
point alone-comp-26-empty $PATCHED compressed $T26 $EMPTY false
|
|
point alone-core-25-v1 $PATCHED core $T25 $V1 false
|
|
point alone-core-26-v1 $PATCHED core $T26 $V1 false
|
|
if [ $SMALL = 1 ]; then point alone-comp-25-v1 $PATCHED compressed $T25 $V1 false; point alone-core-24-v1 $PATCHED core $T24 $V1 false; fi
|
|
# 7. beside the miner
|
|
miner_start; say "miner warming 45 s for the beside rows"; sleep 45
|
|
MINER_RES="$(q memory.used)"; echo "RESULT miner_resident mem_mib=$MINER_RES mhs=$(miner_rate 4)"
|
|
point miner-comp-26-v1 $PATCHED compressed $T26 $V1 true
|
|
[ "$TOTAL" -ge 15000 ] && point miner-comp-27-v1 $PATCHED compressed $T27 $V1 true
|
|
point miner-core-25-v1 $PATCHED core $T25 $V1 true
|
|
point miner-core-26-v1 $PATCHED core $T26 $V1 true
|
|
if [ $SMALL = 1 ]; then point miner-core-24-v1 $PATCHED core $T24 $V1 true; fi
|
|
MINER_MHS_BESIDE="$(miner_rate 30)"
|
|
miner_stop
|
|
echo "RESULT miner_beside mhs_mean_during_points=$MINER_MHS_BESIDE"
|
|
|
|
python3 - "$OUT" "$LABEL" "$NAME" "$TOTAL" "$DRV" "$IDLE_MIB" "$IDLE_W" "$PLIM" <<'PY'
|
|
import json, sys
|
|
out, label, name, total, drv, idle, idle_w, plim = sys.argv[1:]
|
|
rows = [json.loads(l) for l in open(f"{out}/rows.jsonl") if l.strip()]
|
|
json.dump({"label": label, "card": name, "total_mib": int(total), "driver": drv, "idle_mib": int(idle), "idle_w": float(idle_w) if idle_w.replace(".", "").isdigit() else 0, "power_limit_w": float(plim) if plim.replace(".", "").isdigit() else 0, "rows": rows}, open(f"{out}/matrix.json", "w"), indent=1)
|
|
PY
|
|
echo "RESULT matrix_done $(stamp) points=$(grep -c '"row":"point"' $ROWS)"
|