84 lines
8 KiB
Bash
Executable file
84 lines
8 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Phase 3 on an 8-card rig (RunPod pod, a container: no systemd, so the rig installer's unit steps are exercised with
|
|
# --preflight-only and --dry-run and the units' own scripts are run by hand). After box-setup.sh (node, workers, the
|
|
# patched server for the rig's arch, the cuda host):
|
|
# A. the rig installer: packaging/linux/install-rig.sh --preflight-only, then --dry-run with the published package
|
|
# B. eight miners on the devnet, one per card, 150 s: MH/s per card and the rig's sum, watts
|
|
# C. seven core-only provers in parallel (one sp1-gpu-server per card, CUDA_VISIBLE_DEVICES=n, socket /tmp/sp1-cuda-n),
|
|
# the v1 shard at 2^25 in a loop for PROVE_SECS: core proofs per minute for the rig and per card, host RAM
|
|
# D. the eighth card compressing the v1 shard in a loop for PROVE_SECS: compressed proofs per minute (the compression
|
|
# throughput of one card)
|
|
# E. the hand-off cost by file: a core proof (2^25, about 25 MB) copied card-to-card over the rig's disk and read back
|
|
# F. the aggregator: --mode chain over 8 block fixtures on the big card (the 8-block chain: shards plus aggregation)
|
|
# Every result a RESULT line in /root/fleet/out/rig.log; rig.json at the end.
|
|
set -uo pipefail
|
|
F=/root/fleet; OUT=$F/out; B=/opt/igneum/pkg/bin; FLOOR=/opt/igneum-floor; HOST=$FLOOR/bin/igneum-prove-host
|
|
FIX=$FLOOR/prove/proving/fixtures; V1=$FIX/fees-v1-shards2.json
|
|
LABEL="${LABEL:-rig}"; WALLET="${WALLET:-0x1919191919191919191919191919191919191919}"; PROVE_SECS="${PROVE_SECS:-300}"; MINE_SECS="${MINE_SECS:-150}"
|
|
mkdir -p $OUT $F/mine; exec >> $OUT/rig.log 2>&1
|
|
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
N=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
|
|
echo "RESULT rig_start $(stamp) label=$LABEL cards=$N $(nvidia-smi --query-gpu=name,memory.total --format=csv,noheader | head -1) host_ram_gb=$(( $(awk '/MemTotal/{print $2}' /proc/meminfo) / 1048576 )) cores=$(nproc)"
|
|
for b in igneum-prove-host igneum-prove-export; do for d in $FLOOR/target/release $FLOOR/prove/proving/igneum-prove/target/release; do [ -x $d/$b ] && ln -sfn $d/$b $FLOOR/bin/$b; done; done
|
|
# A. the installer's real parts
|
|
if [ -d $F/in/linux ] && [ -z "${PROVE_ONLY:-}" ]; then
|
|
bash $F/in/linux/install-rig.sh --preflight-only > $OUT/rig-preflight.log 2>&1; echo "RESULT installer_preflight exit=$? fails=$(grep -c 'FAIL' $OUT/rig-preflight.log) warns=$(grep -c 'warn' $OUT/rig-preflight.log) cards=$(grep -c '^ card' $OUT/rig-preflight.log)"
|
|
bash $F/in/linux/install-rig.sh --dry-run --yes --wallet "$WALLET" --name "$LABEL" --package-url https://dl.igneum.network/dl/public/igneum-hive-0.3.12.tar.gz --package-sha256 7972af92e7cd9a032303eca4d95b533f53e0e68d1b9cae5bfe406a5b7c30a454 --package-size 24506282 > $OUT/rig-dryrun.log 2>&1; echo "RESULT installer_dryrun exit=$? steps=$(grep -c '^\[dry-run\]' $OUT/rig-dryrun.log) manifest=$(grep -c 'signature verifies' $OUT/rig-dryrun.log) package=$(grep -c 'sha256 7972af92' $OUT/rig-dryrun.log) prover_decision=\"$(grep -o 'prover: .*' $OUT/rig-dryrun.log | head -1 | cut -c1-120)\""
|
|
fi
|
|
# B. eight miners
|
|
if [ -z "${PROVE_ONLY:-}" ]; then
|
|
cd $F/mine && rm -rf packs/devnet && $B/igneum-miner export-pack grpc://127.0.0.1:26610 packs/devnet > $OUT/rig-export-pack.log 2>&1
|
|
pids=()
|
|
for d in $(seq 0 $((N-1))); do
|
|
nohup $B/igneum-miner mine grpc://127.0.0.1:26610 1 100000000 "$LABEL-gpu$d" --worker $B/igneum-worker-cuda --worker-args "--device $d --pack packs/devnet" --prepare-packs packs/prepare-$d --exit-on-seed-change --evm-address "$WALLET" --payout-label "$LABEL-gpu$d" --status-secs 10 > $OUT/rig-miner-$d.log 2>&1 &
|
|
pids+=($!)
|
|
done
|
|
cd $F; sleep 60
|
|
( while :; do nvidia-smi --query-gpu=index,power.draw,memory.used,utilization.gpu --format=csv,noheader,nounits; sleep 1; done ) > $OUT/rig-samp-mine.csv & SP=$!
|
|
sleep "$MINE_SECS"; pkill -P $SP; kill $SP 2>/dev/null
|
|
sum=0
|
|
for d in $(seq 0 $((N-1))); do
|
|
r=$(grep STATUS $OUT/rig-miner-$d.log | grep -o ' now=[0-9.]*' | tail -n 10 | cut -d= -f2 | awk '{s+=$1; n++} END {if (n) printf "%.2f", s/n; else print 0}')
|
|
w=$(awk -F', *' -v i=$d '$1==i {s+=$2; n++} END {if (n) printf "%.0f", s/n; else print 0}' $OUT/rig-samp-mine.csv)
|
|
echo "RESULT miner card=$d mhs=$r watts=$w"; sum=$(awk -v a=$sum -v b=$r 'BEGIN {print a+b}')
|
|
done
|
|
echo "RESULT rig_miners $(stamp) cards=$N sum_mhs=$sum watts=$(awk -F', *' '{s+=$2; n++} END {printf "%.0f", s/n*'$N'}' $OUT/rig-samp-mine.csv)"
|
|
for p in "${pids[@]}"; do kill $p 2>/dev/null; done; pkill -f '^/opt/igneum/pkg/bin/igneum-worker-cuda'; sleep 3
|
|
fi
|
|
# C + D. seven core-only provers and one compressing card, in parallel
|
|
pkill -9 -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock
|
|
prove_loop() { # <device> <mode> <threshold> <seconds>
|
|
local d=$1 mode=$2 thr=$3 secs=$4 t0=$(date +%s) n=0 tot=0
|
|
while [ $(( $(date +%s) - t0 )) -lt $secs ]; do
|
|
env HOME=$FLOOR/home IGNEUM_CUDA_DEVICE=$d SP1_PROVER=cuda RUST_LOG=off SP1_GPU_ELEMENT_THRESHOLD=$thr timeout 600 $HOST $V1 --mode $mode --shard 0 --prover "$WALLET" --out $OUT/rig-$mode-$d.json > $OUT/rig-$mode-$d.log 2>&1
|
|
s=$(grep -E "^RESULT ($mode) shard" $OUT/rig-$mode-$d.log | tail -1 | grep -o 'prove [0-9.]* s' | grep -o '[0-9.]*' | head -1)
|
|
v=$(grep -E "^RESULT ($mode) shard" $OUT/rig-$mode-$d.log | tail -1 | grep -c VERIFIED)
|
|
[ "$v" = 1 ] && { n=$((n+1)); tot=$(awk -v a=$tot -v b=${s:-0} 'BEGIN {print a+b}'); }
|
|
echo "$(stamp) card=$d $mode n=$n last_s=${s:-fail} verified=$v" >> $OUT/rig-loop-$d.log
|
|
done
|
|
echo "RESULT prove_loop card=$d mode=$mode threshold=$thr seconds=$secs proofs=$n mean_prove_s=$(awk -v a=$tot -v n=$n 'BEGIN {if (n) printf "%.1f", a/n; else print 0}') per_min=$(awk -v n=$n -v s=$secs 'BEGIN {printf "%.2f", n*60/s}')"
|
|
}
|
|
( while :; do echo "$(date +%s) $(free -m | awk '/Mem:/{print $3}') $(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | tr '\n' ',')"; sleep 5; done ) > $OUT/rig-samp-prove.csv & SP=$!
|
|
lp=()
|
|
for d in $(seq 0 $((N-2))); do prove_loop $d core 33554432 "$PROVE_SECS" & lp+=($!); done
|
|
prove_loop $((N-1)) compressed 67108864 "$PROVE_SECS" & lp+=($!)
|
|
for p in "${lp[@]}"; do wait $p; done
|
|
pkill -P $SP; kill $SP 2>/dev/null
|
|
echo "RESULT rig_prove $(stamp) host_ram_used_mb_max=$(awk '{if ($2>m) m=$2} END {print m}' $OUT/rig-samp-prove.csv) core_cards=$((N-1)) core_per_min=$(grep 'RESULT prove_loop' $OUT/rig.log | grep 'mode=core' | grep -o 'per_min=[0-9.]*' | cut -d= -f2 | awk '{s+=$1} END {printf "%.2f", s}') compressed_per_min=$(grep 'RESULT prove_loop' $OUT/rig.log | grep 'mode=compressed' | grep -o 'per_min=[0-9.]*' | cut -d= -f2)"
|
|
pkill -9 -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock
|
|
# E. the hand-off cost by file
|
|
cf=$(ls -S $F/mine/*.bin $OUT/*.bin 2>/dev/null | head -1); [ -z "$cf" ] && cf=$(find / -name 'block-*-core.bin' -size +1M 2>/dev/null | head -1)
|
|
if [ -n "$cf" ]; then
|
|
sz=$(stat -c %s "$cf"); t0=$(date +%s%N); cp "$cf" /tmp/handoff.bin; sync; t1=$(date +%s%N); cat /tmp/handoff.bin > /dev/null; t2=$(date +%s%N)
|
|
echo "RESULT handoff file=$(basename $cf) bytes=$sz copy_ms=$(( (t1-t0)/1000000 )) read_ms=$(( (t2-t1)/1000000 )) loopback_tcp_ms=$( (nc -l -p 29999 > /dev/null & sleep 0.3; t=$(date +%s%N); nc -q 0 127.0.0.1 29999 < /tmp/handoff.bin; echo $(( ($(date +%s%N)-t)/1000000 )) ) 2>/dev/null)"
|
|
fi
|
|
# F. the 8-block chain on the big card: --mode chain needs CONSECUTIVE live blocks (the chain rule checks number and parent hash), so this
|
|
# runs only when RIG_CHAIN_LIST names eight consecutive fixtures cut from the box's node (after the exec layer executes); else skipped
|
|
list="${RIG_CHAIN_LIST:-}"
|
|
if [ -n "$list" ]; then
|
|
t0=$(date +%s)
|
|
env HOME=$FLOOR/home IGNEUM_CUDA_DEVICE=$((N-1)) SP1_PROVER=cuda RUST_LOG=off timeout 1800 $HOST --mode chain --chain "$list" --prover "$WALLET" --save-shards --out $OUT/rig-chain.json > $OUT/rig-chain.log 2>&1; rc=$?
|
|
echo "RESULT chain rc=$rc wall_s=$(( $(date +%s) - t0 )) blocks=$(echo "$list" | tr ',' '\n' | wc -l) $(grep -E '^RESULT chain' $OUT/rig-chain.log | tail -1 | cut -c1-200)"
|
|
else echo "RESULT chain skipped: no consecutive live fixtures yet (needs the exec layer)"; fi
|
|
pkill -9 -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock
|
|
echo "RESULT rig_done $(stamp)"
|