11c4426a96
The fleet's 22:09 UK incident (a Mac-side pkill -f <log file name> matched nothing, the roll-everything script lived on and wiped a held box) and the day's two pgrep self-matches are one class. The check flags pgrep -f / pkill -f with a plain literal (every one on a line), any pgrep/pkill on a file-name shape, and ps | grep with a literal; it allows the bracket form, -x, -F pidfile, kill $(cat pidfile), a variable and a full path; 11 banned and 16 allowed shapes in its self-test; 0.15 s over the tree. The 25 pkill -f sp1-gpu-server inside bash -c bodies (which matched the calling bash) are pkill -x; the other 11 literals take the bracket form; prover-socket-check accepts both. Row R in the record; the CLAUDE.md rule names the check and covers pkill and file names. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
58 lines
4.8 KiB
PowerShell
58 lines
4.8 KiB
PowerShell
# Proving v1, the 12 GB and 16 GB requirements, part 2 (5 October 2026): the GPU memory peak of one shard proof on PC 2's
|
|
# RTX 5090 WITH THE MINER RUNNING (the mine-and-prove case), the live prover switched off for the run; the full shard
|
|
# at S_p and the empty live shard; then the sp1-gpu-server's own option names (strings, --help) to find any memory knob
|
|
# the environment did not reach in part 1. Leaves the prover ON. Never stops the miners.
|
|
$ErrorActionPreference = 'Continue'
|
|
$urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
if (-not (Test-Path $urlFile)) { $urlFile = Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
$base = (Get-Content $urlFile -Raw).Trim().TrimEnd('/')
|
|
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
|
function Prove($on) { try { (Invoke-RestMethod -Method Post -Uri "$base/api/prove" -ContentType 'application/json' -Body (@{on=$on} | ConvertTo-Json -Compress) -TimeoutSec 10) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
|
"RESULT start $(Stamp) prover off for the run: $(Prove $false)"
|
|
Start-Sleep -Seconds 45
|
|
"RESULT gpus $(Stamp) $((& nvidia-smi --query-gpu=index,name,memory.used,memory.total,utilization.gpu,power.draw --format=csv,noheader,nounits 2>$null) -join ' | ')"
|
|
$job = $env:IGNEUM_JOB_DIR; if (-not $job) { $job = Join-Path $env:TEMP 'igneum-pv1-mem2' }; New-Item -ItemType Directory -Force -Path $job | Out-Null
|
|
function WslPath($p) { $w = (& wsl.exe -d Ubuntu-24.04 -u root -- wslpath -a ($p -replace '\\', '/') 2>$null); if ($w) { ($w -replace "`0", '').Trim() } else { '/mnt/c' + ($p.Substring(2) -replace '\\', '/') } }
|
|
$jobW = WslPath $job
|
|
$kitFile = Join-Path $env:LOCALAPPDATA 'igneum\app\jobs\chain-pc2-pv1c\block-83616.json'
|
|
if (-not (Test-Path $kitFile)) { Write-Output "RESULT kit missing: $kitFile; republish the fetch after any app update (C32)"; exit 2 }
|
|
$emptyW = WslPath $kitFile
|
|
$bash = @"
|
|
set -uo pipefail
|
|
export PATH="`$HOME/.cargo/bin:`$HOME/.sp1/bin:`$PATH"
|
|
CUDA_DIR="`$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "`$CUDA_DIR" ] && export PATH="`$CUDA_DIR/bin:`$PATH" && export LD_LIBRARY_PATH="`$CUDA_DIR/lib64:/usr/lib/wsl/lib:`${LD_LIBRARY_PATH:-}"
|
|
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
JOB='$jobW'; H=/opt/igneum-pv1/igneum-prove-host; FX="`$HOME/igneum-prove-pv1/proving/fixtures"
|
|
EMPTY='$emptyW'; FULL="`$FX/block-338-shard1.json"
|
|
pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock
|
|
echo "RESULT miner_resident_mib `$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits | head -1) (the miner's working set before the prover starts)"
|
|
run() {
|
|
local name="`$1" fx="`$2"; shift 2
|
|
pkill -x sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock
|
|
local csv="`$JOB/smi-`$name-`$(basename `$fx .json).csv"
|
|
nvidia-smi --query-gpu=timestamp,memory.used,utilization.gpu --format=csv,noheader,nounits -l 1 > "`$csv" 2>/dev/null &
|
|
local SMI=`$!
|
|
local t0=`$(date +%s)
|
|
env SP1_PROVER=cuda RUST_LOG=off "`$@" `$H "`$fx" --mode compressed --shard 0 --out "`$JOB/res-`$name-`$(basename `$fx .json).json" > "`$JOB/log-`$name-`$(basename `$fx .json).txt" 2>&1
|
|
local rc=`$?
|
|
local wall=`$(( `$(date +%s) - t0 ))
|
|
kill `$SMI 2>/dev/null; sleep 1
|
|
local peak=`$(awk -F', *' '{ if (`$2+0 > m) m=`$2+0 } END { print m+0 }' "`$csv")
|
|
local line=`$(grep -E "^RESULT compressed shard" "`$JOB/log-`$name-`$(basename `$fx .json).txt" | tail -1 | sed -E 's/.*prove ([0-9.]+) s, proof ([0-9]+) bytes, verify ([0-9.]+) s, ([A-Z ]+);.*/prove_s=\1 bytes=\2 verify_s=\3 \4/')
|
|
local err=`$(grep -iE "error|panick|out of memory|OOM|CUDA" "`$JOB/log-`$name-`$(basename `$fx .json).txt" | head -1 | cut -c1-200)
|
|
echo "RESULT mineprove cfg=`$name fixture=`$(basename `$fx .json) peak_mib=`$peak samples=`$(wc -l < "`$csv") wall_s=`$wall `${line:-no_result} exit=`$rc `${err:+err=`$err}"
|
|
}
|
|
run baseline "`$EMPTY"
|
|
run baseline "`$FULL"
|
|
run baseline "`$EMPTY"
|
|
pkill -x sp1-gpu-server 2>/dev/null; sleep 1; rm -f /tmp/sp1-cuda-*.sock
|
|
S="`$HOME/.sp1/bin/sp1-gpu-server"
|
|
echo "RESULT server_help `$(`$S --help 2>&1 | tr '\n' ' ' | cut -c1-900)"
|
|
echo "RESULT server_env_names `$(strings `$S | grep -oE '^(SP1|MOONGATE|CUDA|GPU|NVIDIA|MEM|TRACE|SHARD|ELEMENT|HEIGHT|FULL)[A-Z0-9_]{3,}$' | sort -u | tr '\n' ' ' | cut -c1-1200)"
|
|
echo "RESULT server_words `$(strings `$S | grep -iE 'gpu memory|vram|out of memory|memory pool|pool size|GiB|24 ?GB|16 ?GB|12 ?GB' | sort -u | head -20 | tr '\n' '|' | cut -c1-1200)"
|
|
echo "RESULT end_wsl `$(stamp)"
|
|
"@
|
|
$bashFile = Join-Path $job 'mem2.sh'
|
|
[IO.File]::WriteAllText($bashFile, ($bash -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
|
& wsl.exe -d Ubuntu-24.04 -u root -- bash (WslPath $bashFile) 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
|
"RESULT end $(Stamp) prover back on: $(Prove $true)"
|