117 lines
9.9 KiB
PowerShell
117 lines
9.9 KiB
PowerShell
# Proving v1, step 1 (5 October 2026): what the prover costs a mining machine. A signed `run` job for PC 2
|
|
# (app/igneum-app/src/jobrun.rs, shell powershell, not elevated). Never stops the miners.
|
|
# 1. waits (up to 20 min) for the NVIDIA worker to report a hash rate, so both phases see the same miner
|
|
# 2. prover OFF (POST /api/prove {"on":false}): 5 min of samples every 15 s
|
|
# 3. prover ON: 5 min of samples; a 1-s nvidia-smi sampler runs underneath for the GPU memory peak
|
|
# 4. inside WSL2: the sp1-gpu-server's version and its compiled SM targets (cuobjdump, strings)
|
|
# 5. leaves the prover ON
|
|
# Every number is a RESULT line; the per-sample lines are SAMPLE lines. Read with `node tools/jobs.mjs <id>`.
|
|
$ErrorActionPreference = 'Continue'
|
|
$urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
if (-not (Test-Path $urlFile)) { $urlFile = Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
$base = (Get-Content $urlFile -Raw).Trim().TrimEnd('/')
|
|
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
|
$script:stateErr = ''
|
|
function State { try { Invoke-RestMethod -Uri "$base/api/state" -TimeoutSec 20 } catch { $script:stateErr = "$_"; $null } }
|
|
"RESULT app_url_file $(Stamp) $urlFile exists=$(Test-Path $urlFile) base_len=$($base.Length)"
|
|
$probe = State
|
|
"RESULT state_probe $(Stamp) ok=$($null -ne $probe) version=$($probe.version) cards=$(($probe.mining.cards | Measure-Object).Count) err=$script:stateErr"
|
|
function Prove($on) { try { (Invoke-RestMethod -Method Post -Uri "$base/api/prove" -ContentType 'application/json' -Body (@{on=$on} | ConvertTo-Json -Compress) -TimeoutSec 10) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
|
function Rpc($port, $method, $params) {
|
|
try { (Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$port" -ContentType 'application/json' -Body (@{jsonrpc='2.0'; id=1; method=$method; params=$params} | ConvertTo-Json -Compress -Depth 6) -TimeoutSec 20).result } catch { $null }
|
|
}
|
|
function Smi { try { (& nvidia-smi --query-gpu=index,name,memory.used,memory.total,utilization.gpu,power.draw --format=csv,noheader,nounits 2>$null) -join ' | ' } catch { 'nvidia-smi failed' } }
|
|
function Ram {
|
|
$os = Get-CimInstance Win32_OperatingSystem
|
|
$usedMb = [math]::Round(($os.TotalVisibleMemorySize - $os.FreePhysicalMemory) / 1024)
|
|
$vm = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'vmmem*' } | Measure-Object WorkingSet64 -Sum).Sum
|
|
$vmMb = if ($vm) { [math]::Round($vm / 1MB) } else { 0 }
|
|
"host_used_mb=$usedMb total_mb=$([math]::Round($os.TotalVisibleMemorySize / 1024)) vmmem_mb=$vmMb"
|
|
}
|
|
function Card($st) { if ($null -eq $st) { return $null }; $st.mining.cards | Where-Object { $_.vendor -eq 'nvidia' } | Select-Object -First 1 }
|
|
function Sample($phase, $i) {
|
|
$st = State; $c = Card $st; $pv = if ($st) { $st.proving } else { $null }
|
|
$cards = if ($st) { ($st.mining.cards | ForEach-Object { "$($_.name):$($_.state):$([math]::Round($_.hash_now,1))" }) -join ',' } else { 'no state' }
|
|
"SAMPLE $phase $i $(Stamp) nvidia_hash_now=$(if ($c) { [math]::Round($c.hash_now,2) } else { 'na' }) nvidia_hash_avg=$(if ($c) { [math]::Round($c.hash_avg,2) } else { 'na' }) nvidia_state=$(if ($c) { $c.state } else { 'na' }) restarts=$(if ($c) { $c.restarts } else { 'na' }) cards=$cards proving=$(if ($pv) { "$($pv.status)/proved=$($pv.proved)/submitted=$($pv.submitted)/last_prove_s=$([math]::Round($pv.last_prove_s,1))/current='$($pv.current)'" } else { 'na' }) smi=[$(Smi)] $(Ram)"
|
|
}
|
|
$evmPort = $null
|
|
foreach ($p in 26790, 26800, 26810) { if (Rpc $p 'igneum_getProvingStatus' @()) { $evmPort = $p; break } }
|
|
$st0 = State
|
|
"RESULT start $(Stamp) app $($st0.version) machine $($st0.machine_id) node_evm_port=$evmPort prove_setting=$($st0.settings.prove) proving_status=$($st0.proving.status)/$($st0.proving.backend)"
|
|
"RESULT gpus $(Stamp) $(Smi)"
|
|
# 1. the NVIDIA worker must be hashing before anything is measured (5 October 2026, 18:35Z: the 5090 worker was
|
|
# exiting on a pack seed mismatch; a baseline of 0 MH/s is not a baseline)
|
|
$waited = 0
|
|
while ($waited -lt 1200) {
|
|
$c = Card (State)
|
|
if ($c -and $c.hash_now -gt 10) { break }
|
|
if ($waited % 120 -eq 0) { "WAIT $(Stamp) nvidia worker: state=$(if ($c) { $c.state } else { 'na' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' }) restarts=$(if ($c) { $c.restarts } else { 'na' }) (waiting up to 20 min)" }
|
|
Start-Sleep -Seconds 20; $waited += 20
|
|
}
|
|
$c = Card (State)
|
|
$minerUp = ($c -and $c.hash_now -gt 10)
|
|
"RESULT miner $(Stamp) nvidia worker $(if ($minerUp) { 'hashing' } else { 'NOT hashing after 20 min: the mining-cost rows below are void' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' }) after waiting $waited s"
|
|
$paid0 = Rpc $evmPort 'igneum_getProvingStatus' @()
|
|
# 2. prover off
|
|
"RESULT prove_off $(Stamp) $(Prove $false)"
|
|
Start-Sleep -Seconds 30
|
|
$off = @(); for ($i = 1; $i -le 20; $i++) { $line = Sample 'off' $i; $line; $off += (State); Start-Sleep -Seconds 15 }
|
|
$pvOffEnd = (State).proving
|
|
# 3. prover on, with a 1-s GPU memory sampler underneath
|
|
$smiFile = Join-Path $env:TEMP "igneum-pv1-smi-$(Get-Date -Format yyyyMMdd-HHmmss).csv"
|
|
$smiProc = Start-Process -FilePath 'nvidia-smi' -ArgumentList '--query-gpu=timestamp,index,memory.used,utilization.gpu,power.draw --format=csv,noheader,nounits -l 1' -RedirectStandardOutput $smiFile -NoNewWindow -PassThru
|
|
"RESULT prove_on $(Stamp) $(Prove $true) (gpu sampler pid $($smiProc.Id) -> $smiFile)"
|
|
$on = @(); for ($i = 1; $i -le 20; $i++) { $line = Sample 'on' $i; $line; $on += (State); Start-Sleep -Seconds 15 }
|
|
try { Stop-Process -Id $smiProc.Id -Force -ErrorAction SilentlyContinue } catch {}
|
|
Start-Sleep -Seconds 2
|
|
$pvOnEnd = (State).proving
|
|
$paid1 = Rpc $evmPort 'igneum_getProvingStatus' @()
|
|
function Stats($states) {
|
|
$h = @(); foreach ($s in $states) { $c = Card $s; if ($c) { $h += [double]$c.hash_now } }
|
|
if ($h.Count -eq 0) { return 'n=0' }
|
|
$m = ($h | Measure-Object -Average -Minimum -Maximum)
|
|
$sorted = $h | Sort-Object; $p50 = $sorted[[math]::Floor(($sorted.Count - 1) / 2)]
|
|
"n=$($h.Count) mean=$([math]::Round($m.Average,2)) p50=$([math]::Round($p50,2)) min=$([math]::Round($m.Minimum,2)) max=$([math]::Round($m.Maximum,2)) MH/s"
|
|
}
|
|
"RESULT mining_alone $(Stamp) nvidia hash_now over 5 min: $(Stats $off); proved during the phase: $($pvOffEnd.proved - $off[0].proving.proved)"
|
|
"RESULT mining_and_proving $(Stamp) nvidia hash_now over 5 min: $(Stats $on); proved during the phase: $($pvOnEnd.proved - $on[0].proving.proved) shards (submitted $($pvOnEnd.submitted - $on[0].proving.submitted)), last_prove_s=$($pvOnEnd.last_prove_s); node paidShards $($paid0.paidShards) -> $($paid1.paidShards)"
|
|
# the GPU memory peak from the 1-s sampler: per card index, max memory.used (MiB) and the sample count
|
|
try {
|
|
$rows = Get-Content $smiFile | Where-Object { $_ -match ',' } | ForEach-Object { $f = $_ -split ',\s*'; [pscustomobject]@{ t = $f[0]; idx = $f[1]; mem = [int]$f[2]; util = [int]$f[3]; power = [double]$f[4] } }
|
|
foreach ($g in ($rows | Group-Object idx)) {
|
|
$mx = ($g.Group | Measure-Object mem -Maximum).Maximum; $mn = ($g.Group | Measure-Object mem -Minimum).Minimum; $ut = ($g.Group | Measure-Object util -Average).Average; $pw = ($g.Group | Measure-Object power -Maximum).Maximum
|
|
"RESULT gpu_memory_peak $(Stamp) index=$($g.Name) samples=$($g.Count) memory_used_min_mib=$mn memory_used_max_mib=$mx util_mean_pct=$([math]::Round($ut,1)) power_max_w=$pw (1-s nvidia-smi samples while mining with the prover on)"
|
|
}
|
|
} catch { "RESULT gpu_memory_peak error: $_" }
|
|
# host RAM peak over both phases, from the 15-s samples (host used and the WSL2 VM's working set)
|
|
$ramPeakOff = 0; $vmPeakOff = 0; $ramPeakOn = 0; $vmPeakOn = 0
|
|
# (re-sampled here from the SAMPLE lines' fields was simpler; the per-sample Ram() strings are above; compute again from a fresh sample for the record)
|
|
"RESULT ram_now $(Stamp) $(Ram)"
|
|
# 4. the SP1 GPU server inside WSL2: version and compiled SM targets
|
|
$bash = @'
|
|
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
|
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH"
|
|
f="$HOME/.sp1/bin/sp1-gpu-server"
|
|
echo "RESULT gpu_server_file $(ls -la "$f" 2>&1)"
|
|
echo "RESULT gpu_server_version $("$f" --version 2>&1 | head -2 | tr '\n' ' ')"
|
|
echo "RESULT gpu_server_sha256 $(sha256sum "$f" 2>/dev/null | cut -c1-64)"
|
|
echo "RESULT nvcc $(nvcc --version 2>&1 | tail -1)"
|
|
echo "RESULT wsl_nvidia_smi $(nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader 2>&1 | head -1)"
|
|
if command -v cuobjdump >/dev/null; then
|
|
echo "RESULT cuobjdump_elf $(cuobjdump --list-elf "$f" 2>&1 | grep -oE 'sm_[0-9]+' | sort | uniq -c | tr '\n' ' ')"
|
|
echo "RESULT cuobjdump_ptx $(cuobjdump --list-ptx "$f" 2>&1 | grep -oE 'sm_[0-9]+|compute_[0-9]+' | sort | uniq -c | tr '\n' ' ')"
|
|
echo "RESULT cuobjdump_head $(cuobjdump --list-elf "$f" 2>&1 | head -5 | tr '\n' ' ')"
|
|
else
|
|
echo "RESULT cuobjdump missing on PATH ($PATH)"
|
|
fi
|
|
echo "RESULT strings_sm $(strings "$f" 2>/dev/null | grep -oE '\b(sm|compute)_[0-9]{2,3}\b' | sort | uniq -c | tr '\n' ' ')"
|
|
echo "RESULT strings_arch_hints $(strings "$f" 2>/dev/null | grep -iE 'cuda_arch|gencode|arch=|--generate-code|nvcc' | sort -u | head -8 | tr '\n' ' ' | cut -c1-600)"
|
|
echo "RESULT host_elf_ids $(/opt/igneum/igneum-prove-host --mode id 2>&1 | tail -1)"
|
|
echo "RESULT free $(free -m | awk '/Mem:/ {print "wsl_total_mb="$2" used_mb="$3}')"
|
|
'@
|
|
$bashFile = Join-Path $env:TEMP 'igneum-pv1-arch.sh'
|
|
[IO.File]::WriteAllText($bashFile, ($bash -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
|
$wslPath = (& wsl.exe -d Ubuntu-24.04 -u root -- wslpath -a ($bashFile -replace '\\', '/') 2>$null)
|
|
if (-not $wslPath) { $wslPath = '/mnt/c' + ($bashFile.Substring(2) -replace '\\', '/') }
|
|
& wsl.exe -d Ubuntu-24.04 -u root -- bash $wslPath 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
|
"RESULT end $(Stamp) prover left ON: $(Prove $true); app proving status $((State).proving.status)"
|