127 lines
9.9 KiB
PowerShell
127 lines
9.9 KiB
PowerShell
# Counter ASIC 3.0 item 8 (6 October 2026): program work in the latency shadow on PC 2's RTX 5090 (machine 1ccfe586),
|
|
# docs/analysis/latency-shadow-2026-10-06.md. A signed `run` job published with --stop-miners: the app stops its miners
|
|
# before this script starts and restarts them when it ends (app/igneum-app/src/jobrun.rs, stop_miners_first); the
|
|
# script never quits, restarts or updates the app. It switches the prover off for the run and back on at the end (as
|
|
# tools/prover-floor/pc2-floor-measure.ps1 does), confirms the card is empty by nvidia-smi's compute-apps list (never
|
|
# api/state, which answers {} on this machine), then runs the INSTALLED igneum-worker-cuda.exe (NVRTC, the pack's own
|
|
# kernel text) with --bench on every pack of the fetched kit (fetch job fetch-ca3-shadow-20261006: the class v3 control
|
|
# mx8-genesis, run first and last, and the shadow ladder sh256x2 .. sh256x88, sh64x52, sh1024x3) while `nvidia-smi -l 1` samples
|
|
# power.draw, utilization.gpu, clocks.sm, clocks.mem and temperature every second into a file. Every number is a RESULT
|
|
# line: the worker's own lines (vectors, fingerprint, MH/s), and per pack the mean, min and max power, the mean
|
|
# utilisation and clocks over the bench's own window (the timed dispatches), plus every raw sample. No power cap is set
|
|
# (the 5 October sweep on this card, counter-asic-2-status.md 22:16, showed the cap never binds above the 400 W floor and
|
|
# -pl 200 / 250 are below power.min_limit); the core clock is locked for the clock rows only if nvidia-smi allows it
|
|
# unprompted (else OWED); nothing elevated. Read back with `node tools/jobs.mjs <job id>`.
|
|
$ErrorActionPreference = 'Continue'
|
|
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
|
$urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
if (-not (Test-Path $urlFile)) { $urlFile = Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
$base = $null
|
|
if (Test-Path $urlFile) { $base = (Get-Content $urlFile -Raw).Trim().TrimEnd('/') }
|
|
function Prove($on) { if (-not $base) { return 'no app.url' }; try { (Invoke-RestMethod -Method Post -Uri "$base/api/prove" -ContentType 'application/json' -Body (@{on=$on} | ConvertTo-Json -Compress) -TimeoutSec 10) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
|
function Smi($q) { try { (& nvidia-smi --query-gpu=$q --format=csv,noheader,nounits 2>$null) -join ' | ' } catch { 'nvidia-smi failed' } }
|
|
function CudaApps { @(& nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv,noheader 2>$null | ForEach-Object { "$_" } | Where-Object { $_ -match 'igneum|sp1|prove' }) }
|
|
"RESULT start $(Stamp) prover off for the run: $(Prove $false)"
|
|
Start-Sleep -Seconds 20
|
|
$t = 0; $apps = @(CudaApps)
|
|
while ($t -lt 120 -and $apps.Count -gt 0) { Start-Sleep -Seconds 10; $t += 10; $apps = @(CudaApps) }
|
|
if ($apps.Count -eq 0) { "RESULT card $(Stamp) empty after $t s (no igneum, sp1 or prove compute app on the card)" } else { "RESULT card $(Stamp) UNCONFIRMED after $t s: the numbers below are beside " + ($apps -join '; ') }
|
|
"RESULT gpus $(Stamp) $(Smi 'index,name,driver_version,memory.used,clocks.sm,clocks.mem,power.draw,power.limit,power.min_limit,power.max_limit,temperature.gpu')"
|
|
|
|
# the kit: the fetch job's folder under the app's jobs folder (tested before use: the wiped-jobs-folder class)
|
|
$jobs = Split-Path $env:IGNEUM_JOB_DIR
|
|
$kit = Join-Path $jobs 'fetch-ca3-shadow-20261006'
|
|
$packs = Join-Path $kit 'packs-ca3-shadow'
|
|
if (-not (Test-Path $packs)) { "RESULT error packs missing at $packs (the fetch job fetch-ca3-shadow-20261006 runs first; republish it after any app update)"; "RESULT end $(Stamp) prover back on: $(Prove $true)"; exit 2 }
|
|
$inst = @("$env:LOCALAPPDATA\Programs\Igneum Miner", "$env:ProgramFiles\Igneum Miner") | Where-Object { Test-Path (Join-Path $_ 'igneum-worker-cuda.exe') } | Select-Object -First 1
|
|
if (-not $inst) { "RESULT error no installed igneum-worker-cuda.exe"; "RESULT end $(Stamp) prover back on: $(Prove $true)"; exit 2 }
|
|
$exe = Join-Path $inst 'igneum-worker-cuda.exe'
|
|
"RESULT worker $exe sha256 $((Get-FileHash -Algorithm SHA256 $exe).Hash.ToLower()) bytes $((Get-Item $exe).Length)"
|
|
foreach ($f in (Get-ChildItem $packs -Recurse -File | Sort-Object FullName)) { "RESULT kitfile $($f.FullName.Substring($packs.Length + 1).Replace('\', '/')) sha256 $((Get-FileHash -Algorithm SHA256 $f.FullName).Hash.ToLower()) bytes $($f.Length)" }
|
|
|
|
# the sampler: nvidia-smi every second into a file for the whole run
|
|
$samples = Join-Path $env:IGNEUM_JOB_DIR 'smi-samples.csv'
|
|
$smiExe = (Get-Command nvidia-smi -ErrorAction SilentlyContinue).Source
|
|
if (-not $smiExe) { $smiExe = Join-Path $env:SystemRoot 'System32\nvidia-smi.exe' }
|
|
$sampler = Start-Process -FilePath $smiExe -ArgumentList @('--query-gpu=timestamp,power.draw,utilization.gpu,clocks.sm,clocks.mem,temperature.gpu,memory.used', '--format=csv,noheader,nounits', '-l', '1') -RedirectStandardOutput $samples -NoNewWindow -PassThru
|
|
Start-Sleep -Seconds 5
|
|
$windows = @()
|
|
$order = @('mx8-genesis', 'sh256x2', 'sh256x7', 'sh256x13', 'sh256x27', 'sh256x40', 'sh256x53', 'sh256x88', 'sh64x52', 'sh1024x3', 'mx8-genesis')
|
|
try {
|
|
"RESULT idle $(Stamp) 10 s before the first bench: $(Smi 'power.draw,utilization.gpu,clocks.sm,clocks.mem')"
|
|
Start-Sleep -Seconds 10
|
|
foreach ($pk in $order) {
|
|
$d = Join-Path $packs $pk
|
|
if (-not (Test-Path $d)) { "RESULT bench $pk missing"; continue }
|
|
foreach ($bw in @(1, 8)) {
|
|
if ($bw -eq 8 -and $pk -notin @('mx8-genesis', 'sh256x13', 'sh256x88')) { continue }
|
|
$batches = if ($bw -eq 1) { 250 } else { 120 }
|
|
$t0 = Get-Date
|
|
"RESULT bench $pk bw$bw start $(Stamp)"
|
|
& $exe --bench --pack $d --batches $batches --batch-log2 24 --block-warps $bw 2>&1 | ForEach-Object { "RESULT bench $pk bw$bw $_" }
|
|
$t1 = Get-Date
|
|
"RESULT bench $pk bw$bw end $(Stamp) exit=$LASTEXITCODE wall_s=$([int]($t1 - $t0).TotalSeconds)"
|
|
$windows += [pscustomobject]@{ pack = $pk; bw = "bw$bw"; t0 = $t0; t1 = $t1 }
|
|
if ($pk -eq 'mx8-genesis' -and $bw -eq 1 -and $windows.Count -gt 2) { break }
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
}
|
|
# the core-clock rows (coordinator, 6 October 2026 08:0x UTC): the 5 October sweep showed `-pl` cannot bind on this kernel
|
|
# (the card draws about 290 to 316 W under a 400 W floor), so the lever that can move the watts is the core clock. Four
|
|
# caps of the Ember plan, at the control and at the largest N that fits the verifier gate, about 75 s each. If
|
|
# nvidia-smi refuses the lock (it needs administrator rights this job does not have and must not ask for), the rows
|
|
# are OWED and nothing else is tried; -rgc runs in the finally block whatever happens.
|
|
$clockOk = $true
|
|
foreach ($pk in @('mx8-genesis', 'sh256x88')) {
|
|
if (-not $clockOk) { break }
|
|
$d = Join-Path $packs $pk
|
|
if (-not (Test-Path $d)) { continue }
|
|
foreach ($mhz in @(2781, 2472, 2163, 1854)) {
|
|
$r = (& nvidia-smi -i 0 -lgc 0,$mhz 2>&1 | Out-String).Trim()
|
|
"RESULT clock $pk $mhz lgc: $($r -replace "`r?`n", ' | ')"
|
|
if ($r -match 'Insufficient|[Pp]ermission|denied|administrator|not supported|Unknown Error|No such') { "RESULT clock OWED: nvidia-smi -lgc refused ($mhz MHz); the rows need rights this job does not have and does not ask for"; $clockOk = $false; break }
|
|
Start-Sleep -Seconds 3
|
|
"RESULT clock $pk $mhz readback: $(Smi 'clocks.sm,clocks.max.sm,clocks.mem,power.draw,power.limit')"
|
|
$t0 = Get-Date
|
|
"RESULT bench $pk clk$mhz start $(Stamp)"
|
|
& $exe --bench --pack $d --batches 600 --batch-log2 24 --block-warps 1 2>&1 | ForEach-Object { "RESULT bench $pk clk$mhz $_" }
|
|
$t1 = Get-Date
|
|
"RESULT bench $pk clk$mhz end $(Stamp) exit=$LASTEXITCODE wall_s=$([int]($t1 - $t0).TotalSeconds)"
|
|
$windows += [pscustomobject]@{ pack = $pk; bw = "clk$mhz"; t0 = $t0; t1 = $t1 }
|
|
Start-Sleep -Seconds 3
|
|
}
|
|
}
|
|
} finally {
|
|
$r = (& nvidia-smi -i 0 -rgc 2>&1 | Out-String).Trim()
|
|
"RESULT clock reset rgc: $($r -replace "`r?`n", ' | ') readback: $(Smi 'clocks.sm,clocks.max.sm,power.limit')"
|
|
"RESULT prover back on (finally): $(Prove $true)"
|
|
Start-Sleep -Seconds 2
|
|
try { Stop-Process -Id $sampler.Id -Force -ErrorAction SilentlyContinue } catch { }
|
|
Start-Sleep -Seconds 1
|
|
}
|
|
# per-window power: the samples inside the bench's window after its first 12 s (compile, fills, warm-up) and before its last 2 s
|
|
$rows = @()
|
|
if (Test-Path $samples) {
|
|
foreach ($l in (Get-Content $samples)) {
|
|
$p = $l -split ',\s*'
|
|
if ($p.Count -lt 7) { continue }
|
|
try { $ts = [datetime]::ParseExact($p[0].Trim(), 'yyyy/MM/dd HH:mm:ss.fff', $null) } catch { continue }
|
|
$rows += [pscustomobject]@{ ts = $ts; w = [double]$p[1]; util = [double]$p[2]; sm = [double]$p[3]; mem = [double]$p[4]; temp = [double]$p[5]; used = [double]$p[6] }
|
|
}
|
|
}
|
|
"RESULT samples total $($rows.Count) file $samples"
|
|
foreach ($win in $windows) {
|
|
$in = @($rows | Where-Object { $_.ts -gt $win.t0.AddSeconds(12) -and $_.ts -lt $win.t1.AddSeconds(-2) })
|
|
if ($in.Count -gt 0) {
|
|
$w = $in | Measure-Object -Property w -Average -Minimum -Maximum
|
|
$u = ($in | Measure-Object -Property util -Average).Average
|
|
$s = ($in | Measure-Object -Property sm -Average).Average
|
|
$m = ($in | Measure-Object -Property mem -Average).Average
|
|
$tp = ($in | Measure-Object -Property temp -Maximum).Maximum
|
|
"RESULT power $($win.pack) $($win.bw) samples=$($in.Count) watts_mean=$([math]::Round($w.Average, 1)) watts_min=$($w.Minimum) watts_max=$($w.Maximum) util_mean=$([math]::Round($u, 1)) sm_mhz_mean=$([math]::Round($s)) mem_mhz_mean=$([math]::Round($m)) temp_max=$tp window_s=$([int]($win.t1 - $win.t0).TotalSeconds)"
|
|
} else { "RESULT power $($win.pack) $($win.bw) no samples in window" }
|
|
}
|
|
foreach ($r in $rows) { "RESULT sample $($r.ts.ToString('HH:mm:ss')) $($r.w) $($r.util) $($r.sm) $($r.mem) $($r.temp) $($r.used)" }
|
|
"RESULT gpus_after $(Stamp) $(Smi 'power.draw,power.limit,clocks.sm,clocks.mem,temperature.gpu,memory.used')"
|
|
"RESULT end $(Stamp) prover state: $(Prove $true)"
|
|
exit 0
|