95707125c5
The dated entries of the 0.3.14 gate expired at 0.3.15. pc2-agg-cost.ps1 no longer switches the 5090 or falls back to /api/pause: the job is published with --cards-off <nvidia key>, the runner switches the card off before the script and puts it back on any exit, and the script fails loud (exit 3) if a CUDA worker is still running. pc2-agg-cost-restore.ps1 keeps the stray and socket clean-up and the read-back only. ember-tune-pc1.ps1 is superseded by the installed-tune playbook on ember-tune. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
49 lines
4.7 KiB
PowerShell
49 lines
4.7 KiB
PowerShell
# Aggregation cost, the restore job (5 October 2026, night): what pc2-agg-cost.ps1's finally block does, for the case
|
|
# where the app restarted under the job (the 0.3.10 rollout; a running job dies with the app and its finally block
|
|
# never runs): stops any miner or CUDA worker the job started itself (a process whose parent is not igneum-app.exe),
|
|
# resumes the app's miners, switches the prover back on, resets the GPU compute-policy timeslice. Idempotent.
|
|
$ErrorActionPreference = 'Continue'
|
|
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
|
$urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
$base = (Get-Content $urlFile -Raw).Trim().TrimEnd('/')
|
|
# the state as text parsed by regex (app 0.3.10's response comes back empty to PowerShell 5.1's JSON reader)
|
|
function State {
|
|
try {
|
|
$raw = (Invoke-WebRequest -UseBasicParsing -Uri "$base/api/state" -TimeoutSec 20).Content
|
|
$cards = @()
|
|
# one card = the text from its "index" key to the next card's (the fields come in any order inside it; the first
|
|
# version took the first "identities" AFTER "vendor", which belongs to the NEXT card: 23:16:53Z, the 5090 set to the
|
|
# iGPU's 2 identities)
|
|
$idx = @([regex]::Matches($raw, '\{"index":\d+,"key":"') | ForEach-Object { $_.Index })
|
|
for ($i = 0; $i -lt $idx.Count; $i++) {
|
|
$end = if ($i + 1 -lt $idx.Count) { $idx[$i + 1] } else { $raw.Length }
|
|
$seg = $raw.Substring($idx[$i], $end - $idx[$i])
|
|
$f = { param($re, $def) $x = [regex]::Match($seg, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
|
$v = (& $f '"vendor":"([^"]*)"' '')
|
|
if (-not $v) { continue }
|
|
$cards += [pscustomobject]@{ key = (& $f '"key":"([^"]*)"' ''); name = (& $f '"name":"([^"]*)"' ''); vendor = $v; enabled = ((& $f '"enabled":(true|false)' 'true') -eq 'true'); identities = [int](& $f '"identities":(\d+)' '1'); state = (& $f '"state":"([^"]*)"' ''); hash_now = [double](& $f '"hash_now":([\d.eE+-]+)' '0') }
|
|
}
|
|
$g = { param($re, $def) $x = [regex]::Match($raw, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
|
return [pscustomobject]@{ version = (& $g '"version":"([^"]*)"' ''); settings = [pscustomobject]@{ prove = ((& $g '"settings":\{[^}]*?"prove":(true|false)' 'false') -eq 'true') }; mining = [pscustomobject]@{ paused = ((& $g '"mining":\{[^}]*?"paused":(true|false)' 'false') -eq 'true'); cards = $cards }; proving = [pscustomobject]@{ status = (& $g '"proving":\{[^}]*?"status":"([^"]*)"' '') } }
|
|
} catch { $null }
|
|
}
|
|
function Post($path, $body) { try { (Invoke-RestMethod -Method Post -Uri "$base$path" -ContentType 'application/json' -Body ($body | ConvertTo-Json -Compress -Depth 5) -TimeoutSec 15) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
|
$st = State
|
|
"RESULT before $(Stamp) app $($st.version) paused=$($st.mining.paused) prove=$($st.settings.prove) proving=$($st.proving.status)"
|
|
$apps = @(Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-app*' } | ForEach-Object { $_.Id })
|
|
"RESULT app_pids $(Stamp) $($apps -join ',')"
|
|
$killed = 0
|
|
foreach ($p in (Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -or $_.Name -like 'igneum-worker*' })) {
|
|
$parent = Get-CimInstance Win32_Process -Filter "ProcessId = $($p.ParentProcessId)" -ErrorAction SilentlyContinue
|
|
$stray = (-not $parent) -or (($parent.Name -notlike 'igneum-app*') -and ($parent.Name -notlike 'igneum-miner*'))
|
|
if ($stray) { "RESULT stray $(Stamp) pid $($p.ProcessId) $($p.Name) parent=$($p.ParentProcessId) ($(if ($parent) { $parent.Name } else { 'gone' })): stopped"; Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue; $killed++ }
|
|
}
|
|
"RESULT strays_stopped $(Stamp) $killed"
|
|
& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
|
# 6 October 2026 (0.3.15): no card switch, resume or start from a script (CLAUDE.md job rule); the runner of the measurement
|
|
# job restores the card and the miners on any exit, so this script only cleans up strays and reads the state back.
|
|
"RESULT timeslice $(Stamp) $((& nvidia-smi compute-policy --set-timeslice=0 2>&1) -join ' ')"
|
|
"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})"
|
|
Start-Sleep -Seconds 20
|
|
$st = State
|
|
"RESULT after $(Stamp) app $($st.version) paused=$($st.mining.paused) prove=$($st.settings.prove) proving=$($st.proving.status) cards=$(($st.mining.cards | ForEach-Object { "$($_.name):$($_.state):$([math]::Round($_.hash_now,1))" }) -join ',')"
|