c4da08d302
The known-failed case, from PC 2's 0.3.9 log (run win-1ccfe586-20261005-200114): 1791234223 pause -> 'stopping the miners (paused)' (every slot's restart_at cleared, the 5090 'off'); 1791235511 '[ok] mining resumed'; then '0.00 MH/s, waiting' at every 30-s status line until the 0.3.10 restart at 21:49:41Z. Cause: Cmd::Resume re-armed only slots whose watchdog said faulted; the 5090's slot was healthy and stopped, so nothing restarted it. The test the_pc2_resume_of_21_25_11z_restarts_under_the_new_rule_and_not_the_old encodes that slot (faulted false, live false): the old rule returns [] (the defect), the new rule [0]. cargo test -p igneum-app resume: 3 passed; provedefault: 6 passed. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
31 lines
2.5 KiB
PowerShell
31 lines
2.5 KiB
PowerShell
# The root-socket fix for PC 2 (5 October 2026): a measurement job that ran igneum-prove-host as root inside WSL2 left
|
|
# /tmp/sp1-cuda-0.sock owned by root (and possibly a root sp1-gpu-server), so the app's prover (its own WSL user) fails
|
|
# every shard with "CudaClientError: Connect(PermissionDenied)". This job, as root: kills every sp1-gpu-server, removes
|
|
# the sockets, prints their owners before and after, then switches the app's prover off and on so its next shard
|
|
# starts a fresh server under the app's user. Never stops the miners. About 60 s.
|
|
$ErrorActionPreference = 'Continue'
|
|
$urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
if (-not (Test-Path $urlFile)) { $urlFile = Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
$base = (Get-Content $urlFile -Raw).Trim().TrimEnd('/')
|
|
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
|
function Prove($on) { try { (Invoke-RestMethod -Method Post -Uri "$base/api/prove" -ContentType 'application/json' -Body (@{on=$on} | ConvertTo-Json -Compress) -TimeoutSec 10) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
|
"RESULT start $(Stamp)"
|
|
$bash = @'
|
|
echo "RESULT sockets_before $(ls -la /tmp/sp1-cuda-*.sock 2>&1 | tr '\n' ' ')"
|
|
echo "RESULT servers_before $(ps -eo user,pid,cmd | grep -E '[s]p1-gpu-server' | tr '\n' ' ' || echo none)"
|
|
pkill -f sp1-gpu-server 2>/dev/null; sleep 2; rm -f /tmp/sp1-cuda-*.sock
|
|
echo "RESULT sockets_after $(ls -la /tmp/sp1-cuda-*.sock 2>&1 | tr '\n' ' ')"
|
|
echo "RESULT servers_after $(ps -eo user,pid,cmd | grep -E '[s]p1-gpu-server' | tr '\n' ' ' || echo none)"
|
|
'@
|
|
$job = $env:IGNEUM_JOB_DIR; if (-not $job) { $job = $env:TEMP }
|
|
$bashFile = Join-Path $job 'socket-fix.sh'
|
|
[IO.File]::WriteAllText($bashFile, ($bash -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
|
$wslPath = (& wsl.exe -d Ubuntu-24.04 -u root -- wslpath -a ($bashFile -replace '\\', '/') 2>$null)
|
|
if (-not $wslPath) { $wslPath = '/mnt/c' + ($bashFile.Substring(2) -replace '\\', '/') }
|
|
& wsl.exe -d Ubuntu-24.04 -u root -- bash (($wslPath -replace "`0", '').Trim()) 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
|
"RESULT prove_off $(Stamp) $(Prove $false)"
|
|
Start-Sleep -Seconds 15
|
|
"RESULT prove_on $(Stamp) $(Prove $true)"
|
|
Start-Sleep -Seconds 40
|
|
$st = $null; try { $st = Invoke-RestMethod -Uri "$base/api/state" -TimeoutSec 20 } catch {}
|
|
"RESULT end $(Stamp) proving status: $($st.proving.status) message: $($st.proving.message)"
|