igneum/relay/playbooks/shard-test.ps1

# Igneum playbook: the shard proof run on the RTX 5090, end to end, unattended. Mac-side reference, 4 October 2026.
# What it does: stops the Igneum Miner app cleanly (stop-igneum.ps1 from the installed app, else the engine's local
# API, else the process), waits for the GPU to go idle, downloads the hosted prove package fresh into C:\igneum-prove,
# runs the toolchain setup only when cargo or ~/.sp1 is missing inside WSL, runs prove-shard.sh non-interactively
# (the one-shard fixture at S_p, then the two- and four-shard blocks) under a 90-minute budget for the whole run,
# posts the results/*.json and the log to the relay as a result of this task, relaunches the app and prints RELAY-DONE.
# Queue: node tools/relay.mjs run PC2 "Shard proof run on the 5090" relay/playbooks/shard-test.ps1 --elevated
# The same work ships as the app's signed `shard-benchmark` job (app/igneum-app/src/jobrun.rs); this file stays as the
# relay form of it and as the model for a `run` job. UNTESTED on a PC as of 4 Oct 2026 (parse-checked by windows.yml).
$ErrorActionPreference = 'Continue'
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
$budgetMinutes = 90
$started = Get-Date
$deadline = $started.AddMinutes($budgetMinutes)
$distro = 'Ubuntu-24.04'
$wslUser = '[user]'
$zipUrl = '__DL_BASE__/igneum-prove-wsl2.zip'
$root = 'C:\igneum-prove'
$pkg = Join-Path $root 'igneum-prove-wsl2'
$zip = Join-Path $env:TEMP 'igneum-prove-wsl2.zip'
$shardFixture = 'block-338-shard1'
$blockFixtures = 'block-341-shards2 block-344-shards4'
if ($env:SHARD_FIXTURE) { $shardFixture = $env:SHARD_FIXTURE }
if ($env:BLOCK_FIXTURES) { $blockFixtures = $env:BLOCK_FIXTURES }
$appDir = Join-Path $env:ProgramFiles 'Igneum Miner'
$taskId = 0
if ($env:RELAY_TASK_ID) { $taskId = [long]$env:RELAY_TASK_ID }
$rc = 1

function Strip([string] $s) { if ($null -eq $s) { return '' }; return ($s -replace "`0", '') }
function Say([string] $m) { Write-Host ("[" + (Get-Date -Format 'HH:mm:ss') + "] " + $m) }
function Minutes-Left { return [int][Math]::Floor(($deadline - (Get-Date)).TotalMinutes) }
function Post-File([string] $path, [string] $title) {
  if (-not $env:RELAY_SEND) { Say ("no RELAY_SEND; not posting " + $path); return }
  if (-not (Test-Path $path)) { return }
  try { & $env:RELAY_SEND -TaskId $taskId -Title $title $path 2>&1 | ForEach-Object { Say ("  " + (Strip "$_")) } } catch { Say ("post failed for " + $path + ": " + $_.Exception.Message) }
}
function Gpu-Util {
  $smi = Get-Command nvidia-smi -ErrorAction SilentlyContinue
  if (-not $smi) { $alt = Join-Path $env:ProgramFiles 'NVIDIA Corporation\NVSMI\nvidia-smi.exe'; if (Test-Path $alt) { $smi = $alt } else { return -1 } }
  try {
    $out = & $smi --query-gpu=utilization.gpu --format=csv,noheader,nounits 2>$null
    $vals = @($out | ForEach-Object { [int]("$_".Trim()) })
    if ($vals.Count -eq 0) { return -1 }
    return ($vals | Measure-Object -Maximum).Maximum
  } catch { return -1 }
}
function Stop-App {
  $stop = Join-Path $appDir 'stop-igneum.ps1'
  if (Test-Path $stop) {
    Say ("stopping the miner app with " + $stop)
    & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $stop 2>&1 | ForEach-Object { Say ("  " + (Strip "$_")) }
    return
  }
  # (5 October 2026 rule: a job never quits the installed app it did not start; the api/quit that stood here is gone.
  #  Stopping the app is the signed `restart` job kind's work; without the stop script this waits for the app to stop.)
  $until = (Get-Date).AddSeconds(50)
  while ((Get-Date) -lt $until) {
    if (@(Get-Process -Name 'igneum-app', 'igneumd' -ErrorAction SilentlyContinue).Count -eq 0) { break }
    Start-Sleep -Milliseconds 500
  }
  foreach ($name in @('Igneum Miner', 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneumd')) {
    Get-Process -Name $name -ErrorAction SilentlyContinue | ForEach-Object { Say ("ending " + $_.ProcessName + " pid " + $_.Id); Stop-Process -Id $_.Id -Force -ErrorAction SilentlyContinue }
  }
}
function Start-App {
  $lnk = Join-Path $env:ProgramData 'Microsoft\Windows\Start Menu\Programs\Igneum Miner\Igneum Miner.lnk'
  $exe = Join-Path $appDir 'igneum-app.exe'
  if (Test-Path $lnk) {
    # explorer.exe starts the shortcut with the signed-in user's token, not the elevated one of this task
    Say ("relaunching the app from " + $lnk)
    Start-Process explorer.exe -ArgumentList ("`"" + $lnk + "`"")
  } elseif (Test-Path $exe) {
    Say ("relaunching " + $exe + " --launch")
    Start-Process -FilePath $exe -ArgumentList '--launch' -WorkingDirectory $appDir
  } else {
    Say 'no installed app found to relaunch (no Start Menu shortcut, no Program Files\Igneum Miner\igneum-app.exe)'
    return
  }
  Start-Sleep 20
  $alive = @(Get-Process -Name 'igneum-app', 'igneumd' -ErrorAction SilentlyContinue | ForEach-Object { $_.ProcessName + ' pid ' + $_.Id })
  if ($alive.Count -gt 0) { Say ("app running: " + ($alive -join ', ')) } else { Say 'WARNING: the app did not come back within 20 s' }
}

try {
  Say ("shard proof run on " + $env:COMPUTERNAME + ", budget " + $budgetMinutes + " min, fixtures " + $shardFixture + " then " + $blockFixtures)
  # 1. the miner app off the GPU
  Stop-App
  $until = (Get-Date).AddSeconds(120)
  while ((Get-Date) -lt $until) {
    $u = Gpu-Util
    if ($u -lt 0) { Say 'nvidia-smi gave no reading; waiting 10 s and going on'; Start-Sleep 10; break }
    if ($u -lt 5) { Say ("GPU idle at " + $u + "%"); break }
    Say ("GPU still at " + $u + "%"); Start-Sleep 3
  }
  # 2. the prove package, fresh
  Say ("downloading " + $zipUrl)
  Invoke-WebRequest -Uri $zipUrl -OutFile $zip -UseBasicParsing -TimeoutSec 600
  New-Item -ItemType Directory -Force -Path $root | Out-Null
  if (Test-Path $pkg) { Remove-Item $pkg -Recurse -Force }
  Expand-Archive -Path $zip -DestinationPath $root -Force
  $script = Join-Path $pkg 'prove-shard.sh'
  if (-not (Test-Path $script)) { throw ("prove-shard.sh missing under " + $pkg + " after the extract") }
  $linuxPkg = '/mnt/c/igneum-prove/igneum-prove-wsl2'
  # 3. the toolchain: setup only when it is missing
  $probe = 'command -v cargo && ls ~/.sp1 && ls ~/igneum-prove'
  & wsl.exe -d $distro -u $wslUser -- bash -lc $probe 2>&1 | ForEach-Object { Say ("  probe: " + (Strip "$_")) }
  if ($LASTEXITCODE -ne 0) {
    Say 'toolchain missing inside WSL: running setup-wsl.sh (15 to 40 minutes, approximate; needs sudo without a password)'
    & wsl.exe -d $distro -u $wslUser -- bash -lc ("sudo -n true 2>/dev/null || echo 'sudo asks for a password: the setup will fail'; cd " + $linuxPkg + " && bash ./setup-wsl.sh") 2>&1 | ForEach-Object { Say ("  setup: " + (Strip "$_")) }
    Say ("setup-wsl.sh exit " + $LASTEXITCODE)
  } else {
    Say 'toolchain present (cargo, ~/.sp1, ~/igneum-prove)'
  }
  # 4. the proof run, under what is left of the budget (5 minutes kept for the upload and the relaunch)
  $left = (Minutes-Left) - 5
  if ($left -lt 10) { throw ("only " + $left + " minutes of the budget left before the proof run; not starting it") }
  Say ("running prove-shard.sh with " + $left + " minutes")
  $outLog = Join-Path $pkg 'prove-shard-stdout.log'
  $errLog = Join-Path $pkg 'prove-shard-stderr.log'
  $p = Start-Process -FilePath 'wsl.exe' -ArgumentList @('-d', $distro, '-u', $wslUser, '--', 'bash', ($linuxPkg + '/prove-shard.sh'), $shardFixture, ('"' + $blockFixtures + '"')) -WorkingDirectory $pkg -NoNewWindow -PassThru -RedirectStandardOutput $outLog -RedirectStandardError $errLog
  $proveDeadline = (Get-Date).AddMinutes($left)
  $lastShown = 0
  while (-not $p.HasExited) {
    if ((Get-Date) -ge $proveDeadline) {
      Say 'time budget reached: ending the proof run'
      Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue
      & wsl.exe -d $distro -u $wslUser -- bash -c 'pkill -f "[i]gneum-prove-host"; pkill -f "[p]rove-shard.sh"; true' 2>&1 | Out-Null
      break
    }
    Start-Sleep 30
    if (Test-Path $outLog) {
      $lines = @(Get-Content $outLog -ErrorAction SilentlyContinue)
      if ($lines.Count -gt $lastShown) {
        $lines[$lastShown..($lines.Count - 1)] | Where-Object { $_ -match '^(RESULT|STAGE|===|BUILD FAILED|building|upload)' } | ForEach-Object { Say ("  " + (Strip "$_")) }
        $lastShown = $lines.Count
      }
    }
  }
  if ($p.HasExited) { $rc = $p.ExitCode; Say ("prove-shard.sh exit " + $rc) } else { $rc = 124 }
  # 5. the results to the relay
  Say 'RESULT and STAGE lines:'
  if (Test-Path $outLog) { Select-String -Path $outLog -Pattern '^(RESULT|STAGE|BUILD FAILED)' | ForEach-Object { Say ("  " + (Strip $_.Line)) } }
  $results = Join-Path $pkg 'results'
  if (Test-Path $results) {
    Get-ChildItem $results -Filter '*.json' | Where-Object { $_.LastWriteTime -ge $started } | ForEach-Object { Post-File $_.FullName ("prove result " + $_.Name) }
  }
  Get-ChildItem $pkg -Filter 'prove-shards-*.log' | Sort-Object LastWriteTime | Select-Object -Last 1 | ForEach-Object { Post-File $_.FullName 'prove-shard log' }
  Post-File $outLog 'prove-shard stdout'
} catch {
  Say ("ERROR: " + $_.Exception.Message)
  $rc = 1
} finally {
  # 6. mining back, whatever happened above
  Start-App
  Say ("done after " + [int]((Get-Date) - $started).TotalMinutes + " min, exit " + $rc)
  Write-Host 'RELAY-DONE'
}
exit $rc