diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 185fa0f42..138a5d156 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,8 @@ jobs: run: bash tools/ci/no-conflict-markers.sh - name: copied sources are re-stamped before a build run: bash tools/ci/copied-sources-check.sh + - name: second-engine playbooks log to a file and end their tree (C35) + run: bash tools/ci/second-engine-check.sh - name: pinned guest programs match their manifest and are built only by pin-guests.sh run: bash tools/ci/pinned-guests-check.sh - name: no secret file names and no 64-hex secrets in the tree (self-test first, then the tree) diff --git a/docs/plans/ember-tune.md b/docs/plans/ember-tune.md index 61a3d730b..4e0c9fea2 100644 --- a/docs/plans/ember-tune.md +++ b/docs/plans/ember-tune.md @@ -99,6 +99,8 @@ race has run). | A signed prior is only ever a starting point inside the card's OWN reported limits (`power.min_limit` to `power.max_limit`, the clock floor to `clocks.max.gr` or the ADLX `gmax_range`), never a memory clock, never a value the card did not report; the confirm step measures it and the full plan replaces it when a neighbour beats it, so a bad prior costs the fleet one confirm step per card, not a setting. The signing key (K1, docs/security/keys.md) therefore cannot push a card past its vendor ceiling or under its floor | `Plan::confirm` clamps through `Limits::clamp_clock` and `power_pct.clamp(50, 100)`; proven by `ember::tests::the_confirm_plan_checks_the_prior_and_its_neighbour` (a prior of 9,000 MHz at 30% becomes 3,090 MHz at 50%) and `limits_never_exceed_the_vendor_or_undercut_the_floor` | | No prompt the user did not ask for: the NVIDIA helper starts only with Power control on; the `--sweep` job never counts as permission | `sweep_probe_known`, `sweep_helper_start` | | The elevated helper restores the limit and resets the clocks by itself after 20 idle minutes | `sweep::helper_script_*` | +| A playbook that starts a second engine beside the installed app (the PC measurement jobs) gives it NO pipe (its output goes to a file the script tails: a pipe's write end is inherited by the engine's miners, and the installed app's jobs runner then waits forever for EOF after an abort; C35, PC 1 22:31 UTC, a 24-minute hang and orphaned miners), ends the engine's whole process tree at the end and on the budget (`taskkill /T /F`), and lets the installed app's miners come back only after that | `relay/playbooks/ember-tune-pc1.ps1`, `sweep-5090.ps1`; CI `tools/ci/second-engine-check.sh` fails any playbook without both | +| Every `quit:` line in the app log names its source (the window host's stdin, the host gone, `POST /api/quit`, the `--sweep` run's end) | `Cmd::Quit(&'static str)` (b671c8b) | ## 6. Tests diff --git a/relay/playbooks/ember-tune-pc1.ps1 b/relay/playbooks/ember-tune-pc1.ps1 index 268528300..0ce8566cc 100644 --- a/relay/playbooks/ember-tune-pc1.ps1 +++ b/relay/playbooks/ember-tune-pc1.ps1 @@ -14,6 +14,7 @@ # ends. Not elevated: nothing asks for administrator rights (the project lead asleep, 5 October 2026); the NVIDIA card is # therefore measure only tonight unless the engine finds itself elevated. $ErrorActionPreference = 'Continue' +$resultTag = 'TUNE' $budgetMinutes = 35 if (-not ($budgetMinutes -is [int]) -or $budgetMinutes -lt 5) { $budgetMinutes = 35 } # a budget under 5 minutes is a bug, not a budget (C35) $started = Get-Date @@ -94,29 +95,28 @@ Snapshot 'before' $env:IGNEUM_APP_DATA = $root $env:IGNEUM_APP_LOGS = $sLogs $env:IGNEUM_APP_STATUS_SECS = '10' -$psi = New-Object System.Diagnostics.ProcessStartInfo -$psi.FileName = $exe -$psi.Arguments = '--sweep' -$psi.WorkingDirectory = $bin -$psi.UseShellExecute = $false -$psi.RedirectStandardOutput = $true -$psi.RedirectStandardError = $true -$psi.CreateNoWindow = $true -$p = New-Object System.Diagnostics.Process -$p.StartInfo = $psi -$lines = New-Object System.Collections.ArrayList -$h = { if ($EventArgs.Data) { [void]$Event.MessageData.Add($EventArgs.Data) } } -Register-ObjectEvent -InputObject $p -EventName OutputDataReceived -Action $h -MessageData $lines | Out-Null -Register-ObjectEvent -InputObject $p -EventName ErrorDataReceived -Action $h -MessageData $lines | Out-Null -[void]$p.Start() -$p.BeginOutputReadLine(); $p.BeginErrorReadLine() -Say ("tune engine started, pid " + $p.Id + ", data " + $root) +# C35 (5 October 2026): the engine's output goes to a FILE, never a pipe. A pipe's write end is inherited by every +# process the engine starts (its miners and workers), so after an abort the installed app's jobs runner waits for an +# EOF that never comes and hangs in its own quit; and the engine's whole tree is killed at the end (nothing orphaned). +$outFile = Join-Path $root 'engine-stdout.log' +$errFile = Join-Path $root 'engine-stderr.log' +Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue +$p = Start-Process -FilePath $exe -ArgumentList '--sweep' -WorkingDirectory (Split-Path $exe) -WindowStyle Hidden -PassThru -RedirectStandardOutput $outFile -RedirectStandardError $errFile +Say ("engine started, pid " + $p.Id + ", data " + $root + ", stdout " + $outFile) +function EndTree([int] $procId, [string] $why) { + $before = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count + & taskkill /T /F /PID $procId 2>&1 | Out-Null + Start-Sleep -Seconds 2 + $after = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count + Write-Output ('RESULT ' + $resultTag + ' tree ended (' + $why + '): igneum processes ' + $before + ' -> ' + $after + ' (the installed app''s own miners are stopped and held by the job)') +} $seen = 0 $rows = 0 while (-not $p.HasExited) { Start-Sleep -Seconds 5 - while ($seen -lt $lines.Count) { - $l = [string]$lines[$seen]; $seen++ + $all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } + while ($seen -lt $all.Count) { + $l = [string]$all[$seen]; $seen++ if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } elseif ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l) } elseif ($l -match '^(URL|STATE) ') { } @@ -135,12 +135,14 @@ while (-not $p.HasExited) { try { Invoke-WebRequest -Uri ((Get-Content -LiteralPath $u -Raw).Trim() + 'api/quit') -Method POST -Body '{}' -ContentType 'application/json' -UseBasicParsing -TimeoutSec 5 | Out-Null } catch { } } else { Write-Output ('RESULT TUNE quit not sent: no URL file at ' + $u + '; killing pid ' + $p.Id) } Start-Sleep -Seconds 20 - if (-not $p.HasExited) { $p.Kill() } + if (-not $p.HasExited) { EndTree $p.Id 'budget' } Write-Output 'RESULT TUNE error=budget_exceeded' } } -while ($seen -lt $lines.Count) { $l = [string]$lines[$seen]; $seen++; if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } } +$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } +while ($seen -lt $all.Count) { $l = [string]$all[$seen]; $seen++; if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } } Say ("tune engine exited " + $p.ExitCode + " after " + [int]((Get-Date) - $started).TotalSeconds + " s, " + $rows + " table rows") +EndTree $p.Id 'end of run' Snapshot 'after' # the tune engine's own log: the TUNE lines and what happened around them $log = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 diff --git a/relay/playbooks/sweep-5090.ps1 b/relay/playbooks/sweep-5090.ps1 index da6c2ebb0..bb0a3a778 100644 --- a/relay/playbooks/sweep-5090.ps1 +++ b/relay/playbooks/sweep-5090.ps1 @@ -8,6 +8,7 @@ # miners restart when the job ends. Elevated, so nvidia-smi -pl needs no prompt (the engine detects that: mode=direct). # UNTESTED on a PC as of 4 Oct 2026 (parse-checked only). $ErrorActionPreference = 'Continue' +$resultTag = 'SWEEP' $budgetMinutes = 40 $started = Get-Date $deadline = $started.AddMinutes($budgetMinutes) @@ -59,29 +60,28 @@ if (Test-Path $smi) { $env:IGNEUM_APP_DATA = $root $env:IGNEUM_APP_LOGS = $sLogs $env:IGNEUM_APP_STATUS_SECS = '10' -$psi = New-Object System.Diagnostics.ProcessStartInfo -$psi.FileName = $exe -$psi.Arguments = '--sweep' -$psi.WorkingDirectory = Split-Path $exe -$psi.UseShellExecute = $false -$psi.RedirectStandardOutput = $true -$psi.RedirectStandardError = $true -$psi.CreateNoWindow = $true -$p = New-Object System.Diagnostics.Process -$p.StartInfo = $psi -$lines = New-Object System.Collections.ArrayList -$h = { if ($EventArgs.Data) { [void]$Event.MessageData.Add($EventArgs.Data) } } -Register-ObjectEvent -InputObject $p -EventName OutputDataReceived -Action $h -MessageData $lines | Out-Null -Register-ObjectEvent -InputObject $p -EventName ErrorDataReceived -Action $h -MessageData $lines | Out-Null -[void]$p.Start() -$p.BeginOutputReadLine(); $p.BeginErrorReadLine() -Say ("sweep engine started, pid " + $p.Id + ", data " + $root) +# C35 (5 October 2026): the engine's output goes to a FILE, never a pipe. A pipe's write end is inherited by every +# process the engine starts (its miners and workers), so after an abort the installed app's jobs runner waits for an +# EOF that never comes and hangs in its own quit; and the engine's whole tree is killed at the end (nothing orphaned). +$outFile = Join-Path $root 'engine-stdout.log' +$errFile = Join-Path $root 'engine-stderr.log' +Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue +$p = Start-Process -FilePath $exe -ArgumentList '--sweep' -WorkingDirectory (Split-Path $exe) -WindowStyle Hidden -PassThru -RedirectStandardOutput $outFile -RedirectStandardError $errFile +Say ("engine started, pid " + $p.Id + ", data " + $root + ", stdout " + $outFile) +function EndTree([int] $procId, [string] $why) { + $before = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count + & taskkill /T /F /PID $procId 2>&1 | Out-Null + Start-Sleep -Seconds 2 + $after = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count + Write-Output ('RESULT ' + $resultTag + ' tree ended (' + $why + '): igneum processes ' + $before + ' -> ' + $after + ' (the installed app''s own miners are stopped and held by the job)') +} $seen = 0 $rows = 0 while (-not $p.HasExited) { Start-Sleep -Seconds 5 - while ($seen -lt $lines.Count) { - $l = [string]$lines[$seen]; $seen++ + $all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } + while ($seen -lt $all.Count) { + $l = [string]$all[$seen]; $seen++ if ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l); if ($l -match '^SWEEP card=') { $rows++ } } elseif ($l -match '^(URL|STATE) ') { } else { Say $l } @@ -91,12 +91,14 @@ while (-not $p.HasExited) { $u = Join-Path $sApp 'app.url' if (Test-Path $u) { try { Invoke-WebRequest -Uri ((Get-Content -LiteralPath $u -Raw).Trim() + 'api/quit') -Method POST -Body '{}' -ContentType 'application/json' -UseBasicParsing -TimeoutSec 5 | Out-Null } catch { } } Start-Sleep -Seconds 20 - if (-not $p.HasExited) { $p.Kill() } + if (-not $p.HasExited) { EndTree $p.Id 'budget' } Write-Output 'RESULT SWEEP error=budget_exceeded' } } -while ($seen -lt $lines.Count) { $l = [string]$lines[$seen]; $seen++; if ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l); if ($l -match '^SWEEP card=') { $rows++ } } } +$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } +while ($seen -lt $all.Count) { $l = [string]$all[$seen]; $seen++; if ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l); if ($l -match '^SWEEP card=') { $rows++ } } } Say ("sweep engine exited " + $p.ExitCode + " after " + [int]((Get-Date) - $started).TotalSeconds + " s, " + $rows + " table rows") +EndTree $p.Id 'end of run' if (Test-Path $smi) { $q = (& $smi --query-gpu=index,power.draw,power.limit --format=csv,noheader 2>&1 | Out-String).Trim() Write-Output ("RESULT SWEEP after " + ($q -replace "`r?`n", ' | ')) diff --git a/tools/ci/second-engine-check.sh b/tools/ci/second-engine-check.sh new file mode 100755 index 000000000..564d390f1 --- /dev/null +++ b/tools/ci/second-engine-check.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# The second-engine class (C35, 5 October 2026, PC 1 22:31 UTC): a playbook started a second Igneum engine beside the +# installed app through a redirected PIPE (PowerShell's Process.Start with RedirectStandardOutput). A pipe's write end is +# inherited by every process the engine starts (its miners and workers); when the job was aborted, the installed app's +# jobs runner waited for an EOF the orphaned grandchildren never sent and its quit hung for 24 minutes, and the second +# engine's miners mined on against the relaunched app. Rule for every playbook that starts an engine: (1) the engine's +# output goes to a FILE (Start-Process -RedirectStandardOutput ), never a pipe into the script; (2) the engine's +# whole process tree is ended at the end and on the budget (taskkill /T /F), so nothing is orphaned; the installed +# app's miners come back only after that (the job runner restarts them when the script ends). This check fails CI when +# a playbook starts an engine without both. +set -euo pipefail +cd "$(dirname "$0")/../.." +fail=0 +while IFS= read -r f; do + grep -qE "igneum-app(\.exe)?['\"]? *(--sweep|-ArgumentList '--sweep'|--no-open)|ArgumentList '--sweep'|\.Arguments = '--sweep'" "$f" || continue + if grep -qE 'RedirectStandardOutput *= *\$true|UseShellExecute *= *\$false|Register-ObjectEvent|BeginOutputReadLine' "$f"; then + echo "second-engine: $f starts an engine through a pipe (RedirectStandardOutput/BeginOutputReadLine); use Start-Process -RedirectStandardOutput "; fail=1 + fi + if ! grep -qE 'taskkill /T /F' "$f"; then + echo "second-engine: $f starts an engine without ending its process tree (taskkill /T /F) at the end"; fail=1 + fi +done < <(git ls-files 'relay/playbooks/**' 'tools/windows/**' 'packaging/**' | grep -E '\.ps1$') +[ "$fail" = 0 ] && echo "second-engine: every playbook that starts an engine logs to a file and ends its tree" +exit $fail