From d19441c14deaa8ed7a197fe39513a47434b68b61 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:00:06 +0000 Subject: [PATCH] C35 class: a second engine gets no pipe (its output goes to a file the playbook tails) and its whole tree is ended at the end and on the budget; ember-tune-pc1.ps1 and sweep-5090.ps1 fixed; tools/ci/second-engine-check.sh fails any playbook without both; the rule in ember-tune.md PC 1, 22:31 UTC: the installed engine's quit hung 24 minutes in the jobs runner's abort, waiting for EOF on the script's stdout pipe whose write end the second engine and its miners had inherited (Process.Start with redirection inherits every inheritable handle), while the orphaned miners mined on against the relaunched app. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 2 ++ docs/plans/ember-tune.md | 2 ++ relay/playbooks/ember-tune-pc1.ps1 | 44 ++++++++++++++++-------------- relay/playbooks/sweep-5090.ps1 | 44 ++++++++++++++++-------------- tools/ci/second-engine-check.sh | 24 ++++++++++++++++ 5 files changed, 74 insertions(+), 42 deletions(-) create mode 100755 tools/ci/second-engine-check.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 185fa0f42..138a5d156 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,8 @@ jobs: run: bash tools/ci/no-conflict-markers.sh - name: copied sources are re-stamped before a build run: bash tools/ci/copied-sources-check.sh + - name: second-engine playbooks log to a file and end their tree (C35) + run: bash tools/ci/second-engine-check.sh - name: pinned guest programs match their manifest and are built only by pin-guests.sh run: bash tools/ci/pinned-guests-check.sh - name: no secret file names and no 64-hex secrets in the tree (self-test first, then the tree) diff --git a/docs/plans/ember-tune.md b/docs/plans/ember-tune.md index 61a3d730b..4e0c9fea2 100644 --- a/docs/plans/ember-tune.md +++ b/docs/plans/ember-tune.md @@ -99,6 +99,8 @@ race has run). | A signed prior is only ever a starting point inside the card's OWN reported limits (`power.min_limit` to `power.max_limit`, the clock floor to `clocks.max.gr` or the ADLX `gmax_range`), never a memory clock, never a value the card did not report; the confirm step measures it and the full plan replaces it when a neighbour beats it, so a bad prior costs the fleet one confirm step per card, not a setting. The signing key (K1, docs/security/keys.md) therefore cannot push a card past its vendor ceiling or under its floor | `Plan::confirm` clamps through `Limits::clamp_clock` and `power_pct.clamp(50, 100)`; proven by `ember::tests::the_confirm_plan_checks_the_prior_and_its_neighbour` (a prior of 9,000 MHz at 30% becomes 3,090 MHz at 50%) and `limits_never_exceed_the_vendor_or_undercut_the_floor` | | No prompt the user did not ask for: the NVIDIA helper starts only with Power control on; the `--sweep` job never counts as permission | `sweep_probe_known`, `sweep_helper_start` | | The elevated helper restores the limit and resets the clocks by itself after 20 idle minutes | `sweep::helper_script_*` | +| A playbook that starts a second engine beside the installed app (the PC measurement jobs) gives it NO pipe (its output goes to a file the script tails: a pipe's write end is inherited by the engine's miners, and the installed app's jobs runner then waits forever for EOF after an abort; C35, PC 1 22:31 UTC, a 24-minute hang and orphaned miners), ends the engine's whole process tree at the end and on the budget (`taskkill /T /F`), and lets the installed app's miners come back only after that | `relay/playbooks/ember-tune-pc1.ps1`, `sweep-5090.ps1`; CI `tools/ci/second-engine-check.sh` fails any playbook without both | +| Every `quit:` line in the app log names its source (the window host's stdin, the host gone, `POST /api/quit`, the `--sweep` run's end) | `Cmd::Quit(&'static str)` (b671c8b) | ## 6. Tests diff --git a/relay/playbooks/ember-tune-pc1.ps1 b/relay/playbooks/ember-tune-pc1.ps1 index 268528300..0ce8566cc 100644 --- a/relay/playbooks/ember-tune-pc1.ps1 +++ b/relay/playbooks/ember-tune-pc1.ps1 @@ -14,6 +14,7 @@ # ends. Not elevated: nothing asks for administrator rights (the project lead asleep, 5 October 2026); the NVIDIA card is # therefore measure only tonight unless the engine finds itself elevated. $ErrorActionPreference = 'Continue' +$resultTag = 'TUNE' $budgetMinutes = 35 if (-not ($budgetMinutes -is [int]) -or $budgetMinutes -lt 5) { $budgetMinutes = 35 } # a budget under 5 minutes is a bug, not a budget (C35) $started = Get-Date @@ -94,29 +95,28 @@ Snapshot 'before' $env:IGNEUM_APP_DATA = $root $env:IGNEUM_APP_LOGS = $sLogs $env:IGNEUM_APP_STATUS_SECS = '10' -$psi = New-Object System.Diagnostics.ProcessStartInfo -$psi.FileName = $exe -$psi.Arguments = '--sweep' -$psi.WorkingDirectory = $bin -$psi.UseShellExecute = $false -$psi.RedirectStandardOutput = $true -$psi.RedirectStandardError = $true -$psi.CreateNoWindow = $true -$p = New-Object System.Diagnostics.Process -$p.StartInfo = $psi -$lines = New-Object System.Collections.ArrayList -$h = { if ($EventArgs.Data) { [void]$Event.MessageData.Add($EventArgs.Data) } } -Register-ObjectEvent -InputObject $p -EventName OutputDataReceived -Action $h -MessageData $lines | Out-Null -Register-ObjectEvent -InputObject $p -EventName ErrorDataReceived -Action $h -MessageData $lines | Out-Null -[void]$p.Start() -$p.BeginOutputReadLine(); $p.BeginErrorReadLine() -Say ("tune engine started, pid " + $p.Id + ", data " + $root) +# C35 (5 October 2026): the engine's output goes to a FILE, never a pipe. A pipe's write end is inherited by every +# process the engine starts (its miners and workers), so after an abort the installed app's jobs runner waits for an +# EOF that never comes and hangs in its own quit; and the engine's whole tree is killed at the end (nothing orphaned). +$outFile = Join-Path $root 'engine-stdout.log' +$errFile = Join-Path $root 'engine-stderr.log' +Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue +$p = Start-Process -FilePath $exe -ArgumentList '--sweep' -WorkingDirectory (Split-Path $exe) -WindowStyle Hidden -PassThru -RedirectStandardOutput $outFile -RedirectStandardError $errFile +Say ("engine started, pid " + $p.Id + ", data " + $root + ", stdout " + $outFile) +function EndTree([int] $procId, [string] $why) { + $before = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count + & taskkill /T /F /PID $procId 2>&1 | Out-Null + Start-Sleep -Seconds 2 + $after = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count + Write-Output ('RESULT ' + $resultTag + ' tree ended (' + $why + '): igneum processes ' + $before + ' -> ' + $after + ' (the installed app''s own miners are stopped and held by the job)') +} $seen = 0 $rows = 0 while (-not $p.HasExited) { Start-Sleep -Seconds 5 - while ($seen -lt $lines.Count) { - $l = [string]$lines[$seen]; $seen++ + $all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } + while ($seen -lt $all.Count) { + $l = [string]$all[$seen]; $seen++ if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } elseif ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l) } elseif ($l -match '^(URL|STATE) ') { } @@ -135,12 +135,14 @@ while (-not $p.HasExited) { try { Invoke-WebRequest -Uri ((Get-Content -LiteralPath $u -Raw).Trim() + 'api/quit') -Method POST -Body '{}' -ContentType 'application/json' -UseBasicParsing -TimeoutSec 5 | Out-Null } catch { } } else { Write-Output ('RESULT TUNE quit not sent: no URL file at ' + $u + '; killing pid ' + $p.Id) } Start-Sleep -Seconds 20 - if (-not $p.HasExited) { $p.Kill() } + if (-not $p.HasExited) { EndTree $p.Id 'budget' } Write-Output 'RESULT TUNE error=budget_exceeded' } } -while ($seen -lt $lines.Count) { $l = [string]$lines[$seen]; $seen++; if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } } +$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } +while ($seen -lt $all.Count) { $l = [string]$all[$seen]; $seen++; if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } } Say ("tune engine exited " + $p.ExitCode + " after " + [int]((Get-Date) - $started).TotalSeconds + " s, " + $rows + " table rows") +EndTree $p.Id 'end of run' Snapshot 'after' # the tune engine's own log: the TUNE lines and what happened around them $log = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 diff --git a/relay/playbooks/sweep-5090.ps1 b/relay/playbooks/sweep-5090.ps1 index da6c2ebb0..bb0a3a778 100644 --- a/relay/playbooks/sweep-5090.ps1 +++ b/relay/playbooks/sweep-5090.ps1 @@ -8,6 +8,7 @@ # miners restart when the job ends. Elevated, so nvidia-smi -pl needs no prompt (the engine detects that: mode=direct). # UNTESTED on a PC as of 4 Oct 2026 (parse-checked only). $ErrorActionPreference = 'Continue' +$resultTag = 'SWEEP' $budgetMinutes = 40 $started = Get-Date $deadline = $started.AddMinutes($budgetMinutes) @@ -59,29 +60,28 @@ if (Test-Path $smi) { $env:IGNEUM_APP_DATA = $root $env:IGNEUM_APP_LOGS = $sLogs $env:IGNEUM_APP_STATUS_SECS = '10' -$psi = New-Object System.Diagnostics.ProcessStartInfo -$psi.FileName = $exe -$psi.Arguments = '--sweep' -$psi.WorkingDirectory = Split-Path $exe -$psi.UseShellExecute = $false -$psi.RedirectStandardOutput = $true -$psi.RedirectStandardError = $true -$psi.CreateNoWindow = $true -$p = New-Object System.Diagnostics.Process -$p.StartInfo = $psi -$lines = New-Object System.Collections.ArrayList -$h = { if ($EventArgs.Data) { [void]$Event.MessageData.Add($EventArgs.Data) } } -Register-ObjectEvent -InputObject $p -EventName OutputDataReceived -Action $h -MessageData $lines | Out-Null -Register-ObjectEvent -InputObject $p -EventName ErrorDataReceived -Action $h -MessageData $lines | Out-Null -[void]$p.Start() -$p.BeginOutputReadLine(); $p.BeginErrorReadLine() -Say ("sweep engine started, pid " + $p.Id + ", data " + $root) +# C35 (5 October 2026): the engine's output goes to a FILE, never a pipe. A pipe's write end is inherited by every +# process the engine starts (its miners and workers), so after an abort the installed app's jobs runner waits for an +# EOF that never comes and hangs in its own quit; and the engine's whole tree is killed at the end (nothing orphaned). +$outFile = Join-Path $root 'engine-stdout.log' +$errFile = Join-Path $root 'engine-stderr.log' +Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue +$p = Start-Process -FilePath $exe -ArgumentList '--sweep' -WorkingDirectory (Split-Path $exe) -WindowStyle Hidden -PassThru -RedirectStandardOutput $outFile -RedirectStandardError $errFile +Say ("engine started, pid " + $p.Id + ", data " + $root + ", stdout " + $outFile) +function EndTree([int] $procId, [string] $why) { + $before = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count + & taskkill /T /F /PID $procId 2>&1 | Out-Null + Start-Sleep -Seconds 2 + $after = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count + Write-Output ('RESULT ' + $resultTag + ' tree ended (' + $why + '): igneum processes ' + $before + ' -> ' + $after + ' (the installed app''s own miners are stopped and held by the job)') +} $seen = 0 $rows = 0 while (-not $p.HasExited) { Start-Sleep -Seconds 5 - while ($seen -lt $lines.Count) { - $l = [string]$lines[$seen]; $seen++ + $all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } + while ($seen -lt $all.Count) { + $l = [string]$all[$seen]; $seen++ if ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l); if ($l -match '^SWEEP card=') { $rows++ } } elseif ($l -match '^(URL|STATE) ') { } else { Say $l } @@ -91,12 +91,14 @@ while (-not $p.HasExited) { $u = Join-Path $sApp 'app.url' if (Test-Path $u) { try { Invoke-WebRequest -Uri ((Get-Content -LiteralPath $u -Raw).Trim() + 'api/quit') -Method POST -Body '{}' -ContentType 'application/json' -UseBasicParsing -TimeoutSec 5 | Out-Null } catch { } } Start-Sleep -Seconds 20 - if (-not $p.HasExited) { $p.Kill() } + if (-not $p.HasExited) { EndTree $p.Id 'budget' } Write-Output 'RESULT SWEEP error=budget_exceeded' } } -while ($seen -lt $lines.Count) { $l = [string]$lines[$seen]; $seen++; if ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l); if ($l -match '^SWEEP card=') { $rows++ } } } +$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } +while ($seen -lt $all.Count) { $l = [string]$all[$seen]; $seen++; if ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l); if ($l -match '^SWEEP card=') { $rows++ } } } Say ("sweep engine exited " + $p.ExitCode + " after " + [int]((Get-Date) - $started).TotalSeconds + " s, " + $rows + " table rows") +EndTree $p.Id 'end of run' if (Test-Path $smi) { $q = (& $smi --query-gpu=index,power.draw,power.limit --format=csv,noheader 2>&1 | Out-String).Trim() Write-Output ("RESULT SWEEP after " + ($q -replace "`r?`n", ' | ')) diff --git a/tools/ci/second-engine-check.sh b/tools/ci/second-engine-check.sh new file mode 100755 index 000000000..564d390f1 --- /dev/null +++ b/tools/ci/second-engine-check.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# The second-engine class (C35, 5 October 2026, PC 1 22:31 UTC): a playbook started a second Igneum engine beside the +# installed app through a redirected PIPE (PowerShell's Process.Start with RedirectStandardOutput). A pipe's write end is +# inherited by every process the engine starts (its miners and workers); when the job was aborted, the installed app's +# jobs runner waited for an EOF the orphaned grandchildren never sent and its quit hung for 24 minutes, and the second +# engine's miners mined on against the relaunched app. Rule for every playbook that starts an engine: (1) the engine's +# output goes to a FILE (Start-Process -RedirectStandardOutput ), never a pipe into the script; (2) the engine's +# whole process tree is ended at the end and on the budget (taskkill /T /F), so nothing is orphaned; the installed +# app's miners come back only after that (the job runner restarts them when the script ends). This check fails CI when +# a playbook starts an engine without both. +set -euo pipefail +cd "$(dirname "$0")/../.." +fail=0 +while IFS= read -r f; do + grep -qE "igneum-app(\.exe)?['\"]? *(--sweep|-ArgumentList '--sweep'|--no-open)|ArgumentList '--sweep'|\.Arguments = '--sweep'" "$f" || continue + if grep -qE 'RedirectStandardOutput *= *\$true|UseShellExecute *= *\$false|Register-ObjectEvent|BeginOutputReadLine' "$f"; then + echo "second-engine: $f starts an engine through a pipe (RedirectStandardOutput/BeginOutputReadLine); use Start-Process -RedirectStandardOutput "; fail=1 + fi + if ! grep -qE 'taskkill /T /F' "$f"; then + echo "second-engine: $f starts an engine without ending its process tree (taskkill /T /F) at the end"; fail=1 + fi +done < <(git ls-files 'relay/playbooks/**' 'tools/windows/**' 'packaging/**' | grep -E '\.ps1$') +[ "$fail" = 0 ] && echo "second-engine: every playbook that starts an engine logs to a file and ends its tree" +exit $fail