From 10c9aad1c1d4a05fe5feba49a9e2df1f8d1aca3d Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:38:32 +0000 Subject: [PATCH] 0.3.15: the playbook-quit allow list is only the installer's stop step; the agg-cost scripts on the runner's --cards-off; ember-tune-pc1.ps1 deleted The dated entries of the 0.3.14 gate expired at 0.3.15. pc2-agg-cost.ps1 no longer switches the 5090 or falls back to /api/pause: the job is published with --cards-off , the runner switches the card off before the script and puts it back on any exit, and the script fails loud (exit 3) if a CUDA worker is still running. pc2-agg-cost-restore.ps1 keeps the stray and socket clean-up and the read-back only. ember-tune-pc1.ps1 is superseded by the installed-tune playbook on ember-tune. Co-Authored-By: Claude Fable 5.1 --- relay/playbooks/ember-tune-pc1.ps1 | 265 ---------------------- tools/ci/playbook-quit-check.sh | 13 +- tools/proving-v1/pc2-agg-cost-restore.ps1 | 12 +- tools/proving-v1/pc2-agg-cost.ps1 | 26 +-- 4 files changed, 15 insertions(+), 301 deletions(-) delete mode 100644 relay/playbooks/ember-tune-pc1.ps1 diff --git a/relay/playbooks/ember-tune-pc1.ps1 b/relay/playbooks/ember-tune-pc1.ps1 deleted file mode 100644 index beab8e1ea..000000000 --- a/relay/playbooks/ember-tune-pc1.ps1 +++ /dev/null @@ -1,265 +0,0 @@ -# Igneum run job: Ember Tune end to end on PC 1 (machine ae432dc7), unattended. 5 October 2026. -# Published as a `run` job with --stop-miners (docs/plans/ember-tune.md): the installed app stops its miners and -# holds them; this script takes the engine that carries src/ember.rs (the one the fetch job put in -# \jobs\ember-kit-1\igneum-app-ember.exe, else the installed one; the AMD helper with the --tune and --set -# commands from the telemetry agent's fetch job, \jobs\amd-kit-1\kit\igneum-gpu-telemetry.exe), copies the install folder to a scratch -# folder beside it, swaps the engine in, and starts that SECOND engine with `--sweep` in a scratch data folder (the -# real settings.json, machine-id and wallet.json copied in; remote jobs, auto-update and proving switched off -# there). That engine finds the installed app's node on 127.0.0.1:26610, mines on every card with the live program, -# tunes them one after the other (the full two-knob plan where the card can be controlled, the baseline measurement -# where it cannot: NVIDIA without administrator rights, Apple), prints every TUNE line on stdout, uploads its log -# (the TUNE {json} record reaches the intake) and quits. Every TUNE line is re-emitted as a RESULT line, so -# `node tools/jobs.mjs ` shows the table. Before and after, nvidia-smi's limits and clocks and the AMD -# helper's `--tune` lines are printed, so the restore can be read. The installed app's miners restart when the job -# ends. Not elevated: nothing asks for administrator rights (the project lead asleep, 5 October 2026); the NVIDIA card is -# therefore measure only tonight unless the engine finds itself elevated. -$ErrorActionPreference = 'Continue' -$resultTag = 'TUNE' -$budgetMinutes = 55 -if (-not ($budgetMinutes -is [int]) -or $budgetMinutes -lt 5) { $budgetMinutes = 35 } # a budget under 5 minutes is a bug, not a budget (C35) -$started = Get-Date -$deadline = $started.AddMinutes($budgetMinutes) -function Say([string] $m) { Write-Host ("[" + (Get-Date -Format 'HH:mm:ss') + "] " + $m) } - -# the installed engine: the per-user install (0.3.3+), else Program Files -$installDir = $null -foreach ($d in @((Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner'), (Join-Path $env:ProgramFiles 'Igneum Miner'))) { - if (Test-Path (Join-Path $d 'igneum-app.exe')) { $installDir = $d; break } -} -if (-not $installDir) { Write-Output 'RESULT TUNE error=no_engine reason=igneum-app.exe_not_found'; exit 2 } -$appData = $env:IGNEUM_APP_DATA -if (-not $appData) { $appData = Join-Path $env:LOCALAPPDATA 'igneum' } -$appDir = $env:IGNEUM_APP_DIR -if (-not $appDir) { $appDir = Join-Path $appData 'app' } - -# the scratch install: the whole folder (workers, node, helper, DLLs) with the Ember engine swapped in -# a FRESH scratch root per run (run 3, 6 October 2026, 11:45Z: the folder an earlier elevated engine had locked to itself -# refused the settings copy, the engine started on stale files and exited in 6 s); old roots are small and left alone -$root = Join-Path $env:LOCALAPPDATA ('igneum-tune-' + (Get-Date -Format 'yyyyMMdd-HHmmss')) -$bin = Join-Path $root 'bin' -$sApp = Join-Path $root 'app' -$sLogs = Join-Path $root 'logs' -New-Item -ItemType Directory -Force -Path $root, $sApp, $sLogs | Out-Null -if (Test-Path $bin) { Remove-Item -LiteralPath $bin -Recurse -Force -ErrorAction SilentlyContinue } -Copy-Item -LiteralPath $installDir -Destination $bin -Recurse -Force -# the installed engine carries Ember Tune from 0.3.12 on: prefer it; the kit is for a PC still on an older app -$installedVer = (& (Join-Path $installDir 'igneum-app.exe') --version 2>&1 | Out-String).Trim() -$installedHasEmber = $false -if ($installedVer -match 'igneum-app (\d+)\.(\d+)\.(\d+)') { $installedHasEmber = ([int]$Matches[1] -gt 0) -or ([int]$Matches[2] -gt 3) -or (([int]$Matches[2] -eq 3) -and ([int]$Matches[3] -ge 12)) } -$ember = $null -# a kit is used when the installed app predates Ember Tune (0.3.13), or when a run carries fixes the installed app -# lacks (ember-kit-6: the cmd quote fix and the self-registering elevated engine of 6 October 2026); a kit engine -# registers the Power Helper on the INSTALLED exe's path (powertask::task_exe), never its own -$k7 = Join-Path $appDir 'jobs\ember-kit-7\igneum-app-ember.exe' -$k6 = Join-Path $appDir 'jobs\ember-kit-6\igneum-app-ember.exe' -if (Test-Path $k7) { $ember = $k7 } -elseif (Test-Path $k6) { $ember = $k6 } -elseif (-not $installedHasEmber) { foreach ($cand in @((Join-Path $appDir 'jobs\ember-kit-2\igneum-app-ember.exe'), (Join-Path $appDir 'jobs\ember-kit-1\igneum-app-ember.exe'))) { if (Test-Path $cand) { $ember = $cand; break } } } -if ($ember) { - Copy-Item -LiteralPath $ember -Destination (Join-Path $bin 'igneum-app.exe') -Force - Say ("engine: the Ember build from " + $ember) -} else { - Say ('engine: the installed one (' + $installedVer + $(if ($installedHasEmber) { ', carries Ember Tune' } else { '; no kit found: an older engine ignores the tune and reports no_rows' }) + ')') -} -$helper = $null -$found = Get-ChildItem -Path (Join-Path $appDir 'jobs') -Recurse -Filter 'igneum-gpu-telemetry.exe' -ErrorAction SilentlyContinue | Where-Object { $_.FullName -match 'amd-kit' } | Sort-Object LastWriteTime -Descending | Select-Object -First 1 -if ($found) { $helper = $found.FullName } -if ($helper) { - Copy-Item -LiteralPath $helper -Destination (Join-Path $bin 'igneum-gpu-telemetry.exe') -Force - Say ("helper: the Ember build of igneum-gpu-telemetry from " + $helper + " sha256=" + (Get-FileHash -LiteralPath $helper -Algorithm SHA256).Hash.ToLower()) -} else { Say 'helper: the installed igneum-gpu-telemetry (no jobs\amd-kit-1\kit\igneum-gpu-telemetry.exe); without --tune the AMD card measures only' } -$exe = Join-Path $bin 'igneum-app.exe' -$ver = (& $exe --version 2>&1 | Out-String).Trim() -Say ("engine: " + $exe + " (" + $ver + ")") -Write-Output ("RESULT TUNE engine " + $ver + " sha256=" + (Get-FileHash -LiteralPath $exe -Algorithm SHA256).Hash.ToLower()) -if ($ver -notmatch 'igneum-app (\d+)\.(\d+)\.(\d+)') { Write-Output 'RESULT TUNE error=version_unknown'; exit 2 } - -foreach ($f in @('settings.json', 'machine-id', 'wallet.json', 'tuning.json', 'firewall-rule.json')) { # the firewall flag too: an older kit then asks nothing - $src = Join-Path $appDir $f - if (Test-Path $src) { Copy-Item -LiteralPath $src -Destination (Join-Path $sApp $f) -Force } -} -# the copies are VERBATIM (run 4, 6 October 2026: a PowerShell ConvertFrom-Json | ConvertTo-Json round trip rewrote big -# integers as doubles, the engine read the file as defaults, no payout address, every card off, 96 old jobs run in -# the scratch root); the engine itself switches remote jobs, updates, proving and Power control off under --sweep -# (Settings::for_measurement) and makes every card due. Only a report line is read here. -$sj = Join-Path $sApp 'settings.json' -if (-not (Test-Path -LiteralPath $sj)) { Write-Output 'RESULT TUNE error=no_settings reason=the_installed_app_has_no_settings.json'; exit 2 } -$installedPowerControl = 'unknown'; $addr = ''; $ncards = 0 -try { $back = Get-Content -LiteralPath $sj -Raw | ConvertFrom-Json; $addr = [string]$back.address; if ($back.cards) { $ncards = @($back.cards.PSObject.Properties).Count }; if ($back.PSObject.Properties.Name -contains 'power_control') { $installedPowerControl = ([bool]$back.power_control).ToString().ToLower() } } catch { } -$bom = (Get-Content -LiteralPath $sj -Encoding Byte -TotalCount 3 -ErrorAction SilentlyContinue) -join ',' -Write-Output ('RESULT TUNE installed_power_control=' + $installedPowerControl + ' (the tune engine runs with it off: no prompt unless the job itself is elevated)') -Write-Output ('RESULT TUNE scratch settings (verbatim copy): address ' + $(if ($addr) { $addr.Substring(0, [Math]::Min(10, $addr.Length)) + '...' } else { 'EMPTY' }) + ', cards ' + $ncards + ', first bytes ' + $bom + ', ' + (Get-Item -LiteralPath $sj).Length + ' bytes') -if (-not $addr -and -not (Test-Path (Join-Path $sApp 'wallet.json'))) { Write-Output 'RESULT TUNE error=no_address reason=the_copied_settings_carry_no_payout_address_and_no_wallet.json'; exit 2 } -Remove-Item -LiteralPath (Join-Path $sApp 'app.url') -Force -ErrorAction SilentlyContinue - -# the state before, for the report -$smi = Join-Path $env:ProgramFiles 'NVIDIA Corporation\NVSMI\nvidia-smi.exe' -if (-not (Test-Path $smi)) { $smi = Join-Path $env:SystemRoot 'System32\nvidia-smi.exe' } -$tele = Join-Path $bin 'igneum-gpu-telemetry.exe' -function Snapshot([string] $tag) { - if (Test-Path $smi) { - $q = (& $smi --query-gpu=index,name,driver_version,power.draw,power.limit,power.default_limit,power.min_limit,power.max_limit,clocks.gr,clocks.max.gr,clocks.mem --format=csv,noheader 2>&1 | Out-String).Trim() - Write-Output ("RESULT TUNE " + $tag + " nvidia " + ($q -replace "`r?`n", ' | ')) - } - if (Test-Path $tele) { - $t = (& $tele --tune 2>&1 | Out-String).Trim() - Write-Output ("RESULT TUNE " + $tag + " amd " + ($t -replace "`r?`n", ' | ')) - } else { Write-Output ("RESULT TUNE " + $tag + " amd no_helper") } -} -Snapshot 'before' - -# the tune engine: status every 10 s (6 rate samples per 60 s hold) -$env:IGNEUM_APP_DATA = $root -$env:IGNEUM_APP_LOGS = $sLogs -$env:IGNEUM_APP_STATUS_SECS = '10' -$env:IGNEUM_APP_NO_OTA = '1' # C35: a second engine never runs the updater (the installer would quit the installed app) -# C35 (5 October 2026): the engine's output goes to a FILE, never a pipe. A pipe's write end is inherited by every -# process the engine starts (its miners and workers), so after an abort the installed app's jobs runner waits for an -# EOF that never comes and hangs in its own quit; and the engine's whole tree is killed at the end (nothing orphaned). -$outFile = Join-Path $root 'engine-stdout.log' -$errFile = Join-Path $root 'engine-stderr.log' -Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue -$p = Start-Process -FilePath $exe -ArgumentList '--sweep' -WorkingDirectory (Split-Path $exe) -WindowStyle Hidden -PassThru -RedirectStandardOutput $outFile -RedirectStandardError $errFile -Say ("engine started, pid " + $p.Id + ", data " + $root + ", stdout " + $outFile) -function EndTree([int] $procId, [string] $why) { - $before = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count - & taskkill /T /F /PID $procId 2>&1 | Out-Null - Start-Sleep -Seconds 2 - $after = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count - Write-Output ('RESULT ' + $resultTag + ' tree ended (' + $why + '): igneum processes ' + $before + ' -> ' + $after + ' (the installed app''s own miners are stopped and held by the job)') -} -$seen = 0 -$rows = 0 -# the installed app shows the tune instead of a bare "off" (the project lead, 6 October 2026): every `TUNE progress` line and the -# `TUNE chosen` line go to its /api/tune-progress (a POST of state, never quit, pause or resume: the 5 October rule) -$installedUrl = '' -try { $installedUrl = (Get-Content -LiteralPath (Join-Path $appDir 'app.url') -Raw -ErrorAction Stop).Trim() } catch { } -function Forward([string] $line) { - if (-not $installedUrl) { return } - $kv = @{} - foreach ($tok in ($line -split ' ')) { if ($tok -match '^([a-z_]+)=(.*)$') { $kv[$Matches[1]] = $Matches[2] } } - $body = @{ key = ($kv['card'] -replace '_', ' '); done = ($line -match '^TUNE chosen ') } - if ($kv['key']) { $body.key = ($kv['key'] -replace '_', ' ') } - foreach ($n in @('step', 'of', 'eta_s', 'mhs', 'watts', 'clock', 'cap')) { if ($kv[$n] -ne $null) { $body[$n] = [double]$kv[$n] } } - if ($kv['cap'] -ne $null) { $body['power_pct'] = [double]$kv['cap'] } - if ($kv['clock'] -ne $null) { $body['clock_mhz'] = [double]$kv['clock'] } - if ($kv['plan']) { $body.plan = $kv['plan'] } - if ($kv['note']) { $body.note = ($kv['note'] -replace '_', ' ') } - if ($body.done) { $body.note = ('tuned: ' + $kv['mhs'] + ' MH/s at ' + $kv['watts'] + ' W') } - try { Invoke-WebRequest -Uri ($installedUrl + 'api/tune-progress') -Method POST -Body ($body | ConvertTo-Json -Compress) -ContentType 'application/json' -UseBasicParsing -TimeoutSec 3 | Out-Null } catch { } -} -# the watchdog (coordinator, 6 October 2026): a tune engine that mines nothing for 120 s after its first status line -# (every card "waiting" or 0.00 MH/s: no payout address, no worker, no node) fails the job at once with the engine's -# last log line in the RESULT, its tree ended, mining restored by the job runner; a job that cannot mine never burns -# its budget silently again -$firstStatusAt = $null -$lastMining = $null -$idleSince = $null -$idleSamples = 0 -$lastIdleStatus = '' -$lastEngineLine = '' -function EngineLogDump([string] $why) { - Write-Output ('===== engine log tail (' + $why + ')') - $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 - if ($t) { Get-Content -LiteralPath $t.FullName -ErrorAction SilentlyContinue | Where-Object { $_ -notmatch 'status: accepted 0 blocks' } | Select-Object -Last 80 | ForEach-Object { Write-Output (' ' + $_) } } else { Write-Output ' (no app-*.log in the scratch logs folder)' } - if (Test-Path -LiteralPath $errFile) { Write-Output '===== engine stderr'; Get-Content -LiteralPath $errFile -ErrorAction SilentlyContinue | Select-Object -Last 20 | ForEach-Object { Write-Output (' ' + $_) } } -} -# the newest `status:` line among the engine log's last 60 lines (run 5, 15:39Z: a single sampled tail line missed the -# status lines between TUNE and telemetry lines for 300 s and the watchdog killed a live three-card tune) -function EngineStatus() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $ls = @(Get-Content -LiteralPath $t.FullName -Tail 60 -ErrorAction SilentlyContinue); for ($i = $ls.Count - 1; $i -ge 0; $i--) { if ([string]$ls[$i] -match ' status: ') { return [string]$ls[$i] } } }; return '' } -function EngineTail() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $l = Get-Content -LiteralPath $t.FullName -Tail 1 -ErrorAction SilentlyContinue; if ($l) { return [string]$l } }; return '' } -function Exit3([string] $why, [string] $reason) { - Write-Output ('RESULT TUNE error=' + $reason + ' last_status_line=' + ((EngineStatus) -replace '\s+', '_') + ' last_log_line=' + ($lastEngineLine -replace '\s+', '_')) - EngineLogDump $why - EndTree $p.Id $why - Snapshot 'after' - Write-Output 'RESULT TUNE error=no_rows' - exit 3 -} -# the AMD watts source (coordinator, 6 October 2026 16:00Z): the 3.0 coordinator's job read ZERO watts for the 9070 XT -# from igneum-gpu-telemetry.exe while the installed app had read 198.9 W the day before. The tune engine samples its -# own helper; this script also samples the installed app's /api/state power_w for the AMD card every 10 s (a GET, -# never a write) and, when the engine's AMD progress lines carry no watts, prints a fallback ladder from those -# samples (median per step) and names the source it used -$amdApp = @(); $amdStep = 0; $amdMhs = 0.0; $amdEngNZ = 0; $amdEngZ = 0; $tick = 0; $amdMhsByStep = @{} -while (-not $p.HasExited) { - Start-Sleep -Seconds 5 - $tick++ - if ($installedUrl -and ($tick % 2 -eq 0)) { - try { - $js = (Invoke-WebRequest -Uri ($installedUrl + 'api/state') -UseBasicParsing -TimeoutSec 3).Content | ConvertFrom-Json - foreach ($c in @($js.mining.cards)) { if ($c.vendor -eq 'amd') { $amdApp += ,@{ step = $amdStep; w = [double]$c.power_w } } } - } catch { } - } - $tailLine = EngineTail - if ($tailLine) { $lastEngineLine = $tailLine } - $statusLine = EngineStatus - if ($statusLine) { - if (-not $firstStatusAt) { $firstStatusAt = Get-Date } - $rate = 0.0; if ($statusLine -match '(\d+\.\d+) MH/s') { $rate = [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) } - # idle = three consecutive 10-s samples whose newest status line reads 0.00 MH/s (never a missing match) - if ($rate -gt 0) { $lastMining = Get-Date; $idleSamples = 0; $idleSince = $null } - elseif ($statusLine -ne $lastIdleStatus) { $lastIdleStatus = $statusLine; $idleSamples++; if (-not $idleSince) { $idleSince = Get-Date } } - } - if ($firstStatusAt -and -not $lastMining -and ((Get-Date) - $firstStatusAt).TotalSeconds -gt 120) { Exit3 'watchdog: not mining' 'not_mining reason=no_card_mined_within_120_s_of_the_first_status_line' } - if ($lastMining -and $idleSamples -ge 3 -and $idleSince -and ((Get-Date) - $idleSince).TotalSeconds -gt 300) { Exit3 'watchdog: stopped mining' 'stopped_mining reason=three_status_lines_read_0.00_MH/s_and_300_s_passed' } - $all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } - while ($seen -lt $all.Count) { - $l = [string]$all[$seen]; $seen++ - if ($l -match '^TUNE progress ') { - Forward $l - if ($l -match ' key=amd' -and $l -match ' step=(\d+)') { $amdStep = [int]$Matches[1] - if ($l -match ' mhs=([\d.]+)') { $amdMhsByStep[$amdStep] = [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) } - if ($l -match ' watts=([\d.]+)' -and [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) -gt 0) { $amdEngNZ++ } else { $amdEngZ++ } } - } - elseif ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ }; if ($l -match '^TUNE chosen ') { Forward $l } } - elseif ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l) } - elseif ($l -match '^(URL|STATE) ') { } - else { Say $l } - } - if ((Get-Date) -gt $deadline) { - Say ("budget of " + $budgetMinutes + " min spent; asking the tune engine to quit") - # C35 (5 October 2026): the only quit this script may send goes to the TUNE engine's own URL file in the scratch - # root, never to a file under the installed app's folder; the RESULT line names the file it used - $u = Join-Path $sApp 'app.url' - # the only URL file this script may quit is its own scratch root's; the installed app's folder is refused by name - if ((Resolve-Path -LiteralPath $u -ErrorAction SilentlyContinue).Path -like (Join-Path $appDir '*') -or $u -like '*\igneum\app\*') { - Write-Output ('RESULT TUNE quit refused: ' + $u + ' is the installed app''s URL file') - } elseif (Test-Path -LiteralPath $u) { - Write-Output ('RESULT TUNE quit asked of the tune engine through ' + $u + ' (pid ' + $p.Id + ')') - try { Invoke-WebRequest -Uri ((Get-Content -LiteralPath $u -Raw).Trim() + 'api/quit') -Method POST -Body '{}' -ContentType 'application/json' -UseBasicParsing -TimeoutSec 5 | Out-Null } catch { } - } else { Write-Output ('RESULT TUNE quit not sent: no URL file at ' + $u + '; killing pid ' + $p.Id) } - Start-Sleep -Seconds 20 - if (-not $p.HasExited) { EndTree $p.Id 'budget' } - EngineLogDump 'budget' - Write-Output 'RESULT TUNE error=budget_exceeded' - } -} -$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } -while ($seen -lt $all.Count) { $l = [string]$all[$seen]; $seen++; if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } } -$amdNZ = @($amdApp | Where-Object { $_.w -gt 0 }) -Write-Output ('RESULT TUNE amd_watts engine_progress nonzero=' + $amdEngNZ + ' zero=' + $amdEngZ + ' app_state samples=' + $amdApp.Count + ' nonzero=' + $amdNZ.Count) -$amdSource = 'none' -if ($amdEngNZ -gt 0) { $amdSource = 'engine_telemetry' } -elseif ($amdNZ.Count -gt 0) { - $amdSource = 'app_state_fallback' - foreach ($g in ($amdNZ | Group-Object { $_.step } | Sort-Object { [int]$_.Name })) { - $ws = @($g.Group | ForEach-Object { $_.w } | Sort-Object); $med = $ws[[int][math]::Floor($ws.Count / 2)] - $mh = 0.0; if ($amdMhsByStep.ContainsKey([int]$g.Name)) { $mh = $amdMhsByStep[[int]$g.Name] } - $eff = 0.0; if ($med -gt 0) { $eff = $mh / $med } - Write-Output ('RESULT TUNE amd_fallback source=app_state step=' + $g.Name + ' watts=' + ('{0:F1}' -f $med) + ' mhs=' + ('{0:F2}' -f $mh) + ' eff=' + ('{0:F4}' -f $eff) + ' samples=' + $ws.Count) - } -} -Write-Output ('RESULT TUNE amd_watts_source=' + $amdSource) -Say ("tune engine exited " + $p.ExitCode + " after " + [int]((Get-Date) - $started).TotalSeconds + " s, " + $rows + " table rows") -EndTree $p.Id 'end of run' -# the project lead, 6 October 2026, 07:25Z: both cards stay on their best MH/W points (the tune pins them); no factory reset here. -Snapshot 'after' -# the tune engine's own log: the TUNE lines and what happened around them -$log = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 -if ($log) { - Say ("engine log " + $log.FullName + ":") - Get-Content -LiteralPath $log.FullName | Where-Object { $_ -match 'TUNE|tune|power cap|GPUs:|worker ready|STATUS|exited|upload' } | Select-Object -Last 100 | ForEach-Object { Say (' ' + $_) } -} -if ($rows -eq 0) { Write-Output 'RESULT TUNE error=no_rows'; exit 1 } -exit 0 diff --git a/tools/ci/playbook-quit-check.sh b/tools/ci/playbook-quit-check.sh index aa0f48417..3f298a683 100755 --- a/tools/ci/playbook-quit-check.sh +++ b/tools/ci/playbook-quit-check.sh @@ -49,17 +49,10 @@ fi # PC 2 family probe and derive job; and the two hand-restore scripts pc1-amd-identities.ps1 and pc1-amd-flag-amd1.ps1, # allowed by the rule owner (6 October 2026, 19:xx UTC) because they post enabled TRUE only (a restore, never a switch off) PRE_RULE='^(relay/playbooks/(mixer-x4-pc1-bench|mixer-x4-5090-bench|mixer-x4-9070-bench|readwidth-5090-bench|readwidth-5090|readwidth-9070-bench|readwidth-9070|ca2-era-pc1|ca2-hot-5090-bench|ca2-hot-9070-bench|dot4-probe|ca3-derive-pc2)\.ps1|tools/ca3-reserve/pc2-family-probe\.ps1|tools/ca3-pc1-amd/pc1-amd-identities\.ps1|tools/ca3-pc1-amd/pc1-amd-flag-amd1\.ps1)$' -# 6 October 2026 (0.3.14 gate): ember-tune-pc1.ps1 and the two agg-cost scripts are allowed pending the agg-cost --stop-miners change -# and the Ember playbook's rewrite (the coordinator's 15:30Z ruling); EXPIRES 0.3.15 (the check below fails the tree at 0.3.15 or later while they stand) -ALLOW='^(packaging/windows/stop-igneum\.ps1|relay/playbooks/ember-tune-pc1\.ps1|tools/proving-v1/pc2-agg-cost\.ps1|tools/proving-v1/pc2-agg-cost-restore\.ps1)$' +# 6 October 2026 (0.3.15): the dated allow entries are gone (ember-tune-pc1.ps1 deleted, the agg-cost scripts on the +# runner's --cards-off); only the installer's own stop step remains. +ALLOW='^(packaging/windows/stop-igneum\.ps1)$' fail=0 while IFS= read -r f; do [[ "$f" =~ $ALLOW ]] && continue; [[ "$f" =~ $PRE_RULE ]] && continue; check_file "$f" || fail=1; done < <(git ls-files 'relay/playbooks/**' 'tools/**' 'packaging/**' | grep -E '\.(ps1|sh)$' | grep -v '^tools/ci/') -# expiry of the dated allow entries (6 October 2026): at 0.3.15 or later the three must be gone from ALLOW -ver="$(sed -n 's/^version = "\(.*\)"/\1/p' app/igneum-app/Cargo.toml 2>/dev/null | head -1)" -if [ -n "$ver" ] && [ "$(printf '%s\n0.3.15\n' "$ver" | sort -V | head -1)" = "0.3.15" ]; then - for e in relay/playbooks/ember-tune-pc1.ps1 tools/proving-v1/pc2-agg-cost.ps1 tools/proving-v1/pc2-agg-cost-restore.ps1; do - case "$ALLOW" in *"${e//./\\.}"*) echo "playbook-quit: the allow entry for $e expired at 0.3.15 (the tree says $ver): the pause/resume rule applies to it now"; fail=1 ;; esac - done -fi [ "$fail" = 0 ] && echo "playbook-quit: no playbook quits, pauses, resumes or switches the cards of the installed app" exit $fail diff --git a/tools/proving-v1/pc2-agg-cost-restore.ps1 b/tools/proving-v1/pc2-agg-cost-restore.ps1 index 9a735e591..af648c580 100644 --- a/tools/proving-v1/pc2-agg-cost-restore.ps1 +++ b/tools/proving-v1/pc2-agg-cost-restore.ps1 @@ -40,17 +40,9 @@ foreach ($p in (Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'ig } "RESULT strays_stopped $(Stamp) $killed" & wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') } -function Card($st) { if ($null -eq $st) { return $null }; $st.mining.cards | Where-Object { $_.vendor -eq 'nvidia' } | Select-Object -First 1 } -$c = Card (State) -if ($c) { - $stc = State - "RESULT cards_before $(Stamp) $(($stc.mining.cards | ForEach-Object { "$($_.name):enabled=$($_.enabled):identities=$($_.identities):$($_.state)" }) -join ',')" - $list = @(); foreach ($k in $stc.mining.cards) { $list += @{ key = $k.key; enabled = $(if ($k.key -eq $c.key) { $true } else { [bool]$k.enabled }); identities = [int]$k.identities } } - "RESULT card_on $(Stamp) $(Post '/api/cards' @{ cards = $list })" -} +# 6 October 2026 (0.3.15): no card switch, resume or start from a script (CLAUDE.md job rule); the runner of the measurement +# job restores the card and the miners on any exit, so this script only cleans up strays and reads the state back. "RESULT timeslice $(Stamp) $((& nvidia-smi compute-policy --set-timeslice=0 2>&1) -join ' ')" -"RESULT resume $(Stamp) $(Post '/api/resume' @{})" -"RESULT start $(Stamp) $(Post '/api/start' @{})" "RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})" Start-Sleep -Seconds 20 $st = State diff --git a/tools/proving-v1/pc2-agg-cost.ps1 b/tools/proving-v1/pc2-agg-cost.ps1 index d7263fc4d..78ec492c2 100644 --- a/tools/proving-v1/pc2-agg-cost.ps1 +++ b/tools/proving-v1/pc2-agg-cost.ps1 @@ -92,12 +92,7 @@ function CardPrefs { return $prefs } function NvidiaKey { $p = CardPrefs; ($p.Keys | Where-Object { $_ -like 'nvidia:*' } | Select-Object -First 1) } -function CardSwitch($on, $identities) { - $p = CardPrefs; $nk = NvidiaKey - if (-not $nk) { return "no nvidia card key in settings.json (keys: $($p.Keys -join ','))" } - $list = @(); foreach ($k in $p.Keys) { $list += @{ key = $k; enabled = $(if ($k -eq $nk) { [bool]$on } else { [bool]$p[$k].enabled }); identities = $(if ($k -eq $nk -and $identities) { [int]$identities } else { [int]$p[$k].identities }) } } - "prefs_before=$(($p.Keys | ForEach-Object { "$($_):enabled=$($p[$_].enabled):identities=$($p[$_].identities)" }) -join ',') " + (Post '/api/cards' @{ cards = $list }) -} +# CardSwitch removed 6 October 2026: the runner's --cards-off owns the card (see the needIdle block) function CudaWorkers { (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' } | Measure-Object).Count } $job = $env:IGNEUM_JOB_DIR if (-not $job) { $job = Join-Path $env:TEMP 'igneum-agg-cost' } @@ -346,12 +341,15 @@ try { if ($PhaseSet -contains 'B') { RunPhase 'B' "SP1_WORKER_VERIFY_INTERMEDIATES=false SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B.json" } $needIdle = @('C','C1','C2','G0','D','E','E0','F','G','H') | Where-Object { $PhaseSet -contains $_ } if ($needIdle) { - # the 5090 switched off through /api/cards (and on again in the finally block): /api/pause then /api/resume left - # the worker off on 0.3.9 (21:25Z, an app defect in the resume path, fixed for 0.3.11) - "RESULT card_off $(Stamp) $(CardSwitch $false $null)"; $paused = $true - $w = 0; $left = 1 + # 6 October 2026 (0.3.15): the script never switches a card or pauses the installed app (CLAUDE.md job rule, the + # playbook-quit check). The job is published with --cards-off : the RUNNER switches the 5090 off through + # the app's own card path before the script and puts it back on any exit. Here the script only confirms the worker + # is gone, and fails loud if it is not (no pause fallback: a script that dies before its finally left the box + # paused unattended). + $w = 0; $left = CudaWorkers while ($w -lt 120 -and $left -gt 0) { Start-Sleep -Seconds 5; $w += 5; $left = CudaWorkers } - if ($left -gt 0) { "RESULT card_off_failed $(Stamp) the app's CUDA worker is still running after $w s ($left); /api/pause as the fallback: $(Post '/api/pause' @{})"; $w2 = 0; while ($w2 -lt 60 -and (CudaWorkers) -gt 0) { Start-Sleep -Seconds 5; $w2 += 5 }; $left = CudaWorkers } + if ($left -gt 0) { "RESULT card_off_failed $(Stamp) the app's CUDA worker is still running after $w s ($left): publish this job with --cards-off (the runner switches it off); nothing measured"; exit 3 } + $paused = $true "RESULT paused $(Stamp) after $w s; CUDA workers left: $left; $(Smi)" Start-Sleep -Seconds 5 } @@ -423,11 +421,7 @@ try { } finally { StopMiner 'final' & wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') } - if ($paused) { - "RESULT card_on $(Stamp) $(CardSwitch $true $null)"; "RESULT resume $(Stamp) $(Post '/api/resume' @{})" - $w3 = 0; while ($w3 -lt 90 -and (CudaWorkers) -eq 0) { Start-Sleep -Seconds 5; $w3 += 5 } - if ((CudaWorkers) -eq 0) { "RESULT miner_back_failed $(Stamp) no CUDA worker after $w3 s; /api/start: $(Post '/api/start' @{})" } else { "RESULT miner_back $(Stamp) CUDA worker running after $w3 s" } - } + if ($paused) { "RESULT card_restore $(Stamp) left to the runner (--cards-off puts the card back on any exit; the miners follow)" } "RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})" "RESULT end $(Stamp) $(Smi)" }