0.3.15: the playbook-quit allow list is only the installer's stop step; the agg-cost scripts on the runner's --cards-off; ember-tune-pc1.ps1 deleted
The dated entries of the 0.3.14 gate expired at 0.3.15. pc2-agg-cost.ps1 no longer switches the 5090 or falls back to /api/pause: the job is published with --cards-off <nvidia key>, the runner switches the card off before the script and puts it back on any exit, and the script fails loud (exit 3) if a CUDA worker is still running. pc2-agg-cost-restore.ps1 keeps the stray and socket clean-up and the read-back only. ember-tune-pc1.ps1 is superseded by the installed-tune playbook on ember-tune. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
parent
f57c5293bc
commit
95707125c5
4 changed files with 15 additions and 301 deletions
|
|
@ -1,265 +0,0 @@
|
|||
# Igneum run job: Ember Tune end to end on PC 1 (machine ae432dc7), unattended. 5 October 2026.
|
||||
# Published as a `run` job with --stop-miners (docs/plans/ember-tune.md): the installed app stops its miners and
|
||||
# holds them; this script takes the engine that carries src/ember.rs (the one the fetch job put in
|
||||
# <app dir>\jobs\ember-kit-1\igneum-app-ember.exe, else the installed one; the AMD helper with the --tune and --set
|
||||
# commands from the telemetry agent's fetch job, <app dir>\jobs\amd-kit-1\kit\igneum-gpu-telemetry.exe), copies the install folder to a scratch
|
||||
# folder beside it, swaps the engine in, and starts that SECOND engine with `--sweep` in a scratch data folder (the
|
||||
# real settings.json, machine-id and wallet.json copied in; remote jobs, auto-update and proving switched off
|
||||
# there). That engine finds the installed app's node on 127.0.0.1:26610, mines on every card with the live program,
|
||||
# tunes them one after the other (the full two-knob plan where the card can be controlled, the baseline measurement
|
||||
# where it cannot: NVIDIA without administrator rights, Apple), prints every TUNE line on stdout, uploads its log
|
||||
# (the TUNE {json} record reaches the intake) and quits. Every TUNE line is re-emitted as a RESULT line, so
|
||||
# `node tools/jobs.mjs <job id>` shows the table. Before and after, nvidia-smi's limits and clocks and the AMD
|
||||
# helper's `--tune` lines are printed, so the restore can be read. The installed app's miners restart when the job
|
||||
# ends. Not elevated: nothing asks for administrator rights (the project lead asleep, 5 October 2026); the NVIDIA card is
|
||||
# therefore measure only tonight unless the engine finds itself elevated.
|
||||
$ErrorActionPreference = 'Continue'
|
||||
$resultTag = 'TUNE'
|
||||
$budgetMinutes = 55
|
||||
if (-not ($budgetMinutes -is [int]) -or $budgetMinutes -lt 5) { $budgetMinutes = 35 } # a budget under 5 minutes is a bug, not a budget (C35)
|
||||
$started = Get-Date
|
||||
$deadline = $started.AddMinutes($budgetMinutes)
|
||||
function Say([string] $m) { Write-Host ("[" + (Get-Date -Format 'HH:mm:ss') + "] " + $m) }
|
||||
|
||||
# the installed engine: the per-user install (0.3.3+), else Program Files
|
||||
$installDir = $null
|
||||
foreach ($d in @((Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner'), (Join-Path $env:ProgramFiles 'Igneum Miner'))) {
|
||||
if (Test-Path (Join-Path $d 'igneum-app.exe')) { $installDir = $d; break }
|
||||
}
|
||||
if (-not $installDir) { Write-Output 'RESULT TUNE error=no_engine reason=igneum-app.exe_not_found'; exit 2 }
|
||||
$appData = $env:IGNEUM_APP_DATA
|
||||
if (-not $appData) { $appData = Join-Path $env:LOCALAPPDATA 'igneum' }
|
||||
$appDir = $env:IGNEUM_APP_DIR
|
||||
if (-not $appDir) { $appDir = Join-Path $appData 'app' }
|
||||
|
||||
# the scratch install: the whole folder (workers, node, helper, DLLs) with the Ember engine swapped in
|
||||
# a FRESH scratch root per run (run 3, 6 October 2026, 11:45Z: the folder an earlier elevated engine had locked to itself
|
||||
# refused the settings copy, the engine started on stale files and exited in 6 s); old roots are small and left alone
|
||||
$root = Join-Path $env:LOCALAPPDATA ('igneum-tune-' + (Get-Date -Format 'yyyyMMdd-HHmmss'))
|
||||
$bin = Join-Path $root 'bin'
|
||||
$sApp = Join-Path $root 'app'
|
||||
$sLogs = Join-Path $root 'logs'
|
||||
New-Item -ItemType Directory -Force -Path $root, $sApp, $sLogs | Out-Null
|
||||
if (Test-Path $bin) { Remove-Item -LiteralPath $bin -Recurse -Force -ErrorAction SilentlyContinue }
|
||||
Copy-Item -LiteralPath $installDir -Destination $bin -Recurse -Force
|
||||
# the installed engine carries Ember Tune from 0.3.12 on: prefer it; the kit is for a PC still on an older app
|
||||
$installedVer = (& (Join-Path $installDir 'igneum-app.exe') --version 2>&1 | Out-String).Trim()
|
||||
$installedHasEmber = $false
|
||||
if ($installedVer -match 'igneum-app (\d+)\.(\d+)\.(\d+)') { $installedHasEmber = ([int]$Matches[1] -gt 0) -or ([int]$Matches[2] -gt 3) -or (([int]$Matches[2] -eq 3) -and ([int]$Matches[3] -ge 12)) }
|
||||
$ember = $null
|
||||
# a kit is used when the installed app predates Ember Tune (0.3.13), or when a run carries fixes the installed app
|
||||
# lacks (ember-kit-6: the cmd quote fix and the self-registering elevated engine of 6 October 2026); a kit engine
|
||||
# registers the Power Helper on the INSTALLED exe's path (powertask::task_exe), never its own
|
||||
$k7 = Join-Path $appDir 'jobs\ember-kit-7\igneum-app-ember.exe'
|
||||
$k6 = Join-Path $appDir 'jobs\ember-kit-6\igneum-app-ember.exe'
|
||||
if (Test-Path $k7) { $ember = $k7 }
|
||||
elseif (Test-Path $k6) { $ember = $k6 }
|
||||
elseif (-not $installedHasEmber) { foreach ($cand in @((Join-Path $appDir 'jobs\ember-kit-2\igneum-app-ember.exe'), (Join-Path $appDir 'jobs\ember-kit-1\igneum-app-ember.exe'))) { if (Test-Path $cand) { $ember = $cand; break } } }
|
||||
if ($ember) {
|
||||
Copy-Item -LiteralPath $ember -Destination (Join-Path $bin 'igneum-app.exe') -Force
|
||||
Say ("engine: the Ember build from " + $ember)
|
||||
} else {
|
||||
Say ('engine: the installed one (' + $installedVer + $(if ($installedHasEmber) { ', carries Ember Tune' } else { '; no kit found: an older engine ignores the tune and reports no_rows' }) + ')')
|
||||
}
|
||||
$helper = $null
|
||||
$found = Get-ChildItem -Path (Join-Path $appDir 'jobs') -Recurse -Filter 'igneum-gpu-telemetry.exe' -ErrorAction SilentlyContinue | Where-Object { $_.FullName -match 'amd-kit' } | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
||||
if ($found) { $helper = $found.FullName }
|
||||
if ($helper) {
|
||||
Copy-Item -LiteralPath $helper -Destination (Join-Path $bin 'igneum-gpu-telemetry.exe') -Force
|
||||
Say ("helper: the Ember build of igneum-gpu-telemetry from " + $helper + " sha256=" + (Get-FileHash -LiteralPath $helper -Algorithm SHA256).Hash.ToLower())
|
||||
} else { Say 'helper: the installed igneum-gpu-telemetry (no jobs\amd-kit-1\kit\igneum-gpu-telemetry.exe); without --tune the AMD card measures only' }
|
||||
$exe = Join-Path $bin 'igneum-app.exe'
|
||||
$ver = (& $exe --version 2>&1 | Out-String).Trim()
|
||||
Say ("engine: " + $exe + " (" + $ver + ")")
|
||||
Write-Output ("RESULT TUNE engine " + $ver + " sha256=" + (Get-FileHash -LiteralPath $exe -Algorithm SHA256).Hash.ToLower())
|
||||
if ($ver -notmatch 'igneum-app (\d+)\.(\d+)\.(\d+)') { Write-Output 'RESULT TUNE error=version_unknown'; exit 2 }
|
||||
|
||||
foreach ($f in @('settings.json', 'machine-id', 'wallet.json', 'tuning.json', 'firewall-rule.json')) { # the firewall flag too: an older kit then asks nothing
|
||||
$src = Join-Path $appDir $f
|
||||
if (Test-Path $src) { Copy-Item -LiteralPath $src -Destination (Join-Path $sApp $f) -Force }
|
||||
}
|
||||
# the copies are VERBATIM (run 4, 6 October 2026: a PowerShell ConvertFrom-Json | ConvertTo-Json round trip rewrote big
|
||||
# integers as doubles, the engine read the file as defaults, no payout address, every card off, 96 old jobs run in
|
||||
# the scratch root); the engine itself switches remote jobs, updates, proving and Power control off under --sweep
|
||||
# (Settings::for_measurement) and makes every card due. Only a report line is read here.
|
||||
$sj = Join-Path $sApp 'settings.json'
|
||||
if (-not (Test-Path -LiteralPath $sj)) { Write-Output 'RESULT TUNE error=no_settings reason=the_installed_app_has_no_settings.json'; exit 2 }
|
||||
$installedPowerControl = 'unknown'; $addr = ''; $ncards = 0
|
||||
try { $back = Get-Content -LiteralPath $sj -Raw | ConvertFrom-Json; $addr = [string]$back.address; if ($back.cards) { $ncards = @($back.cards.PSObject.Properties).Count }; if ($back.PSObject.Properties.Name -contains 'power_control') { $installedPowerControl = ([bool]$back.power_control).ToString().ToLower() } } catch { }
|
||||
$bom = (Get-Content -LiteralPath $sj -Encoding Byte -TotalCount 3 -ErrorAction SilentlyContinue) -join ','
|
||||
Write-Output ('RESULT TUNE installed_power_control=' + $installedPowerControl + ' (the tune engine runs with it off: no prompt unless the job itself is elevated)')
|
||||
Write-Output ('RESULT TUNE scratch settings (verbatim copy): address ' + $(if ($addr) { $addr.Substring(0, [Math]::Min(10, $addr.Length)) + '...' } else { 'EMPTY' }) + ', cards ' + $ncards + ', first bytes ' + $bom + ', ' + (Get-Item -LiteralPath $sj).Length + ' bytes')
|
||||
if (-not $addr -and -not (Test-Path (Join-Path $sApp 'wallet.json'))) { Write-Output 'RESULT TUNE error=no_address reason=the_copied_settings_carry_no_payout_address_and_no_wallet.json'; exit 2 }
|
||||
Remove-Item -LiteralPath (Join-Path $sApp 'app.url') -Force -ErrorAction SilentlyContinue
|
||||
|
||||
# the state before, for the report
|
||||
$smi = Join-Path $env:ProgramFiles 'NVIDIA Corporation\NVSMI\nvidia-smi.exe'
|
||||
if (-not (Test-Path $smi)) { $smi = Join-Path $env:SystemRoot 'System32\nvidia-smi.exe' }
|
||||
$tele = Join-Path $bin 'igneum-gpu-telemetry.exe'
|
||||
function Snapshot([string] $tag) {
|
||||
if (Test-Path $smi) {
|
||||
$q = (& $smi --query-gpu=index,name,driver_version,power.draw,power.limit,power.default_limit,power.min_limit,power.max_limit,clocks.gr,clocks.max.gr,clocks.mem --format=csv,noheader 2>&1 | Out-String).Trim()
|
||||
Write-Output ("RESULT TUNE " + $tag + " nvidia " + ($q -replace "`r?`n", ' | '))
|
||||
}
|
||||
if (Test-Path $tele) {
|
||||
$t = (& $tele --tune 2>&1 | Out-String).Trim()
|
||||
Write-Output ("RESULT TUNE " + $tag + " amd " + ($t -replace "`r?`n", ' | '))
|
||||
} else { Write-Output ("RESULT TUNE " + $tag + " amd no_helper") }
|
||||
}
|
||||
Snapshot 'before'
|
||||
|
||||
# the tune engine: status every 10 s (6 rate samples per 60 s hold)
|
||||
$env:IGNEUM_APP_DATA = $root
|
||||
$env:IGNEUM_APP_LOGS = $sLogs
|
||||
$env:IGNEUM_APP_STATUS_SECS = '10'
|
||||
$env:IGNEUM_APP_NO_OTA = '1' # C35: a second engine never runs the updater (the installer would quit the installed app)
|
||||
# C35 (5 October 2026): the engine's output goes to a FILE, never a pipe. A pipe's write end is inherited by every
|
||||
# process the engine starts (its miners and workers), so after an abort the installed app's jobs runner waits for an
|
||||
# EOF that never comes and hangs in its own quit; and the engine's whole tree is killed at the end (nothing orphaned).
|
||||
$outFile = Join-Path $root 'engine-stdout.log'
|
||||
$errFile = Join-Path $root 'engine-stderr.log'
|
||||
Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue
|
||||
$p = Start-Process -FilePath $exe -ArgumentList '--sweep' -WorkingDirectory (Split-Path $exe) -WindowStyle Hidden -PassThru -RedirectStandardOutput $outFile -RedirectStandardError $errFile
|
||||
Say ("engine started, pid " + $p.Id + ", data " + $root + ", stdout " + $outFile)
|
||||
function EndTree([int] $procId, [string] $why) {
|
||||
$before = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count
|
||||
& taskkill /T /F /PID $procId 2>&1 | Out-Null
|
||||
Start-Sleep -Seconds 2
|
||||
$after = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count
|
||||
Write-Output ('RESULT ' + $resultTag + ' tree ended (' + $why + '): igneum processes ' + $before + ' -> ' + $after + ' (the installed app''s own miners are stopped and held by the job)')
|
||||
}
|
||||
$seen = 0
|
||||
$rows = 0
|
||||
# the installed app shows the tune instead of a bare "off" (the project lead, 6 October 2026): every `TUNE progress` line and the
|
||||
# `TUNE chosen` line go to its /api/tune-progress (a POST of state, never quit, pause or resume: the 5 October rule)
|
||||
$installedUrl = ''
|
||||
try { $installedUrl = (Get-Content -LiteralPath (Join-Path $appDir 'app.url') -Raw -ErrorAction Stop).Trim() } catch { }
|
||||
function Forward([string] $line) {
|
||||
if (-not $installedUrl) { return }
|
||||
$kv = @{}
|
||||
foreach ($tok in ($line -split ' ')) { if ($tok -match '^([a-z_]+)=(.*)$') { $kv[$Matches[1]] = $Matches[2] } }
|
||||
$body = @{ key = ($kv['card'] -replace '_', ' '); done = ($line -match '^TUNE chosen ') }
|
||||
if ($kv['key']) { $body.key = ($kv['key'] -replace '_', ' ') }
|
||||
foreach ($n in @('step', 'of', 'eta_s', 'mhs', 'watts', 'clock', 'cap')) { if ($kv[$n] -ne $null) { $body[$n] = [double]$kv[$n] } }
|
||||
if ($kv['cap'] -ne $null) { $body['power_pct'] = [double]$kv['cap'] }
|
||||
if ($kv['clock'] -ne $null) { $body['clock_mhz'] = [double]$kv['clock'] }
|
||||
if ($kv['plan']) { $body.plan = $kv['plan'] }
|
||||
if ($kv['note']) { $body.note = ($kv['note'] -replace '_', ' ') }
|
||||
if ($body.done) { $body.note = ('tuned: ' + $kv['mhs'] + ' MH/s at ' + $kv['watts'] + ' W') }
|
||||
try { Invoke-WebRequest -Uri ($installedUrl + 'api/tune-progress') -Method POST -Body ($body | ConvertTo-Json -Compress) -ContentType 'application/json' -UseBasicParsing -TimeoutSec 3 | Out-Null } catch { }
|
||||
}
|
||||
# the watchdog (coordinator, 6 October 2026): a tune engine that mines nothing for 120 s after its first status line
|
||||
# (every card "waiting" or 0.00 MH/s: no payout address, no worker, no node) fails the job at once with the engine's
|
||||
# last log line in the RESULT, its tree ended, mining restored by the job runner; a job that cannot mine never burns
|
||||
# its budget silently again
|
||||
$firstStatusAt = $null
|
||||
$lastMining = $null
|
||||
$idleSince = $null
|
||||
$idleSamples = 0
|
||||
$lastIdleStatus = ''
|
||||
$lastEngineLine = ''
|
||||
function EngineLogDump([string] $why) {
|
||||
Write-Output ('===== engine log tail (' + $why + ')')
|
||||
$t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
||||
if ($t) { Get-Content -LiteralPath $t.FullName -ErrorAction SilentlyContinue | Where-Object { $_ -notmatch 'status: accepted 0 blocks' } | Select-Object -Last 80 | ForEach-Object { Write-Output (' ' + $_) } } else { Write-Output ' (no app-*.log in the scratch logs folder)' }
|
||||
if (Test-Path -LiteralPath $errFile) { Write-Output '===== engine stderr'; Get-Content -LiteralPath $errFile -ErrorAction SilentlyContinue | Select-Object -Last 20 | ForEach-Object { Write-Output (' ' + $_) } }
|
||||
}
|
||||
# the newest `status:` line among the engine log's last 60 lines (run 5, 15:39Z: a single sampled tail line missed the
|
||||
# status lines between TUNE and telemetry lines for 300 s and the watchdog killed a live three-card tune)
|
||||
function EngineStatus() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $ls = @(Get-Content -LiteralPath $t.FullName -Tail 60 -ErrorAction SilentlyContinue); for ($i = $ls.Count - 1; $i -ge 0; $i--) { if ([string]$ls[$i] -match ' status: ') { return [string]$ls[$i] } } }; return '' }
|
||||
function EngineTail() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $l = Get-Content -LiteralPath $t.FullName -Tail 1 -ErrorAction SilentlyContinue; if ($l) { return [string]$l } }; return '' }
|
||||
function Exit3([string] $why, [string] $reason) {
|
||||
Write-Output ('RESULT TUNE error=' + $reason + ' last_status_line=' + ((EngineStatus) -replace '\s+', '_') + ' last_log_line=' + ($lastEngineLine -replace '\s+', '_'))
|
||||
EngineLogDump $why
|
||||
EndTree $p.Id $why
|
||||
Snapshot 'after'
|
||||
Write-Output 'RESULT TUNE error=no_rows'
|
||||
exit 3
|
||||
}
|
||||
# the AMD watts source (coordinator, 6 October 2026 16:00Z): the 3.0 coordinator's job read ZERO watts for the 9070 XT
|
||||
# from igneum-gpu-telemetry.exe while the installed app had read 198.9 W the day before. The tune engine samples its
|
||||
# own helper; this script also samples the installed app's /api/state power_w for the AMD card every 10 s (a GET,
|
||||
# never a write) and, when the engine's AMD progress lines carry no watts, prints a fallback ladder from those
|
||||
# samples (median per step) and names the source it used
|
||||
$amdApp = @(); $amdStep = 0; $amdMhs = 0.0; $amdEngNZ = 0; $amdEngZ = 0; $tick = 0; $amdMhsByStep = @{}
|
||||
while (-not $p.HasExited) {
|
||||
Start-Sleep -Seconds 5
|
||||
$tick++
|
||||
if ($installedUrl -and ($tick % 2 -eq 0)) {
|
||||
try {
|
||||
$js = (Invoke-WebRequest -Uri ($installedUrl + 'api/state') -UseBasicParsing -TimeoutSec 3).Content | ConvertFrom-Json
|
||||
foreach ($c in @($js.mining.cards)) { if ($c.vendor -eq 'amd') { $amdApp += ,@{ step = $amdStep; w = [double]$c.power_w } } }
|
||||
} catch { }
|
||||
}
|
||||
$tailLine = EngineTail
|
||||
if ($tailLine) { $lastEngineLine = $tailLine }
|
||||
$statusLine = EngineStatus
|
||||
if ($statusLine) {
|
||||
if (-not $firstStatusAt) { $firstStatusAt = Get-Date }
|
||||
$rate = 0.0; if ($statusLine -match '(\d+\.\d+) MH/s') { $rate = [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) }
|
||||
# idle = three consecutive 10-s samples whose newest status line reads 0.00 MH/s (never a missing match)
|
||||
if ($rate -gt 0) { $lastMining = Get-Date; $idleSamples = 0; $idleSince = $null }
|
||||
elseif ($statusLine -ne $lastIdleStatus) { $lastIdleStatus = $statusLine; $idleSamples++; if (-not $idleSince) { $idleSince = Get-Date } }
|
||||
}
|
||||
if ($firstStatusAt -and -not $lastMining -and ((Get-Date) - $firstStatusAt).TotalSeconds -gt 120) { Exit3 'watchdog: not mining' 'not_mining reason=no_card_mined_within_120_s_of_the_first_status_line' }
|
||||
if ($lastMining -and $idleSamples -ge 3 -and $idleSince -and ((Get-Date) - $idleSince).TotalSeconds -gt 300) { Exit3 'watchdog: stopped mining' 'stopped_mining reason=three_status_lines_read_0.00_MH/s_and_300_s_passed' }
|
||||
$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) }
|
||||
while ($seen -lt $all.Count) {
|
||||
$l = [string]$all[$seen]; $seen++
|
||||
if ($l -match '^TUNE progress ') {
|
||||
Forward $l
|
||||
if ($l -match ' key=amd' -and $l -match ' step=(\d+)') { $amdStep = [int]$Matches[1]
|
||||
if ($l -match ' mhs=([\d.]+)') { $amdMhsByStep[$amdStep] = [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) }
|
||||
if ($l -match ' watts=([\d.]+)' -and [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) -gt 0) { $amdEngNZ++ } else { $amdEngZ++ } }
|
||||
}
|
||||
elseif ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ }; if ($l -match '^TUNE chosen ') { Forward $l } }
|
||||
elseif ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l) }
|
||||
elseif ($l -match '^(URL|STATE) ') { }
|
||||
else { Say $l }
|
||||
}
|
||||
if ((Get-Date) -gt $deadline) {
|
||||
Say ("budget of " + $budgetMinutes + " min spent; asking the tune engine to quit")
|
||||
# C35 (5 October 2026): the only quit this script may send goes to the TUNE engine's own URL file in the scratch
|
||||
# root, never to a file under the installed app's folder; the RESULT line names the file it used
|
||||
$u = Join-Path $sApp 'app.url'
|
||||
# the only URL file this script may quit is its own scratch root's; the installed app's folder is refused by name
|
||||
if ((Resolve-Path -LiteralPath $u -ErrorAction SilentlyContinue).Path -like (Join-Path $appDir '*') -or $u -like '*\igneum\app\*') {
|
||||
Write-Output ('RESULT TUNE quit refused: ' + $u + ' is the installed app''s URL file')
|
||||
} elseif (Test-Path -LiteralPath $u) {
|
||||
Write-Output ('RESULT TUNE quit asked of the tune engine through ' + $u + ' (pid ' + $p.Id + ')')
|
||||
try { Invoke-WebRequest -Uri ((Get-Content -LiteralPath $u -Raw).Trim() + 'api/quit') -Method POST -Body '{}' -ContentType 'application/json' -UseBasicParsing -TimeoutSec 5 | Out-Null } catch { }
|
||||
} else { Write-Output ('RESULT TUNE quit not sent: no URL file at ' + $u + '; killing pid ' + $p.Id) }
|
||||
Start-Sleep -Seconds 20
|
||||
if (-not $p.HasExited) { EndTree $p.Id 'budget' }
|
||||
EngineLogDump 'budget'
|
||||
Write-Output 'RESULT TUNE error=budget_exceeded'
|
||||
}
|
||||
}
|
||||
$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) }
|
||||
while ($seen -lt $all.Count) { $l = [string]$all[$seen]; $seen++; if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } }
|
||||
$amdNZ = @($amdApp | Where-Object { $_.w -gt 0 })
|
||||
Write-Output ('RESULT TUNE amd_watts engine_progress nonzero=' + $amdEngNZ + ' zero=' + $amdEngZ + ' app_state samples=' + $amdApp.Count + ' nonzero=' + $amdNZ.Count)
|
||||
$amdSource = 'none'
|
||||
if ($amdEngNZ -gt 0) { $amdSource = 'engine_telemetry' }
|
||||
elseif ($amdNZ.Count -gt 0) {
|
||||
$amdSource = 'app_state_fallback'
|
||||
foreach ($g in ($amdNZ | Group-Object { $_.step } | Sort-Object { [int]$_.Name })) {
|
||||
$ws = @($g.Group | ForEach-Object { $_.w } | Sort-Object); $med = $ws[[int][math]::Floor($ws.Count / 2)]
|
||||
$mh = 0.0; if ($amdMhsByStep.ContainsKey([int]$g.Name)) { $mh = $amdMhsByStep[[int]$g.Name] }
|
||||
$eff = 0.0; if ($med -gt 0) { $eff = $mh / $med }
|
||||
Write-Output ('RESULT TUNE amd_fallback source=app_state step=' + $g.Name + ' watts=' + ('{0:F1}' -f $med) + ' mhs=' + ('{0:F2}' -f $mh) + ' eff=' + ('{0:F4}' -f $eff) + ' samples=' + $ws.Count)
|
||||
}
|
||||
}
|
||||
Write-Output ('RESULT TUNE amd_watts_source=' + $amdSource)
|
||||
Say ("tune engine exited " + $p.ExitCode + " after " + [int]((Get-Date) - $started).TotalSeconds + " s, " + $rows + " table rows")
|
||||
EndTree $p.Id 'end of run'
|
||||
# the project lead, 6 October 2026, 07:25Z: both cards stay on their best MH/W points (the tune pins them); no factory reset here.
|
||||
Snapshot 'after'
|
||||
# the tune engine's own log: the TUNE lines and what happened around them
|
||||
$log = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
||||
if ($log) {
|
||||
Say ("engine log " + $log.FullName + ":")
|
||||
Get-Content -LiteralPath $log.FullName | Where-Object { $_ -match 'TUNE|tune|power cap|GPUs:|worker ready|STATUS|exited|upload' } | Select-Object -Last 100 | ForEach-Object { Say (' ' + $_) }
|
||||
}
|
||||
if ($rows -eq 0) { Write-Output 'RESULT TUNE error=no_rows'; exit 1 }
|
||||
exit 0
|
||||
|
|
@ -49,17 +49,10 @@ fi
|
|||
# PC 2 family probe and derive job; and the two hand-restore scripts pc1-amd-identities.ps1 and pc1-amd-flag-amd1.ps1,
|
||||
# allowed by the rule owner (6 October 2026, 19:xx UTC) because they post enabled TRUE only (a restore, never a switch off)
|
||||
PRE_RULE='^(relay/playbooks/(mixer-x4-pc1-bench|mixer-x4-5090-bench|mixer-x4-9070-bench|readwidth-5090-bench|readwidth-5090|readwidth-9070-bench|readwidth-9070|ca2-era-pc1|ca2-hot-5090-bench|ca2-hot-9070-bench|dot4-probe|ca3-derive-pc2)\.ps1|tools/ca3-reserve/pc2-family-probe\.ps1|tools/ca3-pc1-amd/pc1-amd-identities\.ps1|tools/ca3-pc1-amd/pc1-amd-flag-amd1\.ps1)$'
|
||||
# 6 October 2026 (0.3.14 gate): ember-tune-pc1.ps1 and the two agg-cost scripts are allowed pending the agg-cost --stop-miners change
|
||||
# and the Ember playbook's rewrite (the coordinator's 15:30Z ruling); EXPIRES 0.3.15 (the check below fails the tree at 0.3.15 or later while they stand)
|
||||
ALLOW='^(packaging/windows/stop-igneum\.ps1|relay/playbooks/ember-tune-pc1\.ps1|tools/proving-v1/pc2-agg-cost\.ps1|tools/proving-v1/pc2-agg-cost-restore\.ps1)$'
|
||||
# 6 October 2026 (0.3.15): the dated allow entries are gone (ember-tune-pc1.ps1 deleted, the agg-cost scripts on the
|
||||
# runner's --cards-off); only the installer's own stop step remains.
|
||||
ALLOW='^(packaging/windows/stop-igneum\.ps1)$'
|
||||
fail=0
|
||||
while IFS= read -r f; do [[ "$f" =~ $ALLOW ]] && continue; [[ "$f" =~ $PRE_RULE ]] && continue; check_file "$f" || fail=1; done < <(git ls-files 'relay/playbooks/**' 'tools/**' 'packaging/**' | grep -E '\.(ps1|sh)$' | grep -v '^tools/ci/')
|
||||
# expiry of the dated allow entries (6 October 2026): at 0.3.15 or later the three must be gone from ALLOW
|
||||
ver="$(sed -n 's/^version = "\(.*\)"/\1/p' app/igneum-app/Cargo.toml 2>/dev/null | head -1)"
|
||||
if [ -n "$ver" ] && [ "$(printf '%s\n0.3.15\n' "$ver" | sort -V | head -1)" = "0.3.15" ]; then
|
||||
for e in relay/playbooks/ember-tune-pc1.ps1 tools/proving-v1/pc2-agg-cost.ps1 tools/proving-v1/pc2-agg-cost-restore.ps1; do
|
||||
case "$ALLOW" in *"${e//./\\.}"*) echo "playbook-quit: the allow entry for $e expired at 0.3.15 (the tree says $ver): the pause/resume rule applies to it now"; fail=1 ;; esac
|
||||
done
|
||||
fi
|
||||
[ "$fail" = 0 ] && echo "playbook-quit: no playbook quits, pauses, resumes or switches the cards of the installed app"
|
||||
exit $fail
|
||||
|
|
|
|||
|
|
@ -40,17 +40,9 @@ foreach ($p in (Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'ig
|
|||
}
|
||||
"RESULT strays_stopped $(Stamp) $killed"
|
||||
& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
||||
function Card($st) { if ($null -eq $st) { return $null }; $st.mining.cards | Where-Object { $_.vendor -eq 'nvidia' } | Select-Object -First 1 }
|
||||
$c = Card (State)
|
||||
if ($c) {
|
||||
$stc = State
|
||||
"RESULT cards_before $(Stamp) $(($stc.mining.cards | ForEach-Object { "$($_.name):enabled=$($_.enabled):identities=$($_.identities):$($_.state)" }) -join ',')"
|
||||
$list = @(); foreach ($k in $stc.mining.cards) { $list += @{ key = $k.key; enabled = $(if ($k.key -eq $c.key) { $true } else { [bool]$k.enabled }); identities = [int]$k.identities } }
|
||||
"RESULT card_on $(Stamp) $(Post '/api/cards' @{ cards = $list })"
|
||||
}
|
||||
# 6 October 2026 (0.3.15): no card switch, resume or start from a script (CLAUDE.md job rule); the runner of the measurement
|
||||
# job restores the card and the miners on any exit, so this script only cleans up strays and reads the state back.
|
||||
"RESULT timeslice $(Stamp) $((& nvidia-smi compute-policy --set-timeslice=0 2>&1) -join ' ')"
|
||||
"RESULT resume $(Stamp) $(Post '/api/resume' @{})"
|
||||
"RESULT start $(Stamp) $(Post '/api/start' @{})"
|
||||
"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})"
|
||||
Start-Sleep -Seconds 20
|
||||
$st = State
|
||||
|
|
|
|||
|
|
@ -92,12 +92,7 @@ function CardPrefs {
|
|||
return $prefs
|
||||
}
|
||||
function NvidiaKey { $p = CardPrefs; ($p.Keys | Where-Object { $_ -like 'nvidia:*' } | Select-Object -First 1) }
|
||||
function CardSwitch($on, $identities) {
|
||||
$p = CardPrefs; $nk = NvidiaKey
|
||||
if (-not $nk) { return "no nvidia card key in settings.json (keys: $($p.Keys -join ','))" }
|
||||
$list = @(); foreach ($k in $p.Keys) { $list += @{ key = $k; enabled = $(if ($k -eq $nk) { [bool]$on } else { [bool]$p[$k].enabled }); identities = $(if ($k -eq $nk -and $identities) { [int]$identities } else { [int]$p[$k].identities }) } }
|
||||
"prefs_before=$(($p.Keys | ForEach-Object { "$($_):enabled=$($p[$_].enabled):identities=$($p[$_].identities)" }) -join ',') " + (Post '/api/cards' @{ cards = $list })
|
||||
}
|
||||
# CardSwitch removed 6 October 2026: the runner's --cards-off owns the card (see the needIdle block)
|
||||
function CudaWorkers { (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' } | Measure-Object).Count }
|
||||
$job = $env:IGNEUM_JOB_DIR
|
||||
if (-not $job) { $job = Join-Path $env:TEMP 'igneum-agg-cost' }
|
||||
|
|
@ -346,12 +341,15 @@ try {
|
|||
if ($PhaseSet -contains 'B') { RunPhase 'B' "SP1_WORKER_VERIFY_INTERMEDIATES=false SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B.json" }
|
||||
$needIdle = @('C','C1','C2','G0','D','E','E0','F','G','H') | Where-Object { $PhaseSet -contains $_ }
|
||||
if ($needIdle) {
|
||||
# the 5090 switched off through /api/cards (and on again in the finally block): /api/pause then /api/resume left
|
||||
# the worker off on 0.3.9 (21:25Z, an app defect in the resume path, fixed for 0.3.11)
|
||||
"RESULT card_off $(Stamp) $(CardSwitch $false $null)"; $paused = $true
|
||||
$w = 0; $left = 1
|
||||
# 6 October 2026 (0.3.15): the script never switches a card or pauses the installed app (CLAUDE.md job rule, the
|
||||
# playbook-quit check). The job is published with --cards-off <nvidia key>: the RUNNER switches the 5090 off through
|
||||
# the app's own card path before the script and puts it back on any exit. Here the script only confirms the worker
|
||||
# is gone, and fails loud if it is not (no pause fallback: a script that dies before its finally left the box
|
||||
# paused unattended).
|
||||
$w = 0; $left = CudaWorkers
|
||||
while ($w -lt 120 -and $left -gt 0) { Start-Sleep -Seconds 5; $w += 5; $left = CudaWorkers }
|
||||
if ($left -gt 0) { "RESULT card_off_failed $(Stamp) the app's CUDA worker is still running after $w s ($left); /api/pause as the fallback: $(Post '/api/pause' @{})"; $w2 = 0; while ($w2 -lt 60 -and (CudaWorkers) -gt 0) { Start-Sleep -Seconds 5; $w2 += 5 }; $left = CudaWorkers }
|
||||
if ($left -gt 0) { "RESULT card_off_failed $(Stamp) the app's CUDA worker is still running after $w s ($left): publish this job with --cards-off <nvidia card key> (the runner switches it off); nothing measured"; exit 3 }
|
||||
$paused = $true
|
||||
"RESULT paused $(Stamp) after $w s; CUDA workers left: $left; $(Smi)"
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
|
|
@ -423,11 +421,7 @@ try {
|
|||
} finally {
|
||||
StopMiner 'final'
|
||||
& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
||||
if ($paused) {
|
||||
"RESULT card_on $(Stamp) $(CardSwitch $true $null)"; "RESULT resume $(Stamp) $(Post '/api/resume' @{})"
|
||||
$w3 = 0; while ($w3 -lt 90 -and (CudaWorkers) -eq 0) { Start-Sleep -Seconds 5; $w3 += 5 }
|
||||
if ((CudaWorkers) -eq 0) { "RESULT miner_back_failed $(Stamp) no CUDA worker after $w3 s; /api/start: $(Post '/api/start' @{})" } else { "RESULT miner_back $(Stamp) CUDA worker running after $w3 s" }
|
||||
}
|
||||
if ($paused) { "RESULT card_restore $(Stamp) left to the runner (--cards-off puts the card back on any exit; the miners follow)" }
|
||||
"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})"
|
||||
"RESULT end $(Stamp) $(Smi)"
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue