204 lines
16 KiB
PowerShell
204 lines
16 KiB
PowerShell
# Igneum run job: Ember Tune end to end on PC 1 (machine ae432dc7), unattended. 5 October 2026.
|
|
# Published as a `run` job with --stop-miners (docs/plans/ember-tune.md): the installed app stops its miners and
|
|
# holds them; this script takes the engine that carries src/ember.rs (the one the fetch job put in
|
|
# <app dir>\jobs\ember-kit-1\igneum-app-ember.exe, else the installed one; the AMD helper with the --tune and --set
|
|
# commands from the telemetry agent's fetch job, <app dir>\jobs\amd-kit-1\kit\igneum-gpu-telemetry.exe), copies the install folder to a scratch
|
|
# folder beside it, swaps the engine in, and starts that SECOND engine with `--sweep` in a scratch data folder (the
|
|
# real settings.json, machine-id and wallet.json copied in; remote jobs, auto-update and proving switched off
|
|
# there). That engine finds the installed app's node on 127.0.0.1:26610, mines on every card with the live program,
|
|
# tunes them one after the other (the full two-knob plan where the card can be controlled, the baseline measurement
|
|
# where it cannot: NVIDIA without administrator rights, Apple), prints every TUNE line on stdout, uploads its log
|
|
# (the TUNE {json} record reaches the intake) and quits. Every TUNE line is re-emitted as a RESULT line, so
|
|
# `node tools/jobs.mjs <job id>` shows the table. Before and after, nvidia-smi's limits and clocks and the AMD
|
|
# helper's `--tune` lines are printed, so the restore can be read. The installed app's miners restart when the job
|
|
# ends. Not elevated: nothing asks for administrator rights (the project lead asleep, 5 October 2026); the NVIDIA card is
|
|
# therefore measure only tonight unless the engine finds itself elevated.
|
|
$ErrorActionPreference = 'Continue'
|
|
$resultTag = 'TUNE'
|
|
$budgetMinutes = 45
|
|
if (-not ($budgetMinutes -is [int]) -or $budgetMinutes -lt 5) { $budgetMinutes = 35 } # a budget under 5 minutes is a bug, not a budget (C35)
|
|
$started = Get-Date
|
|
$deadline = $started.AddMinutes($budgetMinutes)
|
|
function Say([string] $m) { Write-Host ("[" + (Get-Date -Format 'HH:mm:ss') + "] " + $m) }
|
|
|
|
# the installed engine: the per-user install (0.3.3+), else Program Files
|
|
$installDir = $null
|
|
foreach ($d in @((Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner'), (Join-Path $env:ProgramFiles 'Igneum Miner'))) {
|
|
if (Test-Path (Join-Path $d 'igneum-app.exe')) { $installDir = $d; break }
|
|
}
|
|
if (-not $installDir) { Write-Output 'RESULT TUNE error=no_engine reason=igneum-app.exe_not_found'; exit 2 }
|
|
$appData = $env:IGNEUM_APP_DATA
|
|
if (-not $appData) { $appData = Join-Path $env:LOCALAPPDATA 'igneum' }
|
|
$appDir = $env:IGNEUM_APP_DIR
|
|
if (-not $appDir) { $appDir = Join-Path $appData 'app' }
|
|
|
|
# the scratch install: the whole folder (workers, node, helper, DLLs) with the Ember engine swapped in
|
|
# a FRESH scratch root per run (run 3, 6 October 2026, 11:45Z: the folder an earlier elevated engine had locked to itself
|
|
# refused the settings copy, the engine started on stale files and exited in 6 s); old roots are small and left alone
|
|
$root = Join-Path $env:LOCALAPPDATA ('igneum-tune-' + (Get-Date -Format 'yyyyMMdd-HHmmss'))
|
|
$bin = Join-Path $root 'bin'
|
|
$sApp = Join-Path $root 'app'
|
|
$sLogs = Join-Path $root 'logs'
|
|
New-Item -ItemType Directory -Force -Path $root, $sApp, $sLogs | Out-Null
|
|
if (Test-Path $bin) { Remove-Item -LiteralPath $bin -Recurse -Force -ErrorAction SilentlyContinue }
|
|
Copy-Item -LiteralPath $installDir -Destination $bin -Recurse -Force
|
|
# the installed engine carries Ember Tune from 0.3.12 on: prefer it; the kit is for a PC still on an older app
|
|
$installedVer = (& (Join-Path $installDir 'igneum-app.exe') --version 2>&1 | Out-String).Trim()
|
|
$installedHasEmber = $false
|
|
if ($installedVer -match 'igneum-app (\d+)\.(\d+)\.(\d+)') { $installedHasEmber = ([int]$Matches[1] -gt 0) -or ([int]$Matches[2] -gt 3) -or (([int]$Matches[2] -eq 3) -and ([int]$Matches[3] -ge 12)) }
|
|
$ember = $null
|
|
# ember-kit-3 (the engine with Settings::for_measurement) is preferred when present, whatever the installed version;
|
|
# older kits only when the installed app predates Ember Tune
|
|
$k3 = Join-Path $appDir 'jobs\ember-kit-3\igneum-app-ember.exe'
|
|
if (Test-Path $k3) { $ember = $k3 }
|
|
elseif (-not $installedHasEmber) { foreach ($cand in @((Join-Path $appDir 'jobs\ember-kit-2\igneum-app-ember.exe'), (Join-Path $appDir 'jobs\ember-kit-1\igneum-app-ember.exe'))) { if (Test-Path $cand) { $ember = $cand; break } } }
|
|
if ($ember) {
|
|
Copy-Item -LiteralPath $ember -Destination (Join-Path $bin 'igneum-app.exe') -Force
|
|
Say ("engine: the Ember build from " + $ember)
|
|
} else {
|
|
Say ('engine: the installed one (' + $installedVer + $(if ($installedHasEmber) { ', carries Ember Tune' } else { '; no kit found: an older engine ignores the tune and reports no_rows' }) + ')')
|
|
}
|
|
$helper = $null
|
|
$found = Get-ChildItem -Path (Join-Path $appDir 'jobs') -Recurse -Filter 'igneum-gpu-telemetry.exe' -ErrorAction SilentlyContinue | Where-Object { $_.FullName -match 'amd-kit' } | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
|
if ($found) { $helper = $found.FullName }
|
|
if ($helper) {
|
|
Copy-Item -LiteralPath $helper -Destination (Join-Path $bin 'igneum-gpu-telemetry.exe') -Force
|
|
Say ("helper: the Ember build of igneum-gpu-telemetry from " + $helper + " sha256=" + (Get-FileHash -LiteralPath $helper -Algorithm SHA256).Hash.ToLower())
|
|
} else { Say 'helper: the installed igneum-gpu-telemetry (no jobs\amd-kit-1\kit\igneum-gpu-telemetry.exe); without --tune the AMD card measures only' }
|
|
$exe = Join-Path $bin 'igneum-app.exe'
|
|
$ver = (& $exe --version 2>&1 | Out-String).Trim()
|
|
Say ("engine: " + $exe + " (" + $ver + ")")
|
|
Write-Output ("RESULT TUNE engine " + $ver + " sha256=" + (Get-FileHash -LiteralPath $exe -Algorithm SHA256).Hash.ToLower())
|
|
if ($ver -notmatch 'igneum-app (\d+)\.(\d+)\.(\d+)') { Write-Output 'RESULT TUNE error=version_unknown'; exit 2 }
|
|
|
|
foreach ($f in @('settings.json', 'machine-id', 'wallet.json', 'tuning.json')) {
|
|
$src = Join-Path $appDir $f
|
|
if (Test-Path $src) { Copy-Item -LiteralPath $src -Destination (Join-Path $sApp $f) -Force }
|
|
}
|
|
# the copies are VERBATIM (run 4, 6 October 2026: a PowerShell ConvertFrom-Json | ConvertTo-Json round trip rewrote big
|
|
# integers as doubles, the engine read the file as defaults, no payout address, every card off, 96 old jobs run in
|
|
# the scratch root); the engine itself switches remote jobs, updates, proving and Power control off under --sweep
|
|
# (Settings::for_measurement) and makes every card due. Only a report line is read here.
|
|
$sj = Join-Path $sApp 'settings.json'
|
|
if (-not (Test-Path -LiteralPath $sj)) { Write-Output 'RESULT TUNE error=no_settings reason=the_installed_app_has_no_settings.json'; exit 2 }
|
|
$installedPowerControl = 'unknown'; $addr = ''; $ncards = 0
|
|
try { $back = Get-Content -LiteralPath $sj -Raw | ConvertFrom-Json; $addr = [string]$back.address; if ($back.cards) { $ncards = @($back.cards.PSObject.Properties).Count }; if ($back.PSObject.Properties.Name -contains 'power_control') { $installedPowerControl = ([bool]$back.power_control).ToString().ToLower() } } catch { }
|
|
$bom = (Get-Content -LiteralPath $sj -Encoding Byte -TotalCount 3 -ErrorAction SilentlyContinue) -join ','
|
|
Write-Output ('RESULT TUNE installed_power_control=' + $installedPowerControl + ' (the tune engine runs with it off: no prompt unless the job itself is elevated)')
|
|
Write-Output ('RESULT TUNE scratch settings (verbatim copy): address ' + $(if ($addr) { $addr.Substring(0, [Math]::Min(10, $addr.Length)) + '...' } else { 'EMPTY' }) + ', cards ' + $ncards + ', first bytes ' + $bom + ', ' + (Get-Item -LiteralPath $sj).Length + ' bytes')
|
|
if (-not $addr -and -not (Test-Path (Join-Path $sApp 'wallet.json'))) { Write-Output 'RESULT TUNE error=no_address reason=the_copied_settings_carry_no_payout_address_and_no_wallet.json'; exit 2 }
|
|
Remove-Item -LiteralPath (Join-Path $sApp 'app.url') -Force -ErrorAction SilentlyContinue
|
|
|
|
# the state before, for the report
|
|
$smi = Join-Path $env:ProgramFiles 'NVIDIA Corporation\NVSMI\nvidia-smi.exe'
|
|
if (-not (Test-Path $smi)) { $smi = Join-Path $env:SystemRoot 'System32\nvidia-smi.exe' }
|
|
$tele = Join-Path $bin 'igneum-gpu-telemetry.exe'
|
|
function Snapshot([string] $tag) {
|
|
if (Test-Path $smi) {
|
|
$q = (& $smi --query-gpu=index,name,driver_version,power.draw,power.limit,power.default_limit,power.min_limit,power.max_limit,clocks.gr,clocks.max.gr,clocks.mem --format=csv,noheader 2>&1 | Out-String).Trim()
|
|
Write-Output ("RESULT TUNE " + $tag + " nvidia " + ($q -replace "`r?`n", ' | '))
|
|
}
|
|
if (Test-Path $tele) {
|
|
$t = (& $tele --tune 2>&1 | Out-String).Trim()
|
|
Write-Output ("RESULT TUNE " + $tag + " amd " + ($t -replace "`r?`n", ' | '))
|
|
} else { Write-Output ("RESULT TUNE " + $tag + " amd no_helper") }
|
|
}
|
|
Snapshot 'before'
|
|
|
|
# the tune engine: status every 10 s (6 rate samples per 60 s hold)
|
|
$env:IGNEUM_APP_DATA = $root
|
|
$env:IGNEUM_APP_LOGS = $sLogs
|
|
$env:IGNEUM_APP_STATUS_SECS = '10'
|
|
$env:IGNEUM_APP_NO_OTA = '1' # C35: a second engine never runs the updater (the installer would quit the installed app)
|
|
# C35 (5 October 2026): the engine's output goes to a FILE, never a pipe. A pipe's write end is inherited by every
|
|
# process the engine starts (its miners and workers), so after an abort the installed app's jobs runner waits for an
|
|
# EOF that never comes and hangs in its own quit; and the engine's whole tree is killed at the end (nothing orphaned).
|
|
$outFile = Join-Path $root 'engine-stdout.log'
|
|
$errFile = Join-Path $root 'engine-stderr.log'
|
|
Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue
|
|
$p = Start-Process -FilePath $exe -ArgumentList '--sweep' -WorkingDirectory (Split-Path $exe) -WindowStyle Hidden -PassThru -RedirectStandardOutput $outFile -RedirectStandardError $errFile
|
|
Say ("engine started, pid " + $p.Id + ", data " + $root + ", stdout " + $outFile)
|
|
function EndTree([int] $procId, [string] $why) {
|
|
$before = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count
|
|
& taskkill /T /F /PID $procId 2>&1 | Out-Null
|
|
Start-Sleep -Seconds 2
|
|
$after = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count
|
|
Write-Output ('RESULT ' + $resultTag + ' tree ended (' + $why + '): igneum processes ' + $before + ' -> ' + $after + ' (the installed app''s own miners are stopped and held by the job)')
|
|
}
|
|
$seen = 0
|
|
$rows = 0
|
|
# the watchdog (coordinator, 6 October 2026): a tune engine that mines nothing for 120 s after its first status line
|
|
# (every card "waiting" or 0.00 MH/s: no payout address, no worker, no node) fails the job at once with the engine's
|
|
# last log line in the RESULT, its tree ended, mining restored by the job runner; a job that cannot mine never burns
|
|
# its budget silently again
|
|
$firstStatusAt = $null
|
|
$lastMining = $null
|
|
$lastEngineLine = ''
|
|
function EngineLogDump([string] $why) {
|
|
Write-Output ('===== engine log tail (' + $why + ')')
|
|
$t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
|
if ($t) { Get-Content -LiteralPath $t.FullName -ErrorAction SilentlyContinue | Where-Object { $_ -notmatch 'status: accepted 0 blocks' } | Select-Object -Last 80 | ForEach-Object { Write-Output (' ' + $_) } } else { Write-Output ' (no app-*.log in the scratch logs folder)' }
|
|
if (Test-Path -LiteralPath $errFile) { Write-Output '===== engine stderr'; Get-Content -LiteralPath $errFile -ErrorAction SilentlyContinue | Select-Object -Last 20 | ForEach-Object { Write-Output (' ' + $_) } }
|
|
}
|
|
function EngineTail() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $l = Get-Content -LiteralPath $t.FullName -Tail 1 -ErrorAction SilentlyContinue; if ($l) { return [string]$l } }; return '' }
|
|
while (-not $p.HasExited) {
|
|
Start-Sleep -Seconds 5
|
|
$tailLine = EngineTail
|
|
if ($tailLine) { $lastEngineLine = $tailLine }
|
|
if ($lastEngineLine -match ' status: ') {
|
|
if (-not $firstStatusAt) { $firstStatusAt = Get-Date }
|
|
if ($lastEngineLine -match ', mining \|' -or ($lastEngineLine -match '(\d+\.\d+) MH/s' -and [double]$Matches[1] -gt 0)) { $lastMining = Get-Date }
|
|
}
|
|
if ($firstStatusAt -and -not $lastMining -and ((Get-Date) - $firstStatusAt).TotalSeconds -gt 120) {
|
|
Write-Output ('RESULT TUNE error=not_mining reason=no_card_mined_within_120_s_of_the_first_status_line last_log_line=' + ($lastEngineLine -replace '\s+', '_'))
|
|
EngineLogDump 'watchdog: not mining'
|
|
EndTree $p.Id 'watchdog: not mining'
|
|
Write-Output 'RESULT TUNE error=no_rows'
|
|
exit 3
|
|
}
|
|
if ($lastMining -and ((Get-Date) - $lastMining).TotalSeconds -gt 300) {
|
|
Write-Output ('RESULT TUNE error=stopped_mining reason=every_card_idle_for_300_s last_log_line=' + ($lastEngineLine -replace '\s+', '_'))
|
|
EngineLogDump 'watchdog: stopped mining'
|
|
EndTree $p.Id 'watchdog: stopped mining'
|
|
Write-Output 'RESULT TUNE error=no_rows'
|
|
exit 3
|
|
}
|
|
$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) }
|
|
while ($seen -lt $all.Count) {
|
|
$l = [string]$all[$seen]; $seen++
|
|
if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } }
|
|
elseif ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l) }
|
|
elseif ($l -match '^(URL|STATE) ') { }
|
|
else { Say $l }
|
|
}
|
|
if ((Get-Date) -gt $deadline) {
|
|
Say ("budget of " + $budgetMinutes + " min spent; asking the tune engine to quit")
|
|
# C35 (5 October 2026): the only quit this script may send goes to the TUNE engine's own URL file in the scratch
|
|
# root, never to a file under the installed app's folder; the RESULT line names the file it used
|
|
$u = Join-Path $sApp 'app.url'
|
|
# the only URL file this script may quit is its own scratch root's; the installed app's folder is refused by name
|
|
if ((Resolve-Path -LiteralPath $u -ErrorAction SilentlyContinue).Path -like (Join-Path $appDir '*') -or $u -like '*\igneum\app\*') {
|
|
Write-Output ('RESULT TUNE quit refused: ' + $u + ' is the installed app''s URL file')
|
|
} elseif (Test-Path -LiteralPath $u) {
|
|
Write-Output ('RESULT TUNE quit asked of the tune engine through ' + $u + ' (pid ' + $p.Id + ')')
|
|
try { Invoke-WebRequest -Uri ((Get-Content -LiteralPath $u -Raw).Trim() + 'api/quit') -Method POST -Body '{}' -ContentType 'application/json' -UseBasicParsing -TimeoutSec 5 | Out-Null } catch { }
|
|
} else { Write-Output ('RESULT TUNE quit not sent: no URL file at ' + $u + '; killing pid ' + $p.Id) }
|
|
Start-Sleep -Seconds 20
|
|
if (-not $p.HasExited) { EndTree $p.Id 'budget' }
|
|
Write-Output 'RESULT TUNE error=budget_exceeded'
|
|
}
|
|
}
|
|
$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) }
|
|
while ($seen -lt $all.Count) { $l = [string]$all[$seen]; $seen++; if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } }
|
|
Say ("tune engine exited " + $p.ExitCode + " after " + [int]((Get-Date) - $started).TotalSeconds + " s, " + $rows + " table rows")
|
|
EndTree $p.Id 'end of run'
|
|
# the project lead, 6 October 2026, 07:25Z: both cards stay on their best MH/W points (the tune pins them); no factory reset here.
|
|
Snapshot 'after'
|
|
# the tune engine's own log: the TUNE lines and what happened around them
|
|
$log = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1
|
|
if ($log) {
|
|
Say ("engine log " + $log.FullName + ":")
|
|
Get-Content -LiteralPath $log.FullName | Where-Object { $_ -match 'TUNE|tune|power cap|GPUs:|worker ready|STATUS|exited|upload' } | Select-Object -Last 100 | ForEach-Object { Say (' ' + $_) }
|
|
}
|
|
if ($rows -eq 0) { Write-Output 'RESULT TUNE error=no_rows'; exit 1 }
|
|
exit 0
|