PC 1 queue: the second Power Helper diagnostic (task exe against the running app, scheduler history, crash log, a 20 s unelevated probe) and the counter-asic-4 micro-benchmark job (the research lane's exe, unlocked and at the 1,300 knee, rows labelled ca4-research), with their publish lines

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-07 21:33:00 +00:00
parent 36d777afea
commit 0e689e5798
3 changed files with 131 additions and 0 deletions

View file

@ -0,0 +1,79 @@
# Counter ASIC 4 research (7 October 2026): the per-block micro-benchmark of the research lane (ad6a2bd47d4a46105) on PC 1's RTX 5090,
# the card alone through the RUNNER's --cards-off, no prompt. The exe is the research lane's scratch worker with --microbench
# (igneum-worker-cuda-ca4mb.exe, sha256 49aa60c4b28ed77a54655476d0f07c06a54aa22e4f6f6ccadd4019b84cbf0268; its own fetch job
# fetch-ca4-microbench-exe-20261007). Twice: at the unlocked clock, and at the 1,300 MHz knee lock when the Power Helper task
# answers (the lock path of pc1-v4-efficiency.ps1: the task started by the owning user, `<seq> dev <idx>` + `<seq> lgc 1300` through
# cmd.txt, the base sequence from cmd.txt and helper.log, rgc in finally; a silent helper skips the locked state and says so). Per
# state: `--microbench --mb-seconds 60` (20 probes, about 22 min) with nvidia-smi at 1 Hz; every `RESULT microbench probe=...` line
# of the worker is kept verbatim and a `RESULT ca4-research probe=<name> state=<unlocked|lock1300> watts=<mean> watts_instant= watts_average=
# sm_mhz= mem_mhz= G_ops_s= checksum= status=` row is derived from the probe's start_utc..end_utc window (the mean from 8 s in).
# Compile failures and skips print as the worker prints them. Never quits, pauses, resumes or updates the installed app.
$ErrorActionPreference = 'Continue'
$jobName = 'pc1-ca4-microbench'
$started = Get-Date
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
function Summary([string] $status, [hashtable] $extra) { $o = [ordered]@{ job = $jobName; status = $status; duration_s = [int]((Get-Date) - $started).TotalSeconds; finished_at = (Stamp) }; foreach ($k in $extra.Keys) { $o[$k] = $extra[$k] }; 'SUMMARY ' + ($o | ConvertTo-Json -Compress -Depth 4) }
$jobDir = $env:IGNEUM_JOB_DIR; if (-not $jobDir) { $jobDir = Join-Path $env:TEMP 'igneum-ca4-mb' }
New-Item -ItemType Directory -Force -Path $jobDir | Out-Null
$jobs = Split-Path $jobDir
$exe = Join-Path (Join-Path $jobs 'fetch-ca4-microbench-exe-20261007') 'igneum-worker-cuda-ca4mb.exe'
if (-not (Test-Path $exe)) { "RESULT error exe missing at $exe (the fetch job fetch-ca4-microbench-exe-20261007 runs first)"; Summary 'failed' @{ error = 'exe missing' }; exit 2 }
"RESULT start $(Stamp) job=$jobName exe=$exe sha256=$((Get-FileHash -Algorithm SHA256 $exe).Hash.ToLower()) label=ca4-research"
$smi = (Get-Command nvidia-smi -ErrorAction SilentlyContinue).Source
if (-not $smi) { foreach ($p in @((Join-Path $env:ProgramFiles 'NVIDIA Corporation\NVSMI\nvidia-smi.exe'), (Join-Path $env:SystemRoot 'System32\nvidia-smi.exe'))) { if (Test-Path $p) { $smi = $p; break } } }
$row = @(& $smi --query-gpu=index,name,uuid,pci.bus_id --format=csv,noheader 2>&1 | ForEach-Object { "$_" }) | Where-Object { $_ -match 'RTX 5090' } | Select-Object -First 1
if (-not $row) { 'RESULT error no RTX 5090'; Summary 'failed' @{ error = 'no 5090' }; exit 2 }
$fld = $row -split ',\s*'; $idx = $fld[0].Trim(); $uuid = $fld[2].Trim(); $bus = $fld[3].Trim()
"RESULT card index=$idx uuid=$uuid bus=$bus"
$apps = @(& $smi --query-compute-apps=gpu_bus_id,pid,process_name --format=csv,noheader 2>$null | ForEach-Object { "$_" } | Where-Object { $_ -match [regex]::Escape($bus) -and $_ -match 'igneum' })
$t = 0; while ($t -lt 150 -and $apps.Count -gt 0) { Start-Sleep -Seconds 5; $t += 5; $apps = @(& $smi --query-compute-apps=gpu_bus_id,pid,process_name --format=csv,noheader 2>$null | ForEach-Object { "$_" } | Where-Object { $_ -match [regex]::Escape($bus) -and $_ -match 'igneum' }) }
"RESULT card $(Stamp) $(if ($apps.Count -eq 0) { "quiet after $t s (the runner's --cards-off took)" } else { "UNCONFIRMED: $($apps -join '; ')" })"
# ---- the helper (the lock path), as pc1-v4-efficiency.ps1 ----
$helperDir = Join-Path $env:LOCALAPPDATA 'igneum\app\sweep'; $cmdFile = Join-Path $helperDir 'cmd.txt'; $helperLog = Join-Path $helperDir 'helper.log'
function HelperSeqBase() { $m = 0; foreach ($f in @($cmdFile, $helperLog)) { if (Test-Path $f) { foreach ($l in (Get-Content -LiteralPath $f -ErrorAction SilentlyContinue)) { if ($l -match '(?:^|\s)(\d{1,6}) (?:dev|pl|lgc|rgc|lmc|rmc|nvidia-smi|reregister)\b') { $v = [int]$Matches[1]; if ($v -gt $m) { $m = $v } } } } }; return $m + 100 }
function HelperSend([string] $verb) {
$s0 = HelperSeqBase; $s1 = $s0 + 1
$mark = 0; if (Test-Path $helperLog) { $mark = @(Get-Content -LiteralPath $helperLog -ErrorAction SilentlyContinue).Count }
Set-Content -LiteralPath $cmdFile -Value "$s0 dev $idx`n$s1 $verb`n" -NoNewline -Encoding ascii
$t = 0; while ($t -lt 30) { Start-Sleep -Seconds 1; $t++; $new = @(Get-Content -LiteralPath $helperLog -ErrorAction SilentlyContinue); if ($new.Count -gt $mark) { $hit = $new[$mark..($new.Count - 1)] | Where-Object { $_ -match "\b$s1 nvidia-smi " } | Select-Object -First 1; if ($hit) { return ($hit -replace '^\d+\s+', '') } } }
return "no helper.log line for seq $s1 within 30 s"
}
$viaHelper = $false
if (Get-ScheduledTask -TaskName 'Igneum Power Helper' -ErrorAction SilentlyContinue) { try { Start-ScheduledTask -TaskName 'Igneum Power Helper' -ErrorAction Stop; Start-Sleep -Seconds 3; $viaHelper = $true; 'RESULT helper task started' } catch { "RESULT helper start failed: $_" } } else { 'RESULT helper task not registered: the locked state is skipped' }
# ---- the sampler ----
$samples = Join-Path $jobDir 'nv-samples.csv'; if (Test-Path $samples) { Remove-Item -LiteralPath $samples -Force }
$sampler = Start-Process -FilePath $smi -ArgumentList @('-i', $idx, '--query-gpu=timestamp,power.draw,clocks.sm,clocks.mem,temperature.gpu,utilization.gpu,power.draw.instant,power.draw.average', '--format=csv,noheader,nounits', '-lms', '1000') -RedirectStandardOutput $samples -WindowStyle Hidden -PassThru
Start-Sleep -Seconds 5
"RESULT sampler pid=$($sampler.Id) first=[$((Get-Content -LiteralPath $samples -ErrorAction SilentlyContinue | Select-Object -First 1))]"
function Mean($vals) { $v = @($vals | ForEach-Object { [double]$_ }); if ($v.Count -eq 0) { return $null }; return [math]::Round((($v | Measure-Object -Sum).Sum / $v.Count), 1) }
function Window([datetime] $t0, [datetime] $t1) { $sel = @(); if (Test-Path $samples) { foreach ($l in (Get-Content -LiteralPath $samples -ErrorAction SilentlyContinue)) { $p = $l -split ',\s*'; if ($p.Count -lt 6) { continue }; try { $ts = [datetime]::ParseExact($p[0].Trim(), 'yyyy/MM/dd HH:mm:ss.fff', [cultureinfo]::InvariantCulture) } catch { continue }; if ($ts -ge $t0 -and $ts -le $t1) { $sel += ,$p } } }; return ,$sel }
function ParseUtc([string] $s) { try { return [datetime]::Parse($s, [cultureinfo]::InvariantCulture, [System.Globalization.DateTimeStyles]::AdjustToUniversal).ToLocalTime() } catch { return $null } }
function RunState([string] $state) {
$env:CUDA_VISIBLE_DEVICES = $uuid
"RESULT microbench-state $state start $(Stamp) $((& $smi -i $idx --query-gpu=clocks.sm,clocks.applications.graphics,power.draw --format=csv,noheader 2>&1 | Out-String).Trim())"
$lines = @(& $exe --microbench --mb-seconds 60 --device 0 2>&1 | ForEach-Object { "$_" })
$rc = $LASTEXITCODE
foreach ($l in $lines) { if ($l -match '^RESULT microbench') { "$l state=$state" } elseif ($l -match 'error|fail|compile|skipped') { "RESULT microbench-note state=$state $l" } }
foreach ($l in ($lines | Where-Object { $_ -match '^RESULT microbench probe=' })) {
$name = ''; $g = ''; $ck = ''; $st = ''; $su = ''; $eu = ''
if ($l -match 'probe=(\S+)') { $name = $Matches[1] }; if ($l -match 'G_ops_s=(\S+)') { $g = $Matches[1] }; if ($l -match 'checksum=(\S+)') { $ck = $Matches[1] }; if ($l -match 'status=(\S+)') { $st = $Matches[1] }
if ($l -match 'start_utc=(\S+)') { $su = $Matches[1] }; if ($l -match 'end_utc=(\S+)') { $eu = $Matches[1] }
$t0 = ParseUtc $su; $t1 = ParseUtc $eu
if ($t0 -and $t1) { $w = Window $t0.AddSeconds(8) $t1; $watts = Mean ($w | ForEach-Object { $_[1] }); $sm = Mean ($w | ForEach-Object { $_[2] }); $mem = Mean ($w | ForEach-Object { $_[3] }); $util = Mean ($w | ForEach-Object { $_[5] }); $wi = Mean ($w | Where-Object { $_.Count -ge 8 -and $_[6] -match '^\s*[\d.]+\s*$' } | ForEach-Object { $_[6] }); $wa = Mean ($w | Where-Object { $_.Count -ge 8 -and $_[7] -match '^\s*[\d.]+\s*$' } | ForEach-Object { $_[7] }); "RESULT ca4-research probe=$name state=$state watts=$watts watts_instant=$wi watts_average=$wa sm_mhz=$sm mem_mhz=$mem util=$util G_ops_s=$g checksum=$ck status=$st samples=$($w.Count)" }
else { "RESULT ca4-research probe=$name state=$state watts= (no start_utc/end_utc on the line) G_ops_s=$g checksum=$ck status=$st" }
}
"RESULT microbench-state $state end $(Stamp) exit=$rc lines=$($lines.Count)"
}
$locked = $false
try {
RunState 'unlocked'
if ($viaHelper) {
$r = HelperSend 'lgc 1300'
if ($r -match 'no helper.log') { "RESULT lock1300 skipped: the helper gave no answer ($r)" } else { "RESULT lock1300 set: $r"; $locked = $true; Start-Sleep -Seconds 3; RunState 'lock1300' }
} else { 'RESULT lock1300 skipped: no helper path' }
Summary 'done' @{ locked_state_ran = $locked }
exit 0
} finally {
if ($locked) { $r = HelperSend 'rgc'; "RESULT restore rgc: $r ; applications clock: $((& $smi -i $idx --query-gpu=clocks.applications.graphics --format=csv,noheader 2>&1 | Out-String).Trim())" }
if ($sampler -and -not $sampler.HasExited) { Stop-Process -Id $sampler.Id -Force -ErrorAction SilentlyContinue; 'RESULT restore sampler stopped' }
}

View file

@ -0,0 +1,49 @@
# PC 1: why does the Power Helper process die right after "helper started" (7 October 2026, 21:3x UTC)? Read only, plus one
# 20-second UNELEVATED probe: the task's exe path and the file's mtime, version and sha256 against the running app's exe; the
# task's scheduler history (events 100 to 203 of the last hour: start, action start, action end with its exit code); the helper
# started by hand as a child of this job (`igneum-app.exe --power-helper`, no rights, so a `dev` line is answered with the
# helper's own log line and an nvidia-smi permission text at most, which proves the loop runs) for 20 s, its exit code, and
# helper.log's new lines; then the child is ended by its pid. Nothing is set on any card, nothing posted to the app, settings.json
# untouched, no prompt. Rows: RESULT lines, a SUMMARY {json}.
$ErrorActionPreference = 'Continue'
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
"RESULT start $(Stamp)"
$task = Get-ScheduledTask -TaskName 'Igneum Power Helper' -ErrorAction SilentlyContinue
if (-not $task) { 'RESULT task NOT REGISTERED'; 'SUMMARY {"job":"pc1-helper-diag2","status":"no-task"}'; exit 3 }
$exe = $task.Actions[0].Execute.Trim('"')
$info = Get-ScheduledTaskInfo -TaskName 'Igneum Power Helper' -ErrorAction SilentlyContinue
"RESULT task state=$($task.State) exe=$exe args=$($task.Actions[0].Arguments) runlevel=$($task.Principal.RunLevel) user=$($task.Principal.UserId) last_run=$($info.LastRunTime) last_result=$($info.LastTaskResult) missed=$($info.NumberOfMissedRuns)"
if (Test-Path -LiteralPath $exe) { $f = Get-Item -LiteralPath $exe; "RESULT taskexe mtime=$($f.LastWriteTimeUtc.ToString('yyyy-MM-ddTHH:mm:ssZ')) bytes=$($f.Length) version=$($f.VersionInfo.FileVersion) product=$($f.VersionInfo.ProductVersion) sha256=$((Get-FileHash -Algorithm SHA256 -LiteralPath $exe).Hash.ToLower())" } else { "RESULT taskexe MISSING at $exe" }
$app = @(Get-CimInstance Win32_Process -Filter "Name = 'igneum-app.exe'" -ErrorAction SilentlyContinue)
foreach ($p in $app) { $path = $p.ExecutablePath; $v = ''; $sha = ''; if ($path -and (Test-Path -LiteralPath $path)) { $v = (Get-Item -LiteralPath $path).VersionInfo.FileVersion; $sha = (Get-FileHash -Algorithm SHA256 -LiteralPath $path).Hash.ToLower() }; "RESULT appproc pid=$($p.ProcessId) started=$($p.ConvertToDateTime($p.CreationDate).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')) path=$path version=$v sha256=$sha cmd=$($p.CommandLine)" }
try { $ver = (& $exe --version 2>&1 | Out-String).Trim() -replace '\s+', ' '; "RESULT taskexe --version: $ver" } catch { "RESULT taskexe --version failed: $_" }
# the scheduler's history for the task (needs the TaskScheduler operational log enabled; absent = says so)
try {
$since = (Get-Date).AddHours(-2)
$ev = @(Get-WinEvent -FilterHashtable @{ LogName = 'Microsoft-Windows-TaskScheduler/Operational'; StartTime = $since } -ErrorAction Stop | Where-Object { $_.Message -match 'Igneum Power Helper' } | Sort-Object TimeCreated | Select-Object -Last 30)
if ($ev.Count -eq 0) { 'RESULT taskhistory no events for the task in 2 h (the operational log may be off)' }
foreach ($e in $ev) { "RESULT taskhistory $($e.TimeCreated.ToUniversalTime().ToString('HH:mm:ssZ')) id=$($e.Id) $(($e.Message -replace '\s+', ' ').Substring(0, [math]::Min(160, ($e.Message -replace '\s+', ' ').Length)))" }
} catch { "RESULT taskhistory unreadable: $(($_ | Out-String).Trim() -replace '\s+', ' ')" }
# the application log: a crash of igneum-app.exe in the last 2 h (Windows Error Reporting / Application Error)
try {
$crash = @(Get-WinEvent -FilterHashtable @{ LogName = 'Application'; StartTime = (Get-Date).AddHours(-2) } -ErrorAction Stop | Where-Object { $_.Message -match 'igneum-app' -and ($_.ProviderName -match 'Application Error|Windows Error Reporting|\.NET Runtime') } | Select-Object -Last 6)
if ($crash.Count -eq 0) { 'RESULT crashlog no igneum-app crash events in 2 h' }
foreach ($e in $crash) { "RESULT crashlog $($e.TimeCreated.ToUniversalTime().ToString('HH:mm:ssZ')) $($e.ProviderName): $(($e.Message -replace '\s+', ' ').Substring(0, [math]::Min(220, ($e.Message -replace '\s+', ' ').Length)))" }
} catch { "RESULT crashlog unreadable: $(($_ | Out-String).Trim() -replace '\s+', ' ')" }
# the unelevated probe: the helper loop as this user, 20 s, one dev line
$helperDir = Join-Path $env:LOCALAPPDATA 'igneum\app\sweep'; $cmdFile = Join-Path $helperDir 'cmd.txt'; $helperLog = Join-Path $helperDir 'helper.log'
$mark = 0; if (Test-Path $helperLog) { $mark = @(Get-Content -LiteralPath $helperLog).Count }
$m = 0; foreach ($f in @($cmdFile, $helperLog)) { if (Test-Path $f) { foreach ($l in (Get-Content -LiteralPath $f -ErrorAction SilentlyContinue)) { if ($l -match '(?:^|\s)(\d{1,6}) (?:dev|pl|lgc|rgc|lmc|rmc|nvidia-smi|reregister)\b') { $v = [int]$Matches[1]; if ($v -gt $m) { $m = $v } } } } }
$probeOut = Join-Path $env:TEMP 'igneum-helper-probe.out'; $probeErr = Join-Path $env:TEMP 'igneum-helper-probe.err'
if (Test-Path $probeOut) { Remove-Item -LiteralPath $probeOut -Force }; if (Test-Path $probeErr) { Remove-Item -LiteralPath $probeErr -Force }
$child = $null
try { $child = Start-Process -FilePath $exe -ArgumentList '--power-helper' -WindowStyle Hidden -PassThru -RedirectStandardOutput $probeOut -RedirectStandardError $probeErr; "RESULT probe started pid=$($child.Id) $(Stamp)" } catch { "RESULT probe start failed: $_" }
Start-Sleep -Seconds 4
if ($child) { "RESULT probe alive_after_4s=$(-not $child.HasExited) exit=$(if ($child.HasExited) { $child.ExitCode } else { '' })" }
if ($child -and -not $child.HasExited) { $s0 = $m + 100; Set-Content -LiteralPath $cmdFile -Value "$s0 dev 1`n" -NoNewline -Encoding ascii; "RESULT probe sent '$s0 dev 1'"; Start-Sleep -Seconds 6; "RESULT probe alive_after_10s=$(-not $child.HasExited)" }
Start-Sleep -Seconds 10
if ($child) { "RESULT probe alive_after_20s=$(-not $child.HasExited) exit=$(if ($child.HasExited) { $child.ExitCode } else { '' })"; if (-not $child.HasExited) { try { Stop-Process -Id $child.Id -Force; 'RESULT probe ended by pid' } catch { } } }
foreach ($f in @($probeOut, $probeErr)) { if (Test-Path $f) { $t = (Get-Content -LiteralPath $f -Raw -ErrorAction SilentlyContinue); if ($t) { "RESULT probe $(Split-Path -Leaf $f): $(($t -replace '\s+', ' ').Substring(0, [math]::Min(400, ($t -replace '\s+', ' ').Length)))" } } }
$new = @(Get-Content -LiteralPath $helperLog -ErrorAction SilentlyContinue); if ($new.Count -gt $mark) { foreach ($l in $new[$mark..($new.Count - 1)]) { "RESULT helperlog-new $l" } } else { 'RESULT helperlog-new none' }
'SUMMARY {"job":"pc1-helper-diag2","status":"done"}'
exit 0

View file

@ -30,6 +30,9 @@ case "$STEP" in
ca4-sparse) $P add --kind run --target ae432dc7 --id run-ca4-pc1-ca4sparse-5090-20261007 --cards-off "nvidia:NVIDIA GeForce RTX 5090" --script tools/ca3-v4-amend/pc1-v4-efficiency.ps1 --timeout-minutes 70 --title "CA4 research: SM-sparse worker variants on the RTX 5090, unlocked and at the 1,300 MHz knee, v4 and v3 (the card alone)" --expires-hours 36 $DEPLOY ;;
eff-5090-floor2) $P add --kind run --target ae432dc7 --id run-ca3-pc1-v4-eff-5090-floor2-20261007 --cards-off "nvidia:NVIDIA GeForce RTX 5090" --script tools/ca3-v4-amend/pc1-v4-efficiency.ps1 --timeout-minutes 45 --title "CA3 PC 1: class v4 efficiency pass 3 on the RTX 5090 (1,100 MHz down to the driver floor, v4 against v3, the card alone)" --expires-hours 36 $DEPLOY ;;
eff-5080) $P add --kind run --target ae432dc7 --id run-ca3-pc1-v4-eff-5080-20261007-c --cards-off "nvidia:NVIDIA GeForce RTX 5080" --script tools/ca3-v4-amend/pc1-v4-efficiency.ps1 --timeout-minutes 75 --title "CA3 PC 1: class v4 efficiency pass on the RTX 5080 (clock locks, v4 against v3, the card alone)" --expires-hours 36 $DEPLOY ;;
helper-diag2) $P add --kind run --target ae432dc7 --id run-ca3-pc1-helper-diag2-20261007 --script tools/ca3-v4-amend/pc1-helper-diag2.ps1 --timeout-minutes 5 --title "PC 1: Power Helper diagnostic 2 (the task exe against the running app, scheduler history, crash log, a 20 s unelevated helper probe)" --expires-hours 12 $DEPLOY ;;
ca4-mb-exe) $P add --kind fetch --target ae432dc7 --id fetch-ca4-microbench-exe-20261007 --file "$KITS/ca4/igneum-worker-cuda-ca4mb.exe" --dir jobs --title "CA4 research: the microbench worker exe (scratch build)" --expires-hours 36 $DEPLOY ;;
ca4-mb) $P add --kind run --target ae432dc7 --id run-ca4-pc1-microbench-5090-20261007 --cards-off "nvidia:NVIDIA GeForce RTX 5090" --script tools/ca3-v4-amend/pc1-ca4-microbench.ps1 --timeout-minutes 60 --title "CA4 research: per-block micro-benchmark on the RTX 5090, unlocked and at the 1,300 MHz knee (the card alone)" --expires-hours 36 $DEPLOY ;;
amd-g1) $P add --kind run --target ae432dc7 --id run-ca3-pc1-v4-sub3-amd-g1-20261007 --cards-off amd:gfx1201 --script tools/ca3-v4-amend/pc1-v4-sub3-amd-g1.ps1 --timeout-minutes 45 --title "CA3 PC 1: G1 on the sub-version 3 kit and the AMD ladder on the 9070 XT (the card alone)" --expires-hours 36 $DEPLOY ;;
family) $P add --kind run --target ae432dc7 --id run-ca3-pc1-amd-family-20261007-e --cards-off amd:gfx1201 --script tools/ca3-v4-amend/pc1-amd-family-20261007.ps1 --timeout-minutes 20 --title "CA3 PC 1: item 6 family step costs on the 9070 XT (run e, the card alone)" --expires-hours 36 $DEPLOY ;;
tune-5080) $P add --kind run --target ae432dc7 --id run-ca3-pc1-ember-5080-20261007 --script tools/ca3-v4-amend/pc1-ember-card.ps1 --timeout-minutes 35 --title "CA3 PC 1: Ember Tune on the RTX 5080 (the installed app tunes the one card)" --expires-hours 36 $DEPLOY ;;