From 14dc093549812d40cd23fa3cf12af50ff647fa4a Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:00:30 +0000 Subject: [PATCH 1/4] Ember shipped-app window playbook: the version gate on the first line (0.3.14 or later, the climb field present, else abort before anything is asked), the prompt count from the app's log over the window, the climb against run 6's ladder floor per NVIDIA card Co-Authored-By: Claude Fable 5.1 --- relay/playbooks/ember-installed-tune-pc1.ps1 | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/relay/playbooks/ember-installed-tune-pc1.ps1 b/relay/playbooks/ember-installed-tune-pc1.ps1 index 4426ef81..70a0b7c6 100644 --- a/relay/playbooks/ember-installed-tune-pc1.ps1 +++ b/relay/playbooks/ember-installed-tune-pc1.ps1 @@ -16,6 +16,11 @@ function Post([string] $path, [string] $json) { try { Invoke-WebRequest -Uri ($b $s = State if (-not $s) { Write-Output 'RESULT TUNE error=state_unreadable'; exit 1 } $ver = [string]$s.version +# the first line: the version gate (main, 6 October 2026): 0.3.14 or later carries bd7fcf4 (AMD apply on ack) and +# 5165de7 (api/tune/goal with the climb); an older app aborts here before anything is asked of it +$vok = $false; try { $vok = ([version]($ver -replace '[^0-9.].*$', '')) -ge [version]'0.3.14' } catch { } +Write-Output ('RESULT TUNE version=' + $ver + ' gate=' + $(if ($vok) { 'ok (0.3.14 or later)' } else { 'FAILED (needs 0.3.14 with bd7fcf4 and 5165de7)' }) + ' climb_field=' + $(if ($null -ne $s.settings.tune_climb) { 'present' } else { 'absent' })) +if (-not $vok -or $null -eq $s.settings.tune_climb) { Write-Output 'RESULT TUNE error=version_gate'; exit 1 } Write-Output ('RESULT TUNE installed app ' + $ver + ' power_control=' + $s.settings.power_control + ' tuning_off=' + $s.settings.tuning_off + ' climb_before=' + $s.settings.tune_climb + ' goal_before=' + $s.settings.tune_goal) $t = Get-ScheduledTask -TaskName 'Igneum Power Helper' -ErrorAction SilentlyContinue Write-Output ('RESULT TUNE task ' + $(if ($t) { $t.Actions[0].Execute + ' state=' + $t.State } else { 'NOT REGISTERED' })) @@ -34,6 +39,9 @@ $logs = Join-Path $appDir 'logs' function LogMark() { $l = Get-ChildItem -Path $logs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($l) { return @{ f = $l.FullName; n = (Get-Content -LiteralPath $l.FullName).Count } } else { return @{ f = ''; n = 0 } } } function LogSince($m) { if (-not $m.f) { return @() }; $all = @(Get-Content -LiteralPath $m.f -ErrorAction SilentlyContinue); if ($all.Count -le $m.n) { return @() }; return @($all[$m.n..($all.Count - 1)]) } $rows = 0 +$m0 = LogMark +# run 6's ladder floors (6 October 2026, PC 1), what the climb has to beat +$floor = @{ 'NVIDIA GeForce RTX 5090' = @{ eff = 0.5632; mhs = 127.71; watts = 226.8; point = '1854 MHz at 100%' }; 'NVIDIA GeForce RTX 4070' = @{ eff = 0.3809; mhs = 28.78; watts = 75.6; point = '1863 MHz at 50%' } } foreach ($c in $order) { if ((Get-Date) -gt $deadline) { Say 'budget spent before ' + $c.key; break } $m = LogMark @@ -58,6 +66,24 @@ foreach ($c in $order) { $st = State if ($st) { $cc = $st.mining.cards | Where-Object { $_.key -eq $c.key } | Select-Object -First 1; if ($cc) { Write-Output ('RESULT TUNE after key=' + $c.key + ' power_w=' + $cc.power_w + ' power_pct=' + $cc.power_pct + ' clock_cap=' + $cc.clock_cap_mhz + ' line=' + $cc.tune_line + ' source=' + $cc.tune_source + ' note=' + $cc.sweep_note); if ($cc.tune_curve) { Write-Output ('RESULT TUNE curve key=' + $c.key + ' ' + (($cc.tune_curve | ForEach-Object { ($_.clock_mhz) + 'MHz/' + ($_.mem_mhz) + 'mem/' + ($_.power_pct) + '%:' + $_.mhs + 'MH/s ' + $_.watts + 'W ' + $_.eff + ' ' + $_.mark }) -join ' | ')) } } } } +# the prompt count: the app's own log over the window; a task line says "(no prompt)", a prompted helper says +# "administrator prompt" (the Security log's process audit is off on PC 1, so the app's log is the record) +$win = @(LogSince $m0) +$noPrompt = @($win | Where-Object { $_ -match 'Power Helper task \(no prompt\)' }).Count +$prompted = @($win | Where-Object { $_ -match 'administrator prompt|runas|UAC' }).Count +Write-Output ('RESULT TUNE prompts=' + $prompted + ' task_lines=' + $noPrompt + ' (from the app log over the window)') +# the climb against run 6's ladder floor, per NVIDIA card +$st = State +if ($st) { + foreach ($c in @($cards | Where-Object { $_.vendor -eq 'nvidia' })) { + $cc = $st.mining.cards | Where-Object { $_.key -eq $c.key } | Select-Object -First 1 + $f = $floor[[string]$c.name] + if ($cc -and $f -and $cc.sweep_eff -gt 0) { + $d = [math]::Round(($cc.sweep_eff - $f.eff) / $f.eff * 100, 1) + Write-Output ('RESULT TUNE versus_run6 key=' + $c.key + ' climb=' + $cc.sweep_eff + ' MH/W (' + $cc.sweep_mhs + ' MH/s at ' + $cc.sweep_watts + ' W) ladder_floor=' + $f.eff + ' (' + $f.point + ') delta=' + $d + '% ' + $(if ($cc.sweep_eff -gt $f.eff) { 'the climb beat the floor' } elseif ($cc.sweep_eff -eq $f.eff) { 'equal' } else { 'the ladder floor stands' })) + } elseif ($cc -and $f) { Write-Output ('RESULT TUNE versus_run6 key=' + $c.key + ' no chosen point this run (' + $cc.sweep_note + ')') } + } +} # the climb switch back to what it was Post 'api/tune/goal' ('{"climb":' + $(if ($climbBefore) { 'true' } else { 'false' }) + '}') | Out-Null Write-Output ('RESULT TUNE climb_restored=' + $climbBefore + ' rows=' + $rows) From 0659fab5f2c33878b615d1d6fdd25f4a8eb84b8d Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:31:38 +0000 Subject: [PATCH 2/4] Ember helper, two bugs from PC 1 on 6 October 2026: (A) a helper acts only on lines added after its start, so a stale quit or remove never kills a later start (commands_after + test); (B) the engine starts the task again before every task-path step and the acknowledgement is the helper's own log line for the sequence (up to 15 s), a missing line fails the step with that reason instead of a blind 4-second ack. Playbooks read the app log from \logs, where it lives Co-Authored-By: Claude Fable 5.1 --- app/igneum-app/src/engine.rs | 33 +++++++++++++++++--- app/igneum-app/src/powertask.rs | 27 +++++++++++++--- relay/playbooks/ember-installed-tune-pc1.ps1 | 2 +- relay/playbooks/ember-proof-collect-pc1.ps1 | 4 +-- 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/app/igneum-app/src/engine.rs b/app/igneum-app/src/engine.rs index 11e133f5..dd068802 100644 --- a/app/igneum-app/src/engine.rs +++ b/app/igneum-app/src/engine.rs @@ -2567,11 +2567,36 @@ impl Engine { } let dev = device.to_string(); let cmd = format!("{seq}0 dev {dev}\n{seq}1 pl {w}\n{seq}2 {}\n{seq}3 {}\n", if clock > 0 { format!("lgc {clock}") } else { "rgc".to_string() }, if mem > 0 { format!("lmc {mem}") } else { "rmc".to_string() }); - let _ = std::fs::write(self.helper_cmd_dir().join("cmd.txt"), cmd); - // the helper polls twice a second and nvidia-smi answers within a second or two + let dir = self.helper_cmd_dir(); + let is_task = self.sweep_helper_is_task; + let _ = std::fs::create_dir_all(&dir); + let _ = std::fs::write(dir.join("cmd.txt"), cmd); + // 6 October 2026, PC 1 18:18Z: the task's helper had exited (20 idle minutes) while the engine's flag + // still said it ran, so the commands went to nobody and a blind 4-second sleep called them + // acknowledged. Now: the task is started again before every step (Start-ScheduledTask ignores a + // start while one runs), and the acknowledgement is the helper's own log line for this sequence + // (`3 nvidia-smi ...`), read for up to 15 s; a missing line is a failure with that reason + if is_task { + if let Err(e) = crate::powertask::start() { + self.shared.log(&format!("tune helper: the task could not be started ({e})")); + } + } + let want = format!(" {seq}3 nvidia-smi"); + let log_file = dir.join("helper.log"); std::thread::spawn(move || { - std::thread::sleep(Duration::from_secs(4)); - shared.send(Cmd::TuneSet(seq, Ok("helper".into()))); + let until = Instant::now() + Duration::from_secs(if is_task { 15 } else { 6 }); + loop { + std::thread::sleep(Duration::from_millis(500)); + let text = std::fs::read_to_string(&log_file).unwrap_or_default(); + if let Some(line) = text.lines().rev().find(|l| l.contains(&want)) { + shared.send(Cmd::TuneSet(seq, Ok(format!("helper: {}", line.trim())))); + return; + } + if Instant::now() >= until { + shared.send(Cmd::TuneSet(seq, Err(format!("the helper did not run sequence {seq} within {} s (no line in {})", if is_task { 15 } else { 6 }, log_file.display())))); + return; + } + } }); } "amd" => { diff --git a/app/igneum-app/src/powertask.rs b/app/igneum-app/src/powertask.rs index 232ef558..382f893d 100644 --- a/app/igneum-app/src/powertask.rs +++ b/app/igneum-app/src/powertask.rs @@ -79,6 +79,13 @@ pub fn parse_line(line: &str) -> Option<(u64, HelperCmd)> { } } +/// The commands a helper acts on: the lines added after its start (`skip` = the line count at the start, 0 again +/// when the file shrank). A stale `quit` or `remove` from an earlier engine is never a command. +pub fn commands_after(text: &str, skip: usize) -> Vec<(u64, HelperCmd)> { + let skip = if text.lines().count() < skip { 0 } else { skip }; + text.lines().skip(skip).filter_map(parse_line).collect() +} + /// The nvidia-smi arguments a command becomes (None for the verbs that run nothing). pub fn smi_args(dev: &str, c: &HelperCmd) -> Option> { match c { @@ -195,17 +202,25 @@ pub fn run_helper(dir: &Path) -> i32 { } }; log("helper started (scheduled task, elevated)"); - // a stale file from an earlier run is not a command: only lines after the start count - let mut last_seq: u64 = std::fs::read_to_string(&cmd_file).ok().and_then(|t| t.lines().filter_map(parse_line).map(|(s, _)| s).max()).unwrap_or(0); - let mut last_text = String::new(); + // a stale file from an earlier run is not a command: only lines ADDED after the start count. 6 October 2026, + // PC 1 17:55:55Z: a helper that started after an engine had written `quit` read that line and exited in the + // same second, and every later start did the same; so the lines present at the start are skipped whole + // (quit and remove included), and a file that shrinks starts the count again + let initial = std::fs::read_to_string(&cmd_file).unwrap_or_default(); + let mut last_seq: u64 = initial.lines().filter_map(parse_line).map(|(s, _)| s).max().unwrap_or(0); + let mut skip = initial.lines().count(); + let mut last_text = initial; let mut dev = "0".to_string(); let mut idle = Instant::now(); let smi = crate::platform::tool("nvidia-smi"); loop { let text = std::fs::read_to_string(&cmd_file).unwrap_or_default(); if text != last_text { + if text.lines().count() < skip { + skip = 0; + } last_text = text.clone(); - for (seq, c) in text.lines().filter_map(parse_line) { + for (seq, c) in commands_after(&text, skip) { match c { HelperCmd::Quit => { log("quit"); @@ -296,6 +311,10 @@ mod tests { assert_eq!(parse_line("quit"), Some((0, HelperCmd::Quit))); assert_eq!(parse_line("remove"), Some((0, HelperCmd::Remove))); assert_eq!(parse_line("12 reregister"), Some((12, HelperCmd::Reregister))); + // a stale quit present at the start is skipped; a quit added later counts; a rewritten (shorter) file counts whole + assert_eq!(commands_after("5 dev 0\n6 pl 460\nquit\n", 3), vec![]); + assert_eq!(commands_after("5 dev 0\n6 pl 460\nquit\n7 dev 1\n", 3), vec![(7, HelperCmd::Dev("1".into()))]); + assert_eq!(commands_after("quit\n", 3), vec![(0, HelperCmd::Quit)]); assert_eq!(parse_line("12 reregister C:\\evil.exe"), None, "no path argument: the helper picks the install folder itself"); assert_eq!(smi_args("0", &HelperCmd::Reregister), None); assert!(readback_command().contains("Actions[0].Execute")); diff --git a/relay/playbooks/ember-installed-tune-pc1.ps1 b/relay/playbooks/ember-installed-tune-pc1.ps1 index 70a0b7c6..00da2310 100644 --- a/relay/playbooks/ember-installed-tune-pc1.ps1 +++ b/relay/playbooks/ember-installed-tune-pc1.ps1 @@ -35,7 +35,7 @@ if (-not (Post 'api/tune/goal' '{"goal":"balanced","climb":true}')) { Write-Outp Start-Sleep -Seconds 2 $order = @($cards | Where-Object { $_.vendor -eq 'amd' }) + @($cards | Where-Object { $_.vendor -eq 'nvidia' }) $deadline = (Get-Date).AddMinutes(27) -$logs = Join-Path $appDir 'logs' +$logs = Join-Path $env:LOCALAPPDATA 'igneum\logs' # the app's log folder is \logs (platform::log_dir), not app\logs (6 October 2026: the first window read no log at all) function LogMark() { $l = Get-ChildItem -Path $logs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($l) { return @{ f = $l.FullName; n = (Get-Content -LiteralPath $l.FullName).Count } } else { return @{ f = ''; n = 0 } } } function LogSince($m) { if (-not $m.f) { return @() }; $all = @(Get-Content -LiteralPath $m.f -ErrorAction SilentlyContinue); if ($all.Count -le $m.n) { return @() }; return @($all[$m.n..($all.Count - 1)]) } $rows = 0 diff --git a/relay/playbooks/ember-proof-collect-pc1.ps1 b/relay/playbooks/ember-proof-collect-pc1.ps1 index 3db3b7e8..91fb11a1 100644 --- a/relay/playbooks/ember-proof-collect-pc1.ps1 +++ b/relay/playbooks/ember-proof-collect-pc1.ps1 @@ -3,10 +3,10 @@ # last two hours from the Security log. Never writes, quits, pauses or resumes anything. $ErrorActionPreference = 'Continue' $a = Join-Path $env:LOCALAPPDATA 'igneum\app' -$l = Get-ChildItem (Join-Path $a 'logs') -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 +$l = Get-ChildItem (Join-Path $env:LOCALAPPDATA 'igneum\logs') -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 if ($l) { Write-Output ('RESULT PROOF log=' + $l.FullName) - Get-Content -LiteralPath $l.FullName | Where-Object { $_ -match 'power cap|Power Helper|elevat|UAC|runas|tune' } | Select-Object -Last 24 | ForEach-Object { Write-Output ('RESULT PROOF applog ' + $_) } + Get-Content -LiteralPath $l.FullName | Where-Object { $_ -match 'power cap|Power Helper|elevat|UAC|runas|tune' } | Select-Object -Last 60 | ForEach-Object { Write-Output ('RESULT PROOF applog ' + $_) } } $h = Join-Path $a 'sweep\helper.log' if (Test-Path -LiteralPath $h) { Get-Content -LiteralPath $h | Select-Object -Last 15 | ForEach-Object { Write-Output ('RESULT PROOF helper ' + $_) } } else { Write-Output 'RESULT PROOF helper none' } From 2b244d732e51bf9b030dc9e1eac75011797dc6c9 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:33:19 +0000 Subject: [PATCH 3/4] Ember: a reads-only log grep playbook (the app's own tune lines, the helper log, cmd.txt) Co-Authored-By: Claude Fable 5.1 --- relay/playbooks/ember-log-grep-pc1.ps1 | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 relay/playbooks/ember-log-grep-pc1.ps1 diff --git a/relay/playbooks/ember-log-grep-pc1.ps1 b/relay/playbooks/ember-log-grep-pc1.ps1 new file mode 100644 index 00000000..babe5739 --- /dev/null +++ b/relay/playbooks/ember-log-grep-pc1.ps1 @@ -0,0 +1,12 @@ +# Ember: reads only. The installed app's own tune lines (not the job echoes) from \logs, newest file: +# the engine's "tune:" requests and answers, "tune helper", TUNE start/aborted/chosen/rows, and the helper.log tail. +$ErrorActionPreference = 'Continue' +$l = Get-ChildItem (Join-Path $env:LOCALAPPDATA 'igneum\logs') -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 +if (-not $l) { Write-Output 'RESULT GREP no app log'; exit 1 } +Write-Output ('RESULT GREP log=' + $l.FullName) +Get-Content -LiteralPath $l.FullName | Where-Object { ($_ -match '\] tune|^\d+ tune|tune helper|TUNE (start|aborted|chosen|card=|helper)|refused|did not take|power cap|Power Helper') -and ($_ -notmatch 'job ember-|RESULT ') } | Select-Object -Last 80 | ForEach-Object { Write-Output ('RESULT GREP ' + $_) } +$h = Join-Path $env:LOCALAPPDATA 'igneum\app\sweep\helper.log' +if (Test-Path -LiteralPath $h) { Get-Content -LiteralPath $h | Select-Object -Last 12 | ForEach-Object { Write-Output ('RESULT GREP helper ' + $_) } } +$c = Join-Path $env:LOCALAPPDATA 'igneum\app\sweep\cmd.txt' +if (Test-Path -LiteralPath $c) { Write-Output ('RESULT GREP cmd.txt: ' + ((Get-Content -LiteralPath $c) -join ' / ')) } +exit 0 From 0b48c028f337ac11cd8a1ad9a0f1ee92e08db156 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:39:00 +0000 Subject: [PATCH 4/4] Ember helper for 0.3.15, the 0.3.14 window's five causes: (B, C) the helper writes a heartbeat every poll, the engine starts the task and waits for a fresh heartbeat before writing any command, the acknowledgement is the helper's own log line; (D) a measure-only card is never set (ember::may_set); (E) an explicit Tune now clears the failure back-off and a held row says when the retry comes (ember::retry_note); tests. The shipped-app window playbook's gate at 0.3.15; relay/playbooks/ember-tune-pc1.ps1 deleted (superseded); the post-mortem table and the 0.3.15 next-cut table in the plan; bench log Co-Authored-By: Claude Fable 5.1 --- app/igneum-app/src/ember.rs | 34 +++ app/igneum-app/src/engine.rs | 51 ++-- app/igneum-app/src/powertask.rs | 49 ++++ docs/bench-log.md | 9 + docs/plans/ember-tune.md | 24 ++ relay/playbooks/ember-installed-tune-pc1.ps1 | 6 +- relay/playbooks/ember-log-grep-pc1.ps1 | 2 +- relay/playbooks/ember-tune-pc1.ps1 | 265 ------------------- 8 files changed, 156 insertions(+), 284 deletions(-) delete mode 100644 relay/playbooks/ember-tune-pc1.ps1 diff --git a/app/igneum-app/src/ember.rs b/app/igneum-app/src/ember.rs index 9a9f5a35..60b4325b 100644 --- a/app/igneum-app/src/ember.rs +++ b/app/igneum-app/src/ember.rs @@ -906,6 +906,28 @@ pub fn tuned_line(mhs: f64, watts: f64, eff: f64) -> String { format!("Tuned: {mhs:.1} MH/s at {watts:.0} W ({eff:.3} MH/W)") } +/// (D) May a step be SET on this card? Measure-only (no control: Apple, NVIDIA without Power control, an AMD card +/// whose probe gave no tune line) sets nothing: the run notices the missing acknowledgement and measures. An AMD +/// card also needs its ADLX ordinal. 6 October 2026, PC 1 17:57Z: a measure-only 9070 XT was sent a set, refused. +pub fn may_set(vendor: &str, tune_control: bool, amd_ordinal: i64) -> bool { + match vendor { + "nvidia" => tune_control, + "amd" => tune_control && amd_ordinal >= 0, + _ => false, + } +} + +/// (E) The row's words while a back-off from a failed tune holds: when the retry comes and why it stopped. +pub fn retry_note(remaining_s: u64, last_note: &str) -> String { + let mins = (remaining_s + 59) / 60; + let why = last_note.trim_start_matches("tuning stopped: ").trim_start_matches("tuning: ").trim(); + if why.is_empty() { + format!("tuning: retry in {mins} min") + } else { + format!("tuning: retry in {mins} min ({why})") + } +} + /// The row's line for a baseline (measure-only) result: the card was measured as it runs, nothing was set. /// (6 October 2026: PC 1's rows read "Tuned: 114.2 MH/s at 305 W" for a baseline, which is not a tune.) pub fn measured_line(mhs: f64, watts: f64, eff: f64) -> String { @@ -1007,6 +1029,18 @@ mod tests { assert!(Run::applied(&step, Readback { limit_w: 90.0, acked: false }, 100.0)); } + #[test] + fn a_measure_only_card_is_never_set_and_a_back_off_says_when_the_retry_comes() { + assert!(!may_set("amd", false, 1), "the probe gave no tune line: measure only"); + assert!(may_set("amd", true, 1)); + assert!(!may_set("amd", true, -1), "no ADLX ordinal"); + assert!(!may_set("nvidia", false, -1), "Power control off: measure only"); + assert!(may_set("nvidia", true, -1)); + assert!(!may_set("apple", true, -1)); + assert_eq!(retry_note(3599, "tuning stopped: the card refused the setting (see the log)"), "tuning: retry in 60 min (the card refused the setting (see the log))"); + assert_eq!(retry_note(30, ""), "tuning: retry in 1 min"); + } + #[test] fn a_baseline_result_reads_measured_and_a_tune_reads_tuned() { assert_eq!(result_line(PlanKind::Baseline, 114.2, 305.0, 0.3744), "Measured: 114.2 MH/s at 305 W (0.374 MH/W)"); diff --git a/app/igneum-app/src/engine.rs b/app/igneum-app/src/engine.rs index dd068802..e16c9ad7 100644 --- a/app/igneum-app/src/engine.rs +++ b/app/igneum-app/src/engine.rs @@ -2219,8 +2219,23 @@ impl Engine { if c.tune_control && c.vendor == "nvidia" && !power_control && !forced { // measure-only until Power control is on; the baseline plan covers it below } - if self.sweep_retry.get(&c.index).map(|t| now < *t).unwrap_or(false) { - continue; + // (E) 6 October 2026, PC 1 18:18Z: three "Tune now" requests sat behind the hour's back-off from the + // 17:55 failures and the rows said "queued" with no start; an explicit request now clears the back-off, + // and a held card's row says when the retry comes + if let Some(t) = self.sweep_retry.get(&c.index).copied() { + if now < t { + if forced { + self.sweep_retry.remove(&c.index); + } else { + if c.sweep_pct == 0 || c.sweep_note.starts_with("tuning stopped") { + let note = crate::ember::retry_note((t - now).as_secs(), &c.sweep_note); + if let Some(cc) = self.st().mining.cards.get_mut(c.index) { + cc.sweep_note = note; + } + } + continue; + } + } } let stable = self.miners.iter().find(|m| m.card == c.index).map(|m| m.proc.as_ref().map(|p| p.started.elapsed() >= Duration::from_secs(crate::ember::STABLE_S)).unwrap_or(false) && m.last_status.is_some()).unwrap_or(false); let wait = if c.state != "mining" { @@ -2569,21 +2584,24 @@ impl Engine { let cmd = format!("{seq}0 dev {dev}\n{seq}1 pl {w}\n{seq}2 {}\n{seq}3 {}\n", if clock > 0 { format!("lgc {clock}") } else { "rgc".to_string() }, if mem > 0 { format!("lmc {mem}") } else { "rmc".to_string() }); let dir = self.helper_cmd_dir(); let is_task = self.sweep_helper_is_task; - let _ = std::fs::create_dir_all(&dir); - let _ = std::fs::write(dir.join("cmd.txt"), cmd); - // 6 October 2026, PC 1 18:18Z: the task's helper had exited (20 idle minutes) while the engine's flag - // still said it ran, so the commands went to nobody and a blind 4-second sleep called them - // acknowledged. Now: the task is started again before every step (Start-ScheduledTask ignores a - // start while one runs), and the acknowledgement is the helper's own log line for this sequence - // (`3 nvidia-smi ...`), read for up to 15 s; a missing line is a failure with that reason - if is_task { - if let Err(e) = crate::powertask::start() { - self.shared.log(&format!("tune helper: the task could not be started ({e})")); - } - } + // 6 October 2026, PC 1: (B) the task's helper had exited while the engine's flag still said it ran, + // so commands went to nobody and a blind 4-second sleep called them acknowledged; (C) the first + // request of the first tune was written in the same second the helper started, and the helper + // skipped it as stale. Now, in the task case: the helper's heartbeat says whether it runs; the task + // is started and a fresh heartbeat awaited BEFORE the command is written; and the acknowledgement + // is the helper's own log line for this sequence (`3 nvidia-smi ...`), read for up to 15 s; + // a missing line is a failure with that reason let want = format!(" {seq}3 nvidia-smi"); let log_file = dir.join("helper.log"); std::thread::spawn(move || { + if is_task { + if let Err(e) = crate::powertask::ensure_running(&dir, Duration::from_secs(12)) { + shared.send(Cmd::TuneSet(seq, Err(e))); + return; + } + } + let _ = std::fs::create_dir_all(&dir); + let _ = std::fs::write(dir.join("cmd.txt"), cmd); let until = Instant::now() + Duration::from_secs(if is_task { 15 } else { 6 }); loop { std::thread::sleep(Duration::from_millis(500)); @@ -2601,7 +2619,10 @@ impl Engine { } "amd" => { let Some(exe) = self.bins.telemetry.clone() else { return }; - if c.amd_ordinal < 0 { + // (D) 6 October 2026, PC 1 17:57Z: a measure-only card (the probe gave no tune line) was still sent a + // set, which the helper refused and the tune stopped; measure only means nothing is set, the run + // notices the missing acknowledgement and measures + if !crate::ember::may_set("amd", c.tune_control, c.amd_ordinal) { return; } let n = c.amd_ordinal.to_string(); diff --git a/app/igneum-app/src/powertask.rs b/app/igneum-app/src/powertask.rs index 382f893d..3f22d237 100644 --- a/app/igneum-app/src/powertask.rs +++ b/app/igneum-app/src/powertask.rs @@ -35,6 +35,46 @@ use std::time::{Duration, Instant}; pub const TASK_NAME: &str = "Igneum Power Helper"; /// The helper ends after this long without a new command. pub const IDLE_S: u64 = 20 * 60; +/// The heartbeat file the helper refreshes every poll; an engine judges "the helper runs" by it, never by a flag +/// of its own (6 October 2026, PC 1: the flag said yes after the helper's idle exit, and commands went to nobody). +pub const ALIVE_FILE: &str = "helper.alive"; +/// A heartbeat older than this is a dead helper. +pub const ALIVE_MAX_S: u64 = 4; + +/// Writes the heartbeat: the unix time, as text. +pub fn beat(dir: &Path, now: u64) { + let _ = std::fs::write(dir.join(ALIVE_FILE), now.to_string()); +} + +/// Reads a heartbeat: is the helper alive at `now`? +pub fn alive_at(text: &str, now: u64) -> bool { + text.trim().parse::().map(|t| now.saturating_sub(t) <= ALIVE_MAX_S).unwrap_or(false) +} + +/// Is the helper that reads `dir` alive now? +pub fn alive(dir: &Path) -> bool { + std::fs::read_to_string(dir.join(ALIVE_FILE)).map(|t| alive_at(&t, crate::platform::unix_now())).unwrap_or(false) +} + +/// Starts the task unless the helper already runs, then waits for a fresh heartbeat (up to `wait`). The engine +/// writes a command only after this says Ok: the helper skips what the file held before its start (a stale quit +/// is not a command), so a command written before the start would be skipped with it (6 October 2026, 17:55:25Z +/// on PC 1: the first request of the first tune). +pub fn ensure_running(dir: &Path, wait: Duration) -> Result<(), String> { + if alive(dir) { + return Ok(()); + } + let _ = std::fs::create_dir_all(dir); + start()?; + let until = Instant::now() + wait; + while Instant::now() < until { + std::thread::sleep(Duration::from_millis(250)); + if alive(dir) { + return Ok(()); + } + } + Err(format!("the Igneum Power Helper task gave no heartbeat within {} s of its start", wait.as_secs())) +} /// One parsed command from cmd.txt. #[derive(Clone, Debug, PartialEq, Eq)] @@ -202,6 +242,7 @@ pub fn run_helper(dir: &Path) -> i32 { } }; log("helper started (scheduled task, elevated)"); + beat(dir, crate::platform::unix_now()); // a stale file from an earlier run is not a command: only lines ADDED after the start count. 6 October 2026, // PC 1 17:55:55Z: a helper that started after an engine had written `quit` read that line and exited in the // same second, and every later start did the same; so the lines present at the start are skipped whole @@ -224,6 +265,7 @@ pub fn run_helper(dir: &Path) -> i32 { match c { HelperCmd::Quit => { log("quit"); + let _ = std::fs::remove_file(dir.join(ALIVE_FILE)); return 0; } HelperCmd::Remove => { @@ -276,8 +318,10 @@ pub fn run_helper(dir: &Path) -> i32 { } if idle.elapsed() >= Duration::from_secs(IDLE_S) { log("idle 20 min: exit (the engine starts the task again when it needs it)"); + let _ = std::fs::remove_file(dir.join(ALIVE_FILE)); return 0; } + beat(dir, crate::platform::unix_now()); std::thread::sleep(Duration::from_millis(500)); } } @@ -315,6 +359,11 @@ mod tests { assert_eq!(commands_after("5 dev 0\n6 pl 460\nquit\n", 3), vec![]); assert_eq!(commands_after("5 dev 0\n6 pl 460\nquit\n7 dev 1\n", 3), vec![(7, HelperCmd::Dev("1".into()))]); assert_eq!(commands_after("quit\n", 3), vec![(0, HelperCmd::Quit)]); + // the heartbeat: fresh within ALIVE_MAX_S, dead after, dead when unreadable + assert!(alive_at("1791309325", 1791309325 + ALIVE_MAX_S)); + assert!(!alive_at("1791309325", 1791309325 + ALIVE_MAX_S + 1)); + assert!(!alive_at("", 1791309325)); + assert!(!alive_at("soon", 1791309325)); assert_eq!(parse_line("12 reregister C:\\evil.exe"), None, "no path argument: the helper picks the install folder itself"); assert_eq!(smi_args("0", &HelperCmd::Reregister), None); assert!(readback_command().contains("Actions[0].Execute")); diff --git a/docs/bench-log.md b/docs/bench-log.md index e459ad46..1c43e6fe 100644 --- a/docs/bench-log.md +++ b/docs/bench-log.md @@ -2268,3 +2268,12 @@ task now runs ...Programs\Igneum Miner\igneum-app.exe --power-helper"), then the task with no prompt ("-pl 460: set to 460.00 W from 575.00 W", "-pl 160: set to 160.00 W from 100.00 W"). Task Running, Highest, user Admin. Cards: 5090 221 W at 1845 MHz, 4070 75.8 W at 1860 MHz, 9070 XT 202 W, all mining through the installed app. Ember closed 16:53Z. + +## 6 October 2026, 18:18Z: the shipped 0.3.14 app asked to tune itself on PC 1 (ember-installed-pc1-1): FAILED, no tune ran + +Gate ok (0.3.14, Power control on, the task on the installed exe). Every card's "after" was its 17:55Z failure from +the app's own first tune: 5090 and 4070 "did not take within 30 s (card reports 460 / 160 W, acknowledged true)", +9070 XT "the card refused the setting"; the three "Tune now" requests sat queued behind the hour's back-off. Five +causes (A to E) and their fixes in the plan's 0.3.14 window section; the window repeats on 0.3.15. The cards kept +mining at the driver's held locks: 5090 222.9 W at 1845 MHz, 4070 75.9 W at 1860 MHz, 9070 XT 201 W. Lever-2 TUNING +records during the window: 5090 135.6 MH/s at 236 W (0.574 MH/W), 4070 31.1 MH/s at 79 W (0.393). diff --git a/docs/plans/ember-tune.md b/docs/plans/ember-tune.md index 50487f36..6e806324 100644 --- a/docs/plans/ember-tune.md +++ b/docs/plans/ember-tune.md @@ -296,8 +296,32 @@ record. Known in 0.3.13 on PC 1 until 0.3.14: the rows show the app's own earlie to receive the run's result), and the 80% caps are the app's setting; the draw is the tuned one regardless because the driver holds the clock locks and the caps do not bind. +### The 0.3.14 window (6 October 2026, 18:18 to 18:27Z): no tune ran; five causes + +The shipped 0.3.14 app, Power control on, the task registered, asked over its own API to tune each card +(`relay/playbooks/ember-installed-tune-pc1.ps1`). From its log and helper.log: + +| | When (UTC) | What | Cause | Fix (0.3.15) | +|---|---|---|---|---| +| C | 17:55:25 | the app's own first 5090 tune wrote its first request in the second the helper started; the helper skipped it as "present at my start"; 30 s later "did not take (card reports 460 W, acknowledged true)" | the engine wrote before the helper was up | the helper writes a heartbeat (`helper.alive`) every poll; the engine starts the task and waits for a fresh heartbeat BEFORE writing (`powertask::ensure_running`) | +| A | 17:55:55 | the 4070's helper start read the stale `quit` from the 5090's abort and exited in the same second | quit/remove matched before the sequence check | the helper acts only on lines added after its start (`commands_after`) | +| D | 17:57:27 | the 9070 XT, measure only ("igneum-gpu-telemetry gave no tune line": the 3.0 coordinator's job was on the ADLX helper), was still sent a set; refused; "tuning stopped" | a measure-only card was set | `ember::may_set`: measure only sets nothing | +| E | 18:18 to 18:27 | three "Tune now" requests sat "queued" behind the hour's back-off from the 17:55 failures; none started | `sweep_retry` blocked a forced request silently | a forced request clears the back-off; a held row says "tuning: retry in N min (why)" (`ember::retry_note`) | +| B | (latent) | the engine's helper flag stayed true after the helper's idle exit; a later step would have been acknowledged after a blind 4 s sleep | a flag instead of a fact | the heartbeat decides; the acknowledgement is the helper's own log line for the sequence (up to 15 s), a missing line fails the step with that reason | + +What a user on 0.3.14 sees: Power control on, no prompt, and no card tunes (the rows keep the baseline or +"tuning stopped"); the lever-2 TUNING records still flow. 0.3.15 closes it; the window repeats there. + ## 8a. Next-cut notes (for the 0.3.12 shipper) +For the 0.3.15 cut (6 October 2026 evening), on ember-tune: + +| Commit | What | Where | +|---|---|---| +| a8cab23 | (A) the helper acts only on lines added after its start; the acknowledgement is the helper's log line for the sequence | powertask.rs `commands_after` + test, engine.rs | +| (tip) | (B, C) the heartbeat, `ensure_running` before every task-path write; (D) `may_set`; (E) the back-off yields to a forced request, `retry_note`; the window playbook's gate at 0.3.15; `ember-tune-pc1.ps1` deleted (superseded by the installed-tune playbook) | powertask.rs, engine.rs, ember.rs + tests, relay/playbooks | +| e70ac5e | every task command goes to the helper's fixed folder | platform.rs, powertask.rs, engine.rs, main.rs + test | + For the 0.3.14 cut (6 October 2026, after run 6), on ember-tune: | Commit | What | Where | diff --git a/relay/playbooks/ember-installed-tune-pc1.ps1 b/relay/playbooks/ember-installed-tune-pc1.ps1 index 00da2310..1848b53b 100644 --- a/relay/playbooks/ember-installed-tune-pc1.ps1 +++ b/relay/playbooks/ember-installed-tune-pc1.ps1 @@ -17,9 +17,9 @@ $s = State if (-not $s) { Write-Output 'RESULT TUNE error=state_unreadable'; exit 1 } $ver = [string]$s.version # the first line: the version gate (main, 6 October 2026): 0.3.14 or later carries bd7fcf4 (AMD apply on ack) and -# 5165de7 (api/tune/goal with the climb); an older app aborts here before anything is asked of it -$vok = $false; try { $vok = ([version]($ver -replace '[^0-9.].*$', '')) -ge [version]'0.3.14' } catch { } -Write-Output ('RESULT TUNE version=' + $ver + ' gate=' + $(if ($vok) { 'ok (0.3.14 or later)' } else { 'FAILED (needs 0.3.14 with bd7fcf4 and 5165de7)' }) + ' climb_field=' + $(if ($null -ne $s.settings.tune_climb) { 'present' } else { 'absent' })) +# 5165de7 (api/tune/goal with the climb); 0.3.15 the helper fixes A to E (6 October 2026); an older app aborts here +$vok = $false; try { $vok = ([version]($ver -replace '[^0-9.].*$', '')) -ge [version]'0.3.15' } catch { } +Write-Output ('RESULT TUNE version=' + $ver + ' gate=' + $(if ($vok) { 'ok (0.3.15 or later)' } else { 'FAILED (needs 0.3.15: the helper fixes A to E)' }) + ' climb_field=' + $(if ($null -ne $s.settings.tune_climb) { 'present' } else { 'absent' })) if (-not $vok -or $null -eq $s.settings.tune_climb) { Write-Output 'RESULT TUNE error=version_gate'; exit 1 } Write-Output ('RESULT TUNE installed app ' + $ver + ' power_control=' + $s.settings.power_control + ' tuning_off=' + $s.settings.tuning_off + ' climb_before=' + $s.settings.tune_climb + ' goal_before=' + $s.settings.tune_goal) $t = Get-ScheduledTask -TaskName 'Igneum Power Helper' -ErrorAction SilentlyContinue diff --git a/relay/playbooks/ember-log-grep-pc1.ps1 b/relay/playbooks/ember-log-grep-pc1.ps1 index babe5739..118661bf 100644 --- a/relay/playbooks/ember-log-grep-pc1.ps1 +++ b/relay/playbooks/ember-log-grep-pc1.ps1 @@ -4,7 +4,7 @@ $ErrorActionPreference = 'Continue' $l = Get-ChildItem (Join-Path $env:LOCALAPPDATA 'igneum\logs') -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 if (-not $l) { Write-Output 'RESULT GREP no app log'; exit 1 } Write-Output ('RESULT GREP log=' + $l.FullName) -Get-Content -LiteralPath $l.FullName | Where-Object { ($_ -match '\] tune|^\d+ tune|tune helper|TUNE (start|aborted|chosen|card=|helper)|refused|did not take|power cap|Power Helper') -and ($_ -notmatch 'job ember-|RESULT ') } | Select-Object -Last 80 | ForEach-Object { Write-Output ('RESULT GREP ' + $_) } +Get-Content -LiteralPath $l.FullName | Where-Object { ($_ -match '\] tune|^\d+ tune|tune helper|TUNE (start|aborted|chosen|card=|helper|probe)|refused|did not take|power cap|Power Helper|probe|telemetry|gfx1201|queued|tune due|not due') -and ($_ -notmatch 'job ember-|RESULT ') } | Select-Object -Last 80 | ForEach-Object { Write-Output ('RESULT GREP ' + $_) } $h = Join-Path $env:LOCALAPPDATA 'igneum\app\sweep\helper.log' if (Test-Path -LiteralPath $h) { Get-Content -LiteralPath $h | Select-Object -Last 12 | ForEach-Object { Write-Output ('RESULT GREP helper ' + $_) } } $c = Join-Path $env:LOCALAPPDATA 'igneum\app\sweep\cmd.txt' diff --git a/relay/playbooks/ember-tune-pc1.ps1 b/relay/playbooks/ember-tune-pc1.ps1 deleted file mode 100644 index beab8e1e..00000000 --- a/relay/playbooks/ember-tune-pc1.ps1 +++ /dev/null @@ -1,265 +0,0 @@ -# Igneum run job: Ember Tune end to end on PC 1 (machine ae432dc7), unattended. 5 October 2026. -# Published as a `run` job with --stop-miners (docs/plans/ember-tune.md): the installed app stops its miners and -# holds them; this script takes the engine that carries src/ember.rs (the one the fetch job put in -# \jobs\ember-kit-1\igneum-app-ember.exe, else the installed one; the AMD helper with the --tune and --set -# commands from the telemetry agent's fetch job, \jobs\amd-kit-1\kit\igneum-gpu-telemetry.exe), copies the install folder to a scratch -# folder beside it, swaps the engine in, and starts that SECOND engine with `--sweep` in a scratch data folder (the -# real settings.json, machine-id and wallet.json copied in; remote jobs, auto-update and proving switched off -# there). That engine finds the installed app's node on 127.0.0.1:26610, mines on every card with the live program, -# tunes them one after the other (the full two-knob plan where the card can be controlled, the baseline measurement -# where it cannot: NVIDIA without administrator rights, Apple), prints every TUNE line on stdout, uploads its log -# (the TUNE {json} record reaches the intake) and quits. Every TUNE line is re-emitted as a RESULT line, so -# `node tools/jobs.mjs ` shows the table. Before and after, nvidia-smi's limits and clocks and the AMD -# helper's `--tune` lines are printed, so the restore can be read. The installed app's miners restart when the job -# ends. Not elevated: nothing asks for administrator rights (the project lead asleep, 5 October 2026); the NVIDIA card is -# therefore measure only tonight unless the engine finds itself elevated. -$ErrorActionPreference = 'Continue' -$resultTag = 'TUNE' -$budgetMinutes = 55 -if (-not ($budgetMinutes -is [int]) -or $budgetMinutes -lt 5) { $budgetMinutes = 35 } # a budget under 5 minutes is a bug, not a budget (C35) -$started = Get-Date -$deadline = $started.AddMinutes($budgetMinutes) -function Say([string] $m) { Write-Host ("[" + (Get-Date -Format 'HH:mm:ss') + "] " + $m) } - -# the installed engine: the per-user install (0.3.3+), else Program Files -$installDir = $null -foreach ($d in @((Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner'), (Join-Path $env:ProgramFiles 'Igneum Miner'))) { - if (Test-Path (Join-Path $d 'igneum-app.exe')) { $installDir = $d; break } -} -if (-not $installDir) { Write-Output 'RESULT TUNE error=no_engine reason=igneum-app.exe_not_found'; exit 2 } -$appData = $env:IGNEUM_APP_DATA -if (-not $appData) { $appData = Join-Path $env:LOCALAPPDATA 'igneum' } -$appDir = $env:IGNEUM_APP_DIR -if (-not $appDir) { $appDir = Join-Path $appData 'app' } - -# the scratch install: the whole folder (workers, node, helper, DLLs) with the Ember engine swapped in -# a FRESH scratch root per run (run 3, 6 October 2026, 11:45Z: the folder an earlier elevated engine had locked to itself -# refused the settings copy, the engine started on stale files and exited in 6 s); old roots are small and left alone -$root = Join-Path $env:LOCALAPPDATA ('igneum-tune-' + (Get-Date -Format 'yyyyMMdd-HHmmss')) -$bin = Join-Path $root 'bin' -$sApp = Join-Path $root 'app' -$sLogs = Join-Path $root 'logs' -New-Item -ItemType Directory -Force -Path $root, $sApp, $sLogs | Out-Null -if (Test-Path $bin) { Remove-Item -LiteralPath $bin -Recurse -Force -ErrorAction SilentlyContinue } -Copy-Item -LiteralPath $installDir -Destination $bin -Recurse -Force -# the installed engine carries Ember Tune from 0.3.12 on: prefer it; the kit is for a PC still on an older app -$installedVer = (& (Join-Path $installDir 'igneum-app.exe') --version 2>&1 | Out-String).Trim() -$installedHasEmber = $false -if ($installedVer -match 'igneum-app (\d+)\.(\d+)\.(\d+)') { $installedHasEmber = ([int]$Matches[1] -gt 0) -or ([int]$Matches[2] -gt 3) -or (([int]$Matches[2] -eq 3) -and ([int]$Matches[3] -ge 12)) } -$ember = $null -# a kit is used when the installed app predates Ember Tune (0.3.13), or when a run carries fixes the installed app -# lacks (ember-kit-6: the cmd quote fix and the self-registering elevated engine of 6 October 2026); a kit engine -# registers the Power Helper on the INSTALLED exe's path (powertask::task_exe), never its own -$k7 = Join-Path $appDir 'jobs\ember-kit-7\igneum-app-ember.exe' -$k6 = Join-Path $appDir 'jobs\ember-kit-6\igneum-app-ember.exe' -if (Test-Path $k7) { $ember = $k7 } -elseif (Test-Path $k6) { $ember = $k6 } -elseif (-not $installedHasEmber) { foreach ($cand in @((Join-Path $appDir 'jobs\ember-kit-2\igneum-app-ember.exe'), (Join-Path $appDir 'jobs\ember-kit-1\igneum-app-ember.exe'))) { if (Test-Path $cand) { $ember = $cand; break } } } -if ($ember) { - Copy-Item -LiteralPath $ember -Destination (Join-Path $bin 'igneum-app.exe') -Force - Say ("engine: the Ember build from " + $ember) -} else { - Say ('engine: the installed one (' + $installedVer + $(if ($installedHasEmber) { ', carries Ember Tune' } else { '; no kit found: an older engine ignores the tune and reports no_rows' }) + ')') -} -$helper = $null -$found = Get-ChildItem -Path (Join-Path $appDir 'jobs') -Recurse -Filter 'igneum-gpu-telemetry.exe' -ErrorAction SilentlyContinue | Where-Object { $_.FullName -match 'amd-kit' } | Sort-Object LastWriteTime -Descending | Select-Object -First 1 -if ($found) { $helper = $found.FullName } -if ($helper) { - Copy-Item -LiteralPath $helper -Destination (Join-Path $bin 'igneum-gpu-telemetry.exe') -Force - Say ("helper: the Ember build of igneum-gpu-telemetry from " + $helper + " sha256=" + (Get-FileHash -LiteralPath $helper -Algorithm SHA256).Hash.ToLower()) -} else { Say 'helper: the installed igneum-gpu-telemetry (no jobs\amd-kit-1\kit\igneum-gpu-telemetry.exe); without --tune the AMD card measures only' } -$exe = Join-Path $bin 'igneum-app.exe' -$ver = (& $exe --version 2>&1 | Out-String).Trim() -Say ("engine: " + $exe + " (" + $ver + ")") -Write-Output ("RESULT TUNE engine " + $ver + " sha256=" + (Get-FileHash -LiteralPath $exe -Algorithm SHA256).Hash.ToLower()) -if ($ver -notmatch 'igneum-app (\d+)\.(\d+)\.(\d+)') { Write-Output 'RESULT TUNE error=version_unknown'; exit 2 } - -foreach ($f in @('settings.json', 'machine-id', 'wallet.json', 'tuning.json', 'firewall-rule.json')) { # the firewall flag too: an older kit then asks nothing - $src = Join-Path $appDir $f - if (Test-Path $src) { Copy-Item -LiteralPath $src -Destination (Join-Path $sApp $f) -Force } -} -# the copies are VERBATIM (run 4, 6 October 2026: a PowerShell ConvertFrom-Json | ConvertTo-Json round trip rewrote big -# integers as doubles, the engine read the file as defaults, no payout address, every card off, 96 old jobs run in -# the scratch root); the engine itself switches remote jobs, updates, proving and Power control off under --sweep -# (Settings::for_measurement) and makes every card due. Only a report line is read here. -$sj = Join-Path $sApp 'settings.json' -if (-not (Test-Path -LiteralPath $sj)) { Write-Output 'RESULT TUNE error=no_settings reason=the_installed_app_has_no_settings.json'; exit 2 } -$installedPowerControl = 'unknown'; $addr = ''; $ncards = 0 -try { $back = Get-Content -LiteralPath $sj -Raw | ConvertFrom-Json; $addr = [string]$back.address; if ($back.cards) { $ncards = @($back.cards.PSObject.Properties).Count }; if ($back.PSObject.Properties.Name -contains 'power_control') { $installedPowerControl = ([bool]$back.power_control).ToString().ToLower() } } catch { } -$bom = (Get-Content -LiteralPath $sj -Encoding Byte -TotalCount 3 -ErrorAction SilentlyContinue) -join ',' -Write-Output ('RESULT TUNE installed_power_control=' + $installedPowerControl + ' (the tune engine runs with it off: no prompt unless the job itself is elevated)') -Write-Output ('RESULT TUNE scratch settings (verbatim copy): address ' + $(if ($addr) { $addr.Substring(0, [Math]::Min(10, $addr.Length)) + '...' } else { 'EMPTY' }) + ', cards ' + $ncards + ', first bytes ' + $bom + ', ' + (Get-Item -LiteralPath $sj).Length + ' bytes') -if (-not $addr -and -not (Test-Path (Join-Path $sApp 'wallet.json'))) { Write-Output 'RESULT TUNE error=no_address reason=the_copied_settings_carry_no_payout_address_and_no_wallet.json'; exit 2 } -Remove-Item -LiteralPath (Join-Path $sApp 'app.url') -Force -ErrorAction SilentlyContinue - -# the state before, for the report -$smi = Join-Path $env:ProgramFiles 'NVIDIA Corporation\NVSMI\nvidia-smi.exe' -if (-not (Test-Path $smi)) { $smi = Join-Path $env:SystemRoot 'System32\nvidia-smi.exe' } -$tele = Join-Path $bin 'igneum-gpu-telemetry.exe' -function Snapshot([string] $tag) { - if (Test-Path $smi) { - $q = (& $smi --query-gpu=index,name,driver_version,power.draw,power.limit,power.default_limit,power.min_limit,power.max_limit,clocks.gr,clocks.max.gr,clocks.mem --format=csv,noheader 2>&1 | Out-String).Trim() - Write-Output ("RESULT TUNE " + $tag + " nvidia " + ($q -replace "`r?`n", ' | ')) - } - if (Test-Path $tele) { - $t = (& $tele --tune 2>&1 | Out-String).Trim() - Write-Output ("RESULT TUNE " + $tag + " amd " + ($t -replace "`r?`n", ' | ')) - } else { Write-Output ("RESULT TUNE " + $tag + " amd no_helper") } -} -Snapshot 'before' - -# the tune engine: status every 10 s (6 rate samples per 60 s hold) -$env:IGNEUM_APP_DATA = $root -$env:IGNEUM_APP_LOGS = $sLogs -$env:IGNEUM_APP_STATUS_SECS = '10' -$env:IGNEUM_APP_NO_OTA = '1' # C35: a second engine never runs the updater (the installer would quit the installed app) -# C35 (5 October 2026): the engine's output goes to a FILE, never a pipe. A pipe's write end is inherited by every -# process the engine starts (its miners and workers), so after an abort the installed app's jobs runner waits for an -# EOF that never comes and hangs in its own quit; and the engine's whole tree is killed at the end (nothing orphaned). -$outFile = Join-Path $root 'engine-stdout.log' -$errFile = Join-Path $root 'engine-stderr.log' -Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue -$p = Start-Process -FilePath $exe -ArgumentList '--sweep' -WorkingDirectory (Split-Path $exe) -WindowStyle Hidden -PassThru -RedirectStandardOutput $outFile -RedirectStandardError $errFile -Say ("engine started, pid " + $p.Id + ", data " + $root + ", stdout " + $outFile) -function EndTree([int] $procId, [string] $why) { - $before = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count - & taskkill /T /F /PID $procId 2>&1 | Out-Null - Start-Sleep -Seconds 2 - $after = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count - Write-Output ('RESULT ' + $resultTag + ' tree ended (' + $why + '): igneum processes ' + $before + ' -> ' + $after + ' (the installed app''s own miners are stopped and held by the job)') -} -$seen = 0 -$rows = 0 -# the installed app shows the tune instead of a bare "off" (the project lead, 6 October 2026): every `TUNE progress` line and the -# `TUNE chosen` line go to its /api/tune-progress (a POST of state, never quit, pause or resume: the 5 October rule) -$installedUrl = '' -try { $installedUrl = (Get-Content -LiteralPath (Join-Path $appDir 'app.url') -Raw -ErrorAction Stop).Trim() } catch { } -function Forward([string] $line) { - if (-not $installedUrl) { return } - $kv = @{} - foreach ($tok in ($line -split ' ')) { if ($tok -match '^([a-z_]+)=(.*)$') { $kv[$Matches[1]] = $Matches[2] } } - $body = @{ key = ($kv['card'] -replace '_', ' '); done = ($line -match '^TUNE chosen ') } - if ($kv['key']) { $body.key = ($kv['key'] -replace '_', ' ') } - foreach ($n in @('step', 'of', 'eta_s', 'mhs', 'watts', 'clock', 'cap')) { if ($kv[$n] -ne $null) { $body[$n] = [double]$kv[$n] } } - if ($kv['cap'] -ne $null) { $body['power_pct'] = [double]$kv['cap'] } - if ($kv['clock'] -ne $null) { $body['clock_mhz'] = [double]$kv['clock'] } - if ($kv['plan']) { $body.plan = $kv['plan'] } - if ($kv['note']) { $body.note = ($kv['note'] -replace '_', ' ') } - if ($body.done) { $body.note = ('tuned: ' + $kv['mhs'] + ' MH/s at ' + $kv['watts'] + ' W') } - try { Invoke-WebRequest -Uri ($installedUrl + 'api/tune-progress') -Method POST -Body ($body | ConvertTo-Json -Compress) -ContentType 'application/json' -UseBasicParsing -TimeoutSec 3 | Out-Null } catch { } -} -# the watchdog (coordinator, 6 October 2026): a tune engine that mines nothing for 120 s after its first status line -# (every card "waiting" or 0.00 MH/s: no payout address, no worker, no node) fails the job at once with the engine's -# last log line in the RESULT, its tree ended, mining restored by the job runner; a job that cannot mine never burns -# its budget silently again -$firstStatusAt = $null -$lastMining = $null -$idleSince = $null -$idleSamples = 0 -$lastIdleStatus = '' -$lastEngineLine = '' -function EngineLogDump([string] $why) { - Write-Output ('===== engine log tail (' + $why + ')') - $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 - if ($t) { Get-Content -LiteralPath $t.FullName -ErrorAction SilentlyContinue | Where-Object { $_ -notmatch 'status: accepted 0 blocks' } | Select-Object -Last 80 | ForEach-Object { Write-Output (' ' + $_) } } else { Write-Output ' (no app-*.log in the scratch logs folder)' } - if (Test-Path -LiteralPath $errFile) { Write-Output '===== engine stderr'; Get-Content -LiteralPath $errFile -ErrorAction SilentlyContinue | Select-Object -Last 20 | ForEach-Object { Write-Output (' ' + $_) } } -} -# the newest `status:` line among the engine log's last 60 lines (run 5, 15:39Z: a single sampled tail line missed the -# status lines between TUNE and telemetry lines for 300 s and the watchdog killed a live three-card tune) -function EngineStatus() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $ls = @(Get-Content -LiteralPath $t.FullName -Tail 60 -ErrorAction SilentlyContinue); for ($i = $ls.Count - 1; $i -ge 0; $i--) { if ([string]$ls[$i] -match ' status: ') { return [string]$ls[$i] } } }; return '' } -function EngineTail() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $l = Get-Content -LiteralPath $t.FullName -Tail 1 -ErrorAction SilentlyContinue; if ($l) { return [string]$l } }; return '' } -function Exit3([string] $why, [string] $reason) { - Write-Output ('RESULT TUNE error=' + $reason + ' last_status_line=' + ((EngineStatus) -replace '\s+', '_') + ' last_log_line=' + ($lastEngineLine -replace '\s+', '_')) - EngineLogDump $why - EndTree $p.Id $why - Snapshot 'after' - Write-Output 'RESULT TUNE error=no_rows' - exit 3 -} -# the AMD watts source (coordinator, 6 October 2026 16:00Z): the 3.0 coordinator's job read ZERO watts for the 9070 XT -# from igneum-gpu-telemetry.exe while the installed app had read 198.9 W the day before. The tune engine samples its -# own helper; this script also samples the installed app's /api/state power_w for the AMD card every 10 s (a GET, -# never a write) and, when the engine's AMD progress lines carry no watts, prints a fallback ladder from those -# samples (median per step) and names the source it used -$amdApp = @(); $amdStep = 0; $amdMhs = 0.0; $amdEngNZ = 0; $amdEngZ = 0; $tick = 0; $amdMhsByStep = @{} -while (-not $p.HasExited) { - Start-Sleep -Seconds 5 - $tick++ - if ($installedUrl -and ($tick % 2 -eq 0)) { - try { - $js = (Invoke-WebRequest -Uri ($installedUrl + 'api/state') -UseBasicParsing -TimeoutSec 3).Content | ConvertFrom-Json - foreach ($c in @($js.mining.cards)) { if ($c.vendor -eq 'amd') { $amdApp += ,@{ step = $amdStep; w = [double]$c.power_w } } } - } catch { } - } - $tailLine = EngineTail - if ($tailLine) { $lastEngineLine = $tailLine } - $statusLine = EngineStatus - if ($statusLine) { - if (-not $firstStatusAt) { $firstStatusAt = Get-Date } - $rate = 0.0; if ($statusLine -match '(\d+\.\d+) MH/s') { $rate = [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) } - # idle = three consecutive 10-s samples whose newest status line reads 0.00 MH/s (never a missing match) - if ($rate -gt 0) { $lastMining = Get-Date; $idleSamples = 0; $idleSince = $null } - elseif ($statusLine -ne $lastIdleStatus) { $lastIdleStatus = $statusLine; $idleSamples++; if (-not $idleSince) { $idleSince = Get-Date } } - } - if ($firstStatusAt -and -not $lastMining -and ((Get-Date) - $firstStatusAt).TotalSeconds -gt 120) { Exit3 'watchdog: not mining' 'not_mining reason=no_card_mined_within_120_s_of_the_first_status_line' } - if ($lastMining -and $idleSamples -ge 3 -and $idleSince -and ((Get-Date) - $idleSince).TotalSeconds -gt 300) { Exit3 'watchdog: stopped mining' 'stopped_mining reason=three_status_lines_read_0.00_MH/s_and_300_s_passed' } - $all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } - while ($seen -lt $all.Count) { - $l = [string]$all[$seen]; $seen++ - if ($l -match '^TUNE progress ') { - Forward $l - if ($l -match ' key=amd' -and $l -match ' step=(\d+)') { $amdStep = [int]$Matches[1] - if ($l -match ' mhs=([\d.]+)') { $amdMhsByStep[$amdStep] = [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) } - if ($l -match ' watts=([\d.]+)' -and [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) -gt 0) { $amdEngNZ++ } else { $amdEngZ++ } } - } - elseif ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ }; if ($l -match '^TUNE chosen ') { Forward $l } } - elseif ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l) } - elseif ($l -match '^(URL|STATE) ') { } - else { Say $l } - } - if ((Get-Date) -gt $deadline) { - Say ("budget of " + $budgetMinutes + " min spent; asking the tune engine to quit") - # C35 (5 October 2026): the only quit this script may send goes to the TUNE engine's own URL file in the scratch - # root, never to a file under the installed app's folder; the RESULT line names the file it used - $u = Join-Path $sApp 'app.url' - # the only URL file this script may quit is its own scratch root's; the installed app's folder is refused by name - if ((Resolve-Path -LiteralPath $u -ErrorAction SilentlyContinue).Path -like (Join-Path $appDir '*') -or $u -like '*\igneum\app\*') { - Write-Output ('RESULT TUNE quit refused: ' + $u + ' is the installed app''s URL file') - } elseif (Test-Path -LiteralPath $u) { - Write-Output ('RESULT TUNE quit asked of the tune engine through ' + $u + ' (pid ' + $p.Id + ')') - try { Invoke-WebRequest -Uri ((Get-Content -LiteralPath $u -Raw).Trim() + 'api/quit') -Method POST -Body '{}' -ContentType 'application/json' -UseBasicParsing -TimeoutSec 5 | Out-Null } catch { } - } else { Write-Output ('RESULT TUNE quit not sent: no URL file at ' + $u + '; killing pid ' + $p.Id) } - Start-Sleep -Seconds 20 - if (-not $p.HasExited) { EndTree $p.Id 'budget' } - EngineLogDump 'budget' - Write-Output 'RESULT TUNE error=budget_exceeded' - } -} -$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } -while ($seen -lt $all.Count) { $l = [string]$all[$seen]; $seen++; if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } } -$amdNZ = @($amdApp | Where-Object { $_.w -gt 0 }) -Write-Output ('RESULT TUNE amd_watts engine_progress nonzero=' + $amdEngNZ + ' zero=' + $amdEngZ + ' app_state samples=' + $amdApp.Count + ' nonzero=' + $amdNZ.Count) -$amdSource = 'none' -if ($amdEngNZ -gt 0) { $amdSource = 'engine_telemetry' } -elseif ($amdNZ.Count -gt 0) { - $amdSource = 'app_state_fallback' - foreach ($g in ($amdNZ | Group-Object { $_.step } | Sort-Object { [int]$_.Name })) { - $ws = @($g.Group | ForEach-Object { $_.w } | Sort-Object); $med = $ws[[int][math]::Floor($ws.Count / 2)] - $mh = 0.0; if ($amdMhsByStep.ContainsKey([int]$g.Name)) { $mh = $amdMhsByStep[[int]$g.Name] } - $eff = 0.0; if ($med -gt 0) { $eff = $mh / $med } - Write-Output ('RESULT TUNE amd_fallback source=app_state step=' + $g.Name + ' watts=' + ('{0:F1}' -f $med) + ' mhs=' + ('{0:F2}' -f $mh) + ' eff=' + ('{0:F4}' -f $eff) + ' samples=' + $ws.Count) - } -} -Write-Output ('RESULT TUNE amd_watts_source=' + $amdSource) -Say ("tune engine exited " + $p.ExitCode + " after " + [int]((Get-Date) - $started).TotalSeconds + " s, " + $rows + " table rows") -EndTree $p.Id 'end of run' -# the project lead, 6 October 2026, 07:25Z: both cards stay on their best MH/W points (the tune pins them); no factory reset here. -Snapshot 'after' -# the tune engine's own log: the TUNE lines and what happened around them -$log = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 -if ($log) { - Say ("engine log " + $log.FullName + ":") - Get-Content -LiteralPath $log.FullName | Where-Object { $_ -match 'TUNE|tune|power cap|GPUs:|worker ready|STATUS|exited|upload' } | Select-Object -Last 100 | ForEach-Object { Say (' ' + $_) } -} -if ($rows -eq 0) { Write-Output 'RESULT TUNE error=no_rows'; exit 1 } -exit 0