diff --git a/app/igneum-app/src/ember.rs b/app/igneum-app/src/ember.rs index 9a9f5a35..60b4325b 100644 --- a/app/igneum-app/src/ember.rs +++ b/app/igneum-app/src/ember.rs @@ -906,6 +906,28 @@ pub fn tuned_line(mhs: f64, watts: f64, eff: f64) -> String { format!("Tuned: {mhs:.1} MH/s at {watts:.0} W ({eff:.3} MH/W)") } +/// (D) May a step be SET on this card? Measure-only (no control: Apple, NVIDIA without Power control, an AMD card +/// whose probe gave no tune line) sets nothing: the run notices the missing acknowledgement and measures. An AMD +/// card also needs its ADLX ordinal. 6 October 2026, PC 1 17:57Z: a measure-only 9070 XT was sent a set, refused. +pub fn may_set(vendor: &str, tune_control: bool, amd_ordinal: i64) -> bool { + match vendor { + "nvidia" => tune_control, + "amd" => tune_control && amd_ordinal >= 0, + _ => false, + } +} + +/// (E) The row's words while a back-off from a failed tune holds: when the retry comes and why it stopped. +pub fn retry_note(remaining_s: u64, last_note: &str) -> String { + let mins = (remaining_s + 59) / 60; + let why = last_note.trim_start_matches("tuning stopped: ").trim_start_matches("tuning: ").trim(); + if why.is_empty() { + format!("tuning: retry in {mins} min") + } else { + format!("tuning: retry in {mins} min ({why})") + } +} + /// The row's line for a baseline (measure-only) result: the card was measured as it runs, nothing was set. /// (6 October 2026: PC 1's rows read "Tuned: 114.2 MH/s at 305 W" for a baseline, which is not a tune.) pub fn measured_line(mhs: f64, watts: f64, eff: f64) -> String { @@ -1007,6 +1029,18 @@ mod tests { assert!(Run::applied(&step, Readback { limit_w: 90.0, acked: false }, 100.0)); } + #[test] + fn a_measure_only_card_is_never_set_and_a_back_off_says_when_the_retry_comes() { + assert!(!may_set("amd", false, 1), "the probe gave no tune line: measure only"); + assert!(may_set("amd", true, 1)); + assert!(!may_set("amd", true, -1), "no ADLX ordinal"); + assert!(!may_set("nvidia", false, -1), "Power control off: measure only"); + assert!(may_set("nvidia", true, -1)); + assert!(!may_set("apple", true, -1)); + assert_eq!(retry_note(3599, "tuning stopped: the card refused the setting (see the log)"), "tuning: retry in 60 min (the card refused the setting (see the log))"); + assert_eq!(retry_note(30, ""), "tuning: retry in 1 min"); + } + #[test] fn a_baseline_result_reads_measured_and_a_tune_reads_tuned() { assert_eq!(result_line(PlanKind::Baseline, 114.2, 305.0, 0.3744), "Measured: 114.2 MH/s at 305 W (0.374 MH/W)"); diff --git a/app/igneum-app/src/engine.rs b/app/igneum-app/src/engine.rs index dd068802..e16c9ad7 100644 --- a/app/igneum-app/src/engine.rs +++ b/app/igneum-app/src/engine.rs @@ -2219,8 +2219,23 @@ impl Engine { if c.tune_control && c.vendor == "nvidia" && !power_control && !forced { // measure-only until Power control is on; the baseline plan covers it below } - if self.sweep_retry.get(&c.index).map(|t| now < *t).unwrap_or(false) { - continue; + // (E) 6 October 2026, PC 1 18:18Z: three "Tune now" requests sat behind the hour's back-off from the + // 17:55 failures and the rows said "queued" with no start; an explicit request now clears the back-off, + // and a held card's row says when the retry comes + if let Some(t) = self.sweep_retry.get(&c.index).copied() { + if now < t { + if forced { + self.sweep_retry.remove(&c.index); + } else { + if c.sweep_pct == 0 || c.sweep_note.starts_with("tuning stopped") { + let note = crate::ember::retry_note((t - now).as_secs(), &c.sweep_note); + if let Some(cc) = self.st().mining.cards.get_mut(c.index) { + cc.sweep_note = note; + } + } + continue; + } + } } let stable = self.miners.iter().find(|m| m.card == c.index).map(|m| m.proc.as_ref().map(|p| p.started.elapsed() >= Duration::from_secs(crate::ember::STABLE_S)).unwrap_or(false) && m.last_status.is_some()).unwrap_or(false); let wait = if c.state != "mining" { @@ -2569,21 +2584,24 @@ impl Engine { let cmd = format!("{seq}0 dev {dev}\n{seq}1 pl {w}\n{seq}2 {}\n{seq}3 {}\n", if clock > 0 { format!("lgc {clock}") } else { "rgc".to_string() }, if mem > 0 { format!("lmc {mem}") } else { "rmc".to_string() }); let dir = self.helper_cmd_dir(); let is_task = self.sweep_helper_is_task; - let _ = std::fs::create_dir_all(&dir); - let _ = std::fs::write(dir.join("cmd.txt"), cmd); - // 6 October 2026, PC 1 18:18Z: the task's helper had exited (20 idle minutes) while the engine's flag - // still said it ran, so the commands went to nobody and a blind 4-second sleep called them - // acknowledged. Now: the task is started again before every step (Start-ScheduledTask ignores a - // start while one runs), and the acknowledgement is the helper's own log line for this sequence - // (`3 nvidia-smi ...`), read for up to 15 s; a missing line is a failure with that reason - if is_task { - if let Err(e) = crate::powertask::start() { - self.shared.log(&format!("tune helper: the task could not be started ({e})")); - } - } + // 6 October 2026, PC 1: (B) the task's helper had exited while the engine's flag still said it ran, + // so commands went to nobody and a blind 4-second sleep called them acknowledged; (C) the first + // request of the first tune was written in the same second the helper started, and the helper + // skipped it as stale. Now, in the task case: the helper's heartbeat says whether it runs; the task + // is started and a fresh heartbeat awaited BEFORE the command is written; and the acknowledgement + // is the helper's own log line for this sequence (`3 nvidia-smi ...`), read for up to 15 s; + // a missing line is a failure with that reason let want = format!(" {seq}3 nvidia-smi"); let log_file = dir.join("helper.log"); std::thread::spawn(move || { + if is_task { + if let Err(e) = crate::powertask::ensure_running(&dir, Duration::from_secs(12)) { + shared.send(Cmd::TuneSet(seq, Err(e))); + return; + } + } + let _ = std::fs::create_dir_all(&dir); + let _ = std::fs::write(dir.join("cmd.txt"), cmd); let until = Instant::now() + Duration::from_secs(if is_task { 15 } else { 6 }); loop { std::thread::sleep(Duration::from_millis(500)); @@ -2601,7 +2619,10 @@ impl Engine { } "amd" => { let Some(exe) = self.bins.telemetry.clone() else { return }; - if c.amd_ordinal < 0 { + // (D) 6 October 2026, PC 1 17:57Z: a measure-only card (the probe gave no tune line) was still sent a + // set, which the helper refused and the tune stopped; measure only means nothing is set, the run + // notices the missing acknowledgement and measures + if !crate::ember::may_set("amd", c.tune_control, c.amd_ordinal) { return; } let n = c.amd_ordinal.to_string(); diff --git a/app/igneum-app/src/powertask.rs b/app/igneum-app/src/powertask.rs index 382f893d..3f22d237 100644 --- a/app/igneum-app/src/powertask.rs +++ b/app/igneum-app/src/powertask.rs @@ -35,6 +35,46 @@ use std::time::{Duration, Instant}; pub const TASK_NAME: &str = "Igneum Power Helper"; /// The helper ends after this long without a new command. pub const IDLE_S: u64 = 20 * 60; +/// The heartbeat file the helper refreshes every poll; an engine judges "the helper runs" by it, never by a flag +/// of its own (6 October 2026, PC 1: the flag said yes after the helper's idle exit, and commands went to nobody). +pub const ALIVE_FILE: &str = "helper.alive"; +/// A heartbeat older than this is a dead helper. +pub const ALIVE_MAX_S: u64 = 4; + +/// Writes the heartbeat: the unix time, as text. +pub fn beat(dir: &Path, now: u64) { + let _ = std::fs::write(dir.join(ALIVE_FILE), now.to_string()); +} + +/// Reads a heartbeat: is the helper alive at `now`? +pub fn alive_at(text: &str, now: u64) -> bool { + text.trim().parse::().map(|t| now.saturating_sub(t) <= ALIVE_MAX_S).unwrap_or(false) +} + +/// Is the helper that reads `dir` alive now? +pub fn alive(dir: &Path) -> bool { + std::fs::read_to_string(dir.join(ALIVE_FILE)).map(|t| alive_at(&t, crate::platform::unix_now())).unwrap_or(false) +} + +/// Starts the task unless the helper already runs, then waits for a fresh heartbeat (up to `wait`). The engine +/// writes a command only after this says Ok: the helper skips what the file held before its start (a stale quit +/// is not a command), so a command written before the start would be skipped with it (6 October 2026, 17:55:25Z +/// on PC 1: the first request of the first tune). +pub fn ensure_running(dir: &Path, wait: Duration) -> Result<(), String> { + if alive(dir) { + return Ok(()); + } + let _ = std::fs::create_dir_all(dir); + start()?; + let until = Instant::now() + wait; + while Instant::now() < until { + std::thread::sleep(Duration::from_millis(250)); + if alive(dir) { + return Ok(()); + } + } + Err(format!("the Igneum Power Helper task gave no heartbeat within {} s of its start", wait.as_secs())) +} /// One parsed command from cmd.txt. #[derive(Clone, Debug, PartialEq, Eq)] @@ -202,6 +242,7 @@ pub fn run_helper(dir: &Path) -> i32 { } }; log("helper started (scheduled task, elevated)"); + beat(dir, crate::platform::unix_now()); // a stale file from an earlier run is not a command: only lines ADDED after the start count. 6 October 2026, // PC 1 17:55:55Z: a helper that started after an engine had written `quit` read that line and exited in the // same second, and every later start did the same; so the lines present at the start are skipped whole @@ -224,6 +265,7 @@ pub fn run_helper(dir: &Path) -> i32 { match c { HelperCmd::Quit => { log("quit"); + let _ = std::fs::remove_file(dir.join(ALIVE_FILE)); return 0; } HelperCmd::Remove => { @@ -276,8 +318,10 @@ pub fn run_helper(dir: &Path) -> i32 { } if idle.elapsed() >= Duration::from_secs(IDLE_S) { log("idle 20 min: exit (the engine starts the task again when it needs it)"); + let _ = std::fs::remove_file(dir.join(ALIVE_FILE)); return 0; } + beat(dir, crate::platform::unix_now()); std::thread::sleep(Duration::from_millis(500)); } } @@ -315,6 +359,11 @@ mod tests { assert_eq!(commands_after("5 dev 0\n6 pl 460\nquit\n", 3), vec![]); assert_eq!(commands_after("5 dev 0\n6 pl 460\nquit\n7 dev 1\n", 3), vec![(7, HelperCmd::Dev("1".into()))]); assert_eq!(commands_after("quit\n", 3), vec![(0, HelperCmd::Quit)]); + // the heartbeat: fresh within ALIVE_MAX_S, dead after, dead when unreadable + assert!(alive_at("1791309325", 1791309325 + ALIVE_MAX_S)); + assert!(!alive_at("1791309325", 1791309325 + ALIVE_MAX_S + 1)); + assert!(!alive_at("", 1791309325)); + assert!(!alive_at("soon", 1791309325)); assert_eq!(parse_line("12 reregister C:\\evil.exe"), None, "no path argument: the helper picks the install folder itself"); assert_eq!(smi_args("0", &HelperCmd::Reregister), None); assert!(readback_command().contains("Actions[0].Execute")); diff --git a/docs/bench-log.md b/docs/bench-log.md index e459ad46..1c43e6fe 100644 --- a/docs/bench-log.md +++ b/docs/bench-log.md @@ -2268,3 +2268,12 @@ task now runs ...Programs\Igneum Miner\igneum-app.exe --power-helper"), then the task with no prompt ("-pl 460: set to 460.00 W from 575.00 W", "-pl 160: set to 160.00 W from 100.00 W"). Task Running, Highest, user Admin. Cards: 5090 221 W at 1845 MHz, 4070 75.8 W at 1860 MHz, 9070 XT 202 W, all mining through the installed app. Ember closed 16:53Z. + +## 6 October 2026, 18:18Z: the shipped 0.3.14 app asked to tune itself on PC 1 (ember-installed-pc1-1): FAILED, no tune ran + +Gate ok (0.3.14, Power control on, the task on the installed exe). Every card's "after" was its 17:55Z failure from +the app's own first tune: 5090 and 4070 "did not take within 30 s (card reports 460 / 160 W, acknowledged true)", +9070 XT "the card refused the setting"; the three "Tune now" requests sat queued behind the hour's back-off. Five +causes (A to E) and their fixes in the plan's 0.3.14 window section; the window repeats on 0.3.15. The cards kept +mining at the driver's held locks: 5090 222.9 W at 1845 MHz, 4070 75.9 W at 1860 MHz, 9070 XT 201 W. Lever-2 TUNING +records during the window: 5090 135.6 MH/s at 236 W (0.574 MH/W), 4070 31.1 MH/s at 79 W (0.393). diff --git a/docs/plans/ember-tune.md b/docs/plans/ember-tune.md index 50487f36..6e806324 100644 --- a/docs/plans/ember-tune.md +++ b/docs/plans/ember-tune.md @@ -296,8 +296,32 @@ record. Known in 0.3.13 on PC 1 until 0.3.14: the rows show the app's own earlie to receive the run's result), and the 80% caps are the app's setting; the draw is the tuned one regardless because the driver holds the clock locks and the caps do not bind. +### The 0.3.14 window (6 October 2026, 18:18 to 18:27Z): no tune ran; five causes + +The shipped 0.3.14 app, Power control on, the task registered, asked over its own API to tune each card +(`relay/playbooks/ember-installed-tune-pc1.ps1`). From its log and helper.log: + +| | When (UTC) | What | Cause | Fix (0.3.15) | +|---|---|---|---|---| +| C | 17:55:25 | the app's own first 5090 tune wrote its first request in the second the helper started; the helper skipped it as "present at my start"; 30 s later "did not take (card reports 460 W, acknowledged true)" | the engine wrote before the helper was up | the helper writes a heartbeat (`helper.alive`) every poll; the engine starts the task and waits for a fresh heartbeat BEFORE writing (`powertask::ensure_running`) | +| A | 17:55:55 | the 4070's helper start read the stale `quit` from the 5090's abort and exited in the same second | quit/remove matched before the sequence check | the helper acts only on lines added after its start (`commands_after`) | +| D | 17:57:27 | the 9070 XT, measure only ("igneum-gpu-telemetry gave no tune line": the 3.0 coordinator's job was on the ADLX helper), was still sent a set; refused; "tuning stopped" | a measure-only card was set | `ember::may_set`: measure only sets nothing | +| E | 18:18 to 18:27 | three "Tune now" requests sat "queued" behind the hour's back-off from the 17:55 failures; none started | `sweep_retry` blocked a forced request silently | a forced request clears the back-off; a held row says "tuning: retry in N min (why)" (`ember::retry_note`) | +| B | (latent) | the engine's helper flag stayed true after the helper's idle exit; a later step would have been acknowledged after a blind 4 s sleep | a flag instead of a fact | the heartbeat decides; the acknowledgement is the helper's own log line for the sequence (up to 15 s), a missing line fails the step with that reason | + +What a user on 0.3.14 sees: Power control on, no prompt, and no card tunes (the rows keep the baseline or +"tuning stopped"); the lever-2 TUNING records still flow. 0.3.15 closes it; the window repeats there. + ## 8a. Next-cut notes (for the 0.3.12 shipper) +For the 0.3.15 cut (6 October 2026 evening), on ember-tune: + +| Commit | What | Where | +|---|---|---| +| a8cab23 | (A) the helper acts only on lines added after its start; the acknowledgement is the helper's log line for the sequence | powertask.rs `commands_after` + test, engine.rs | +| (tip) | (B, C) the heartbeat, `ensure_running` before every task-path write; (D) `may_set`; (E) the back-off yields to a forced request, `retry_note`; the window playbook's gate at 0.3.15; `ember-tune-pc1.ps1` deleted (superseded by the installed-tune playbook) | powertask.rs, engine.rs, ember.rs + tests, relay/playbooks | +| e70ac5e | every task command goes to the helper's fixed folder | platform.rs, powertask.rs, engine.rs, main.rs + test | + For the 0.3.14 cut (6 October 2026, after run 6), on ember-tune: | Commit | What | Where | diff --git a/relay/playbooks/ember-installed-tune-pc1.ps1 b/relay/playbooks/ember-installed-tune-pc1.ps1 index 00da2310..1848b53b 100644 --- a/relay/playbooks/ember-installed-tune-pc1.ps1 +++ b/relay/playbooks/ember-installed-tune-pc1.ps1 @@ -17,9 +17,9 @@ $s = State if (-not $s) { Write-Output 'RESULT TUNE error=state_unreadable'; exit 1 } $ver = [string]$s.version # the first line: the version gate (main, 6 October 2026): 0.3.14 or later carries bd7fcf4 (AMD apply on ack) and -# 5165de7 (api/tune/goal with the climb); an older app aborts here before anything is asked of it -$vok = $false; try { $vok = ([version]($ver -replace '[^0-9.].*$', '')) -ge [version]'0.3.14' } catch { } -Write-Output ('RESULT TUNE version=' + $ver + ' gate=' + $(if ($vok) { 'ok (0.3.14 or later)' } else { 'FAILED (needs 0.3.14 with bd7fcf4 and 5165de7)' }) + ' climb_field=' + $(if ($null -ne $s.settings.tune_climb) { 'present' } else { 'absent' })) +# 5165de7 (api/tune/goal with the climb); 0.3.15 the helper fixes A to E (6 October 2026); an older app aborts here +$vok = $false; try { $vok = ([version]($ver -replace '[^0-9.].*$', '')) -ge [version]'0.3.15' } catch { } +Write-Output ('RESULT TUNE version=' + $ver + ' gate=' + $(if ($vok) { 'ok (0.3.15 or later)' } else { 'FAILED (needs 0.3.15: the helper fixes A to E)' }) + ' climb_field=' + $(if ($null -ne $s.settings.tune_climb) { 'present' } else { 'absent' })) if (-not $vok -or $null -eq $s.settings.tune_climb) { Write-Output 'RESULT TUNE error=version_gate'; exit 1 } Write-Output ('RESULT TUNE installed app ' + $ver + ' power_control=' + $s.settings.power_control + ' tuning_off=' + $s.settings.tuning_off + ' climb_before=' + $s.settings.tune_climb + ' goal_before=' + $s.settings.tune_goal) $t = Get-ScheduledTask -TaskName 'Igneum Power Helper' -ErrorAction SilentlyContinue diff --git a/relay/playbooks/ember-log-grep-pc1.ps1 b/relay/playbooks/ember-log-grep-pc1.ps1 index babe5739..118661bf 100644 --- a/relay/playbooks/ember-log-grep-pc1.ps1 +++ b/relay/playbooks/ember-log-grep-pc1.ps1 @@ -4,7 +4,7 @@ $ErrorActionPreference = 'Continue' $l = Get-ChildItem (Join-Path $env:LOCALAPPDATA 'igneum\logs') -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 if (-not $l) { Write-Output 'RESULT GREP no app log'; exit 1 } Write-Output ('RESULT GREP log=' + $l.FullName) -Get-Content -LiteralPath $l.FullName | Where-Object { ($_ -match '\] tune|^\d+ tune|tune helper|TUNE (start|aborted|chosen|card=|helper)|refused|did not take|power cap|Power Helper') -and ($_ -notmatch 'job ember-|RESULT ') } | Select-Object -Last 80 | ForEach-Object { Write-Output ('RESULT GREP ' + $_) } +Get-Content -LiteralPath $l.FullName | Where-Object { ($_ -match '\] tune|^\d+ tune|tune helper|TUNE (start|aborted|chosen|card=|helper|probe)|refused|did not take|power cap|Power Helper|probe|telemetry|gfx1201|queued|tune due|not due') -and ($_ -notmatch 'job ember-|RESULT ') } | Select-Object -Last 80 | ForEach-Object { Write-Output ('RESULT GREP ' + $_) } $h = Join-Path $env:LOCALAPPDATA 'igneum\app\sweep\helper.log' if (Test-Path -LiteralPath $h) { Get-Content -LiteralPath $h | Select-Object -Last 12 | ForEach-Object { Write-Output ('RESULT GREP helper ' + $_) } } $c = Join-Path $env:LOCALAPPDATA 'igneum\app\sweep\cmd.txt' diff --git a/relay/playbooks/ember-tune-pc1.ps1 b/relay/playbooks/ember-tune-pc1.ps1 deleted file mode 100644 index beab8e1e..00000000 --- a/relay/playbooks/ember-tune-pc1.ps1 +++ /dev/null @@ -1,265 +0,0 @@ -# Igneum run job: Ember Tune end to end on PC 1 (machine ae432dc7), unattended. 5 October 2026. -# Published as a `run` job with --stop-miners (docs/plans/ember-tune.md): the installed app stops its miners and -# holds them; this script takes the engine that carries src/ember.rs (the one the fetch job put in -# \jobs\ember-kit-1\igneum-app-ember.exe, else the installed one; the AMD helper with the --tune and --set -# commands from the telemetry agent's fetch job, \jobs\amd-kit-1\kit\igneum-gpu-telemetry.exe), copies the install folder to a scratch -# folder beside it, swaps the engine in, and starts that SECOND engine with `--sweep` in a scratch data folder (the -# real settings.json, machine-id and wallet.json copied in; remote jobs, auto-update and proving switched off -# there). That engine finds the installed app's node on 127.0.0.1:26610, mines on every card with the live program, -# tunes them one after the other (the full two-knob plan where the card can be controlled, the baseline measurement -# where it cannot: NVIDIA without administrator rights, Apple), prints every TUNE line on stdout, uploads its log -# (the TUNE {json} record reaches the intake) and quits. Every TUNE line is re-emitted as a RESULT line, so -# `node tools/jobs.mjs ` shows the table. Before and after, nvidia-smi's limits and clocks and the AMD -# helper's `--tune` lines are printed, so the restore can be read. The installed app's miners restart when the job -# ends. Not elevated: nothing asks for administrator rights (the project lead asleep, 5 October 2026); the NVIDIA card is -# therefore measure only tonight unless the engine finds itself elevated. -$ErrorActionPreference = 'Continue' -$resultTag = 'TUNE' -$budgetMinutes = 55 -if (-not ($budgetMinutes -is [int]) -or $budgetMinutes -lt 5) { $budgetMinutes = 35 } # a budget under 5 minutes is a bug, not a budget (C35) -$started = Get-Date -$deadline = $started.AddMinutes($budgetMinutes) -function Say([string] $m) { Write-Host ("[" + (Get-Date -Format 'HH:mm:ss') + "] " + $m) } - -# the installed engine: the per-user install (0.3.3+), else Program Files -$installDir = $null -foreach ($d in @((Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner'), (Join-Path $env:ProgramFiles 'Igneum Miner'))) { - if (Test-Path (Join-Path $d 'igneum-app.exe')) { $installDir = $d; break } -} -if (-not $installDir) { Write-Output 'RESULT TUNE error=no_engine reason=igneum-app.exe_not_found'; exit 2 } -$appData = $env:IGNEUM_APP_DATA -if (-not $appData) { $appData = Join-Path $env:LOCALAPPDATA 'igneum' } -$appDir = $env:IGNEUM_APP_DIR -if (-not $appDir) { $appDir = Join-Path $appData 'app' } - -# the scratch install: the whole folder (workers, node, helper, DLLs) with the Ember engine swapped in -# a FRESH scratch root per run (run 3, 6 October 2026, 11:45Z: the folder an earlier elevated engine had locked to itself -# refused the settings copy, the engine started on stale files and exited in 6 s); old roots are small and left alone -$root = Join-Path $env:LOCALAPPDATA ('igneum-tune-' + (Get-Date -Format 'yyyyMMdd-HHmmss')) -$bin = Join-Path $root 'bin' -$sApp = Join-Path $root 'app' -$sLogs = Join-Path $root 'logs' -New-Item -ItemType Directory -Force -Path $root, $sApp, $sLogs | Out-Null -if (Test-Path $bin) { Remove-Item -LiteralPath $bin -Recurse -Force -ErrorAction SilentlyContinue } -Copy-Item -LiteralPath $installDir -Destination $bin -Recurse -Force -# the installed engine carries Ember Tune from 0.3.12 on: prefer it; the kit is for a PC still on an older app -$installedVer = (& (Join-Path $installDir 'igneum-app.exe') --version 2>&1 | Out-String).Trim() -$installedHasEmber = $false -if ($installedVer -match 'igneum-app (\d+)\.(\d+)\.(\d+)') { $installedHasEmber = ([int]$Matches[1] -gt 0) -or ([int]$Matches[2] -gt 3) -or (([int]$Matches[2] -eq 3) -and ([int]$Matches[3] -ge 12)) } -$ember = $null -# a kit is used when the installed app predates Ember Tune (0.3.13), or when a run carries fixes the installed app -# lacks (ember-kit-6: the cmd quote fix and the self-registering elevated engine of 6 October 2026); a kit engine -# registers the Power Helper on the INSTALLED exe's path (powertask::task_exe), never its own -$k7 = Join-Path $appDir 'jobs\ember-kit-7\igneum-app-ember.exe' -$k6 = Join-Path $appDir 'jobs\ember-kit-6\igneum-app-ember.exe' -if (Test-Path $k7) { $ember = $k7 } -elseif (Test-Path $k6) { $ember = $k6 } -elseif (-not $installedHasEmber) { foreach ($cand in @((Join-Path $appDir 'jobs\ember-kit-2\igneum-app-ember.exe'), (Join-Path $appDir 'jobs\ember-kit-1\igneum-app-ember.exe'))) { if (Test-Path $cand) { $ember = $cand; break } } } -if ($ember) { - Copy-Item -LiteralPath $ember -Destination (Join-Path $bin 'igneum-app.exe') -Force - Say ("engine: the Ember build from " + $ember) -} else { - Say ('engine: the installed one (' + $installedVer + $(if ($installedHasEmber) { ', carries Ember Tune' } else { '; no kit found: an older engine ignores the tune and reports no_rows' }) + ')') -} -$helper = $null -$found = Get-ChildItem -Path (Join-Path $appDir 'jobs') -Recurse -Filter 'igneum-gpu-telemetry.exe' -ErrorAction SilentlyContinue | Where-Object { $_.FullName -match 'amd-kit' } | Sort-Object LastWriteTime -Descending | Select-Object -First 1 -if ($found) { $helper = $found.FullName } -if ($helper) { - Copy-Item -LiteralPath $helper -Destination (Join-Path $bin 'igneum-gpu-telemetry.exe') -Force - Say ("helper: the Ember build of igneum-gpu-telemetry from " + $helper + " sha256=" + (Get-FileHash -LiteralPath $helper -Algorithm SHA256).Hash.ToLower()) -} else { Say 'helper: the installed igneum-gpu-telemetry (no jobs\amd-kit-1\kit\igneum-gpu-telemetry.exe); without --tune the AMD card measures only' } -$exe = Join-Path $bin 'igneum-app.exe' -$ver = (& $exe --version 2>&1 | Out-String).Trim() -Say ("engine: " + $exe + " (" + $ver + ")") -Write-Output ("RESULT TUNE engine " + $ver + " sha256=" + (Get-FileHash -LiteralPath $exe -Algorithm SHA256).Hash.ToLower()) -if ($ver -notmatch 'igneum-app (\d+)\.(\d+)\.(\d+)') { Write-Output 'RESULT TUNE error=version_unknown'; exit 2 } - -foreach ($f in @('settings.json', 'machine-id', 'wallet.json', 'tuning.json', 'firewall-rule.json')) { # the firewall flag too: an older kit then asks nothing - $src = Join-Path $appDir $f - if (Test-Path $src) { Copy-Item -LiteralPath $src -Destination (Join-Path $sApp $f) -Force } -} -# the copies are VERBATIM (run 4, 6 October 2026: a PowerShell ConvertFrom-Json | ConvertTo-Json round trip rewrote big -# integers as doubles, the engine read the file as defaults, no payout address, every card off, 96 old jobs run in -# the scratch root); the engine itself switches remote jobs, updates, proving and Power control off under --sweep -# (Settings::for_measurement) and makes every card due. Only a report line is read here. -$sj = Join-Path $sApp 'settings.json' -if (-not (Test-Path -LiteralPath $sj)) { Write-Output 'RESULT TUNE error=no_settings reason=the_installed_app_has_no_settings.json'; exit 2 } -$installedPowerControl = 'unknown'; $addr = ''; $ncards = 0 -try { $back = Get-Content -LiteralPath $sj -Raw | ConvertFrom-Json; $addr = [string]$back.address; if ($back.cards) { $ncards = @($back.cards.PSObject.Properties).Count }; if ($back.PSObject.Properties.Name -contains 'power_control') { $installedPowerControl = ([bool]$back.power_control).ToString().ToLower() } } catch { } -$bom = (Get-Content -LiteralPath $sj -Encoding Byte -TotalCount 3 -ErrorAction SilentlyContinue) -join ',' -Write-Output ('RESULT TUNE installed_power_control=' + $installedPowerControl + ' (the tune engine runs with it off: no prompt unless the job itself is elevated)') -Write-Output ('RESULT TUNE scratch settings (verbatim copy): address ' + $(if ($addr) { $addr.Substring(0, [Math]::Min(10, $addr.Length)) + '...' } else { 'EMPTY' }) + ', cards ' + $ncards + ', first bytes ' + $bom + ', ' + (Get-Item -LiteralPath $sj).Length + ' bytes') -if (-not $addr -and -not (Test-Path (Join-Path $sApp 'wallet.json'))) { Write-Output 'RESULT TUNE error=no_address reason=the_copied_settings_carry_no_payout_address_and_no_wallet.json'; exit 2 } -Remove-Item -LiteralPath (Join-Path $sApp 'app.url') -Force -ErrorAction SilentlyContinue - -# the state before, for the report -$smi = Join-Path $env:ProgramFiles 'NVIDIA Corporation\NVSMI\nvidia-smi.exe' -if (-not (Test-Path $smi)) { $smi = Join-Path $env:SystemRoot 'System32\nvidia-smi.exe' } -$tele = Join-Path $bin 'igneum-gpu-telemetry.exe' -function Snapshot([string] $tag) { - if (Test-Path $smi) { - $q = (& $smi --query-gpu=index,name,driver_version,power.draw,power.limit,power.default_limit,power.min_limit,power.max_limit,clocks.gr,clocks.max.gr,clocks.mem --format=csv,noheader 2>&1 | Out-String).Trim() - Write-Output ("RESULT TUNE " + $tag + " nvidia " + ($q -replace "`r?`n", ' | ')) - } - if (Test-Path $tele) { - $t = (& $tele --tune 2>&1 | Out-String).Trim() - Write-Output ("RESULT TUNE " + $tag + " amd " + ($t -replace "`r?`n", ' | ')) - } else { Write-Output ("RESULT TUNE " + $tag + " amd no_helper") } -} -Snapshot 'before' - -# the tune engine: status every 10 s (6 rate samples per 60 s hold) -$env:IGNEUM_APP_DATA = $root -$env:IGNEUM_APP_LOGS = $sLogs -$env:IGNEUM_APP_STATUS_SECS = '10' -$env:IGNEUM_APP_NO_OTA = '1' # C35: a second engine never runs the updater (the installer would quit the installed app) -# C35 (5 October 2026): the engine's output goes to a FILE, never a pipe. A pipe's write end is inherited by every -# process the engine starts (its miners and workers), so after an abort the installed app's jobs runner waits for an -# EOF that never comes and hangs in its own quit; and the engine's whole tree is killed at the end (nothing orphaned). -$outFile = Join-Path $root 'engine-stdout.log' -$errFile = Join-Path $root 'engine-stderr.log' -Remove-Item -LiteralPath $outFile, $errFile -Force -ErrorAction SilentlyContinue -$p = Start-Process -FilePath $exe -ArgumentList '--sweep' -WorkingDirectory (Split-Path $exe) -WindowStyle Hidden -PassThru -RedirectStandardOutput $outFile -RedirectStandardError $errFile -Say ("engine started, pid " + $p.Id + ", data " + $root + ", stdout " + $outFile) -function EndTree([int] $procId, [string] $why) { - $before = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count - & taskkill /T /F /PID $procId 2>&1 | Out-Null - Start-Sleep -Seconds 2 - $after = @(Get-Process -Name 'igneum-app', 'igneum-miner', 'igneum-worker-cuda', 'igneum-worker-opencl', 'igneum-worker-metal' -ErrorAction SilentlyContinue).Count - Write-Output ('RESULT ' + $resultTag + ' tree ended (' + $why + '): igneum processes ' + $before + ' -> ' + $after + ' (the installed app''s own miners are stopped and held by the job)') -} -$seen = 0 -$rows = 0 -# the installed app shows the tune instead of a bare "off" (the project lead, 6 October 2026): every `TUNE progress` line and the -# `TUNE chosen` line go to its /api/tune-progress (a POST of state, never quit, pause or resume: the 5 October rule) -$installedUrl = '' -try { $installedUrl = (Get-Content -LiteralPath (Join-Path $appDir 'app.url') -Raw -ErrorAction Stop).Trim() } catch { } -function Forward([string] $line) { - if (-not $installedUrl) { return } - $kv = @{} - foreach ($tok in ($line -split ' ')) { if ($tok -match '^([a-z_]+)=(.*)$') { $kv[$Matches[1]] = $Matches[2] } } - $body = @{ key = ($kv['card'] -replace '_', ' '); done = ($line -match '^TUNE chosen ') } - if ($kv['key']) { $body.key = ($kv['key'] -replace '_', ' ') } - foreach ($n in @('step', 'of', 'eta_s', 'mhs', 'watts', 'clock', 'cap')) { if ($kv[$n] -ne $null) { $body[$n] = [double]$kv[$n] } } - if ($kv['cap'] -ne $null) { $body['power_pct'] = [double]$kv['cap'] } - if ($kv['clock'] -ne $null) { $body['clock_mhz'] = [double]$kv['clock'] } - if ($kv['plan']) { $body.plan = $kv['plan'] } - if ($kv['note']) { $body.note = ($kv['note'] -replace '_', ' ') } - if ($body.done) { $body.note = ('tuned: ' + $kv['mhs'] + ' MH/s at ' + $kv['watts'] + ' W') } - try { Invoke-WebRequest -Uri ($installedUrl + 'api/tune-progress') -Method POST -Body ($body | ConvertTo-Json -Compress) -ContentType 'application/json' -UseBasicParsing -TimeoutSec 3 | Out-Null } catch { } -} -# the watchdog (coordinator, 6 October 2026): a tune engine that mines nothing for 120 s after its first status line -# (every card "waiting" or 0.00 MH/s: no payout address, no worker, no node) fails the job at once with the engine's -# last log line in the RESULT, its tree ended, mining restored by the job runner; a job that cannot mine never burns -# its budget silently again -$firstStatusAt = $null -$lastMining = $null -$idleSince = $null -$idleSamples = 0 -$lastIdleStatus = '' -$lastEngineLine = '' -function EngineLogDump([string] $why) { - Write-Output ('===== engine log tail (' + $why + ')') - $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 - if ($t) { Get-Content -LiteralPath $t.FullName -ErrorAction SilentlyContinue | Where-Object { $_ -notmatch 'status: accepted 0 blocks' } | Select-Object -Last 80 | ForEach-Object { Write-Output (' ' + $_) } } else { Write-Output ' (no app-*.log in the scratch logs folder)' } - if (Test-Path -LiteralPath $errFile) { Write-Output '===== engine stderr'; Get-Content -LiteralPath $errFile -ErrorAction SilentlyContinue | Select-Object -Last 20 | ForEach-Object { Write-Output (' ' + $_) } } -} -# the newest `status:` line among the engine log's last 60 lines (run 5, 15:39Z: a single sampled tail line missed the -# status lines between TUNE and telemetry lines for 300 s and the watchdog killed a live three-card tune) -function EngineStatus() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $ls = @(Get-Content -LiteralPath $t.FullName -Tail 60 -ErrorAction SilentlyContinue); for ($i = $ls.Count - 1; $i -ge 0; $i--) { if ([string]$ls[$i] -match ' status: ') { return [string]$ls[$i] } } }; return '' } -function EngineTail() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $l = Get-Content -LiteralPath $t.FullName -Tail 1 -ErrorAction SilentlyContinue; if ($l) { return [string]$l } }; return '' } -function Exit3([string] $why, [string] $reason) { - Write-Output ('RESULT TUNE error=' + $reason + ' last_status_line=' + ((EngineStatus) -replace '\s+', '_') + ' last_log_line=' + ($lastEngineLine -replace '\s+', '_')) - EngineLogDump $why - EndTree $p.Id $why - Snapshot 'after' - Write-Output 'RESULT TUNE error=no_rows' - exit 3 -} -# the AMD watts source (coordinator, 6 October 2026 16:00Z): the 3.0 coordinator's job read ZERO watts for the 9070 XT -# from igneum-gpu-telemetry.exe while the installed app had read 198.9 W the day before. The tune engine samples its -# own helper; this script also samples the installed app's /api/state power_w for the AMD card every 10 s (a GET, -# never a write) and, when the engine's AMD progress lines carry no watts, prints a fallback ladder from those -# samples (median per step) and names the source it used -$amdApp = @(); $amdStep = 0; $amdMhs = 0.0; $amdEngNZ = 0; $amdEngZ = 0; $tick = 0; $amdMhsByStep = @{} -while (-not $p.HasExited) { - Start-Sleep -Seconds 5 - $tick++ - if ($installedUrl -and ($tick % 2 -eq 0)) { - try { - $js = (Invoke-WebRequest -Uri ($installedUrl + 'api/state') -UseBasicParsing -TimeoutSec 3).Content | ConvertFrom-Json - foreach ($c in @($js.mining.cards)) { if ($c.vendor -eq 'amd') { $amdApp += ,@{ step = $amdStep; w = [double]$c.power_w } } } - } catch { } - } - $tailLine = EngineTail - if ($tailLine) { $lastEngineLine = $tailLine } - $statusLine = EngineStatus - if ($statusLine) { - if (-not $firstStatusAt) { $firstStatusAt = Get-Date } - $rate = 0.0; if ($statusLine -match '(\d+\.\d+) MH/s') { $rate = [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) } - # idle = three consecutive 10-s samples whose newest status line reads 0.00 MH/s (never a missing match) - if ($rate -gt 0) { $lastMining = Get-Date; $idleSamples = 0; $idleSince = $null } - elseif ($statusLine -ne $lastIdleStatus) { $lastIdleStatus = $statusLine; $idleSamples++; if (-not $idleSince) { $idleSince = Get-Date } } - } - if ($firstStatusAt -and -not $lastMining -and ((Get-Date) - $firstStatusAt).TotalSeconds -gt 120) { Exit3 'watchdog: not mining' 'not_mining reason=no_card_mined_within_120_s_of_the_first_status_line' } - if ($lastMining -and $idleSamples -ge 3 -and $idleSince -and ((Get-Date) - $idleSince).TotalSeconds -gt 300) { Exit3 'watchdog: stopped mining' 'stopped_mining reason=three_status_lines_read_0.00_MH/s_and_300_s_passed' } - $all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } - while ($seen -lt $all.Count) { - $l = [string]$all[$seen]; $seen++ - if ($l -match '^TUNE progress ') { - Forward $l - if ($l -match ' key=amd' -and $l -match ' step=(\d+)') { $amdStep = [int]$Matches[1] - if ($l -match ' mhs=([\d.]+)') { $amdMhsByStep[$amdStep] = [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) } - if ($l -match ' watts=([\d.]+)' -and [double]::Parse($Matches[1], [Globalization.CultureInfo]::InvariantCulture) -gt 0) { $amdEngNZ++ } else { $amdEngZ++ } } - } - elseif ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ }; if ($l -match '^TUNE chosen ') { Forward $l } } - elseif ($l -match '^SWEEP ') { Write-Output ('RESULT ' + $l) } - elseif ($l -match '^(URL|STATE) ') { } - else { Say $l } - } - if ((Get-Date) -gt $deadline) { - Say ("budget of " + $budgetMinutes + " min spent; asking the tune engine to quit") - # C35 (5 October 2026): the only quit this script may send goes to the TUNE engine's own URL file in the scratch - # root, never to a file under the installed app's folder; the RESULT line names the file it used - $u = Join-Path $sApp 'app.url' - # the only URL file this script may quit is its own scratch root's; the installed app's folder is refused by name - if ((Resolve-Path -LiteralPath $u -ErrorAction SilentlyContinue).Path -like (Join-Path $appDir '*') -or $u -like '*\igneum\app\*') { - Write-Output ('RESULT TUNE quit refused: ' + $u + ' is the installed app''s URL file') - } elseif (Test-Path -LiteralPath $u) { - Write-Output ('RESULT TUNE quit asked of the tune engine through ' + $u + ' (pid ' + $p.Id + ')') - try { Invoke-WebRequest -Uri ((Get-Content -LiteralPath $u -Raw).Trim() + 'api/quit') -Method POST -Body '{}' -ContentType 'application/json' -UseBasicParsing -TimeoutSec 5 | Out-Null } catch { } - } else { Write-Output ('RESULT TUNE quit not sent: no URL file at ' + $u + '; killing pid ' + $p.Id) } - Start-Sleep -Seconds 20 - if (-not $p.HasExited) { EndTree $p.Id 'budget' } - EngineLogDump 'budget' - Write-Output 'RESULT TUNE error=budget_exceeded' - } -} -$all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } -while ($seen -lt $all.Count) { $l = [string]$all[$seen]; $seen++; if ($l -match '^TUNE ') { Write-Output ('RESULT ' + $l); if ($l -match '^TUNE card=') { $rows++ } } } -$amdNZ = @($amdApp | Where-Object { $_.w -gt 0 }) -Write-Output ('RESULT TUNE amd_watts engine_progress nonzero=' + $amdEngNZ + ' zero=' + $amdEngZ + ' app_state samples=' + $amdApp.Count + ' nonzero=' + $amdNZ.Count) -$amdSource = 'none' -if ($amdEngNZ -gt 0) { $amdSource = 'engine_telemetry' } -elseif ($amdNZ.Count -gt 0) { - $amdSource = 'app_state_fallback' - foreach ($g in ($amdNZ | Group-Object { $_.step } | Sort-Object { [int]$_.Name })) { - $ws = @($g.Group | ForEach-Object { $_.w } | Sort-Object); $med = $ws[[int][math]::Floor($ws.Count / 2)] - $mh = 0.0; if ($amdMhsByStep.ContainsKey([int]$g.Name)) { $mh = $amdMhsByStep[[int]$g.Name] } - $eff = 0.0; if ($med -gt 0) { $eff = $mh / $med } - Write-Output ('RESULT TUNE amd_fallback source=app_state step=' + $g.Name + ' watts=' + ('{0:F1}' -f $med) + ' mhs=' + ('{0:F2}' -f $mh) + ' eff=' + ('{0:F4}' -f $eff) + ' samples=' + $ws.Count) - } -} -Write-Output ('RESULT TUNE amd_watts_source=' + $amdSource) -Say ("tune engine exited " + $p.ExitCode + " after " + [int]((Get-Date) - $started).TotalSeconds + " s, " + $rows + " table rows") -EndTree $p.Id 'end of run' -# the project lead, 6 October 2026, 07:25Z: both cards stay on their best MH/W points (the tune pins them); no factory reset here. -Snapshot 'after' -# the tune engine's own log: the TUNE lines and what happened around them -$log = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 -if ($log) { - Say ("engine log " + $log.FullName + ":") - Get-Content -LiteralPath $log.FullName | Where-Object { $_ -match 'TUNE|tune|power cap|GPUs:|worker ready|STATUS|exited|upload' } | Select-Object -Last 100 | ForEach-Object { Say (' ' + $_) } -} -if ($rows -eq 0) { Write-Output 'RESULT TUNE error=no_rows'; exit 1 } -exit 0