From 6de4827ccb8896ba129d0d653e6f5ea44b6bda84 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 08:03:13 +0000 Subject: [PATCH] a job that cannot mine never burns its budget silently: the elevated job path follows its output file while the script runs (the 5-minute progress reports carry the lines; 0.3.12), and the tune playbook's watchdog fails a run that mines nothing within 120 s of its first status line (the engine's last log line in the RESULT, the tree ended, mining restored by the runner) Co-Authored-By: Claude Fable 5.1 --- app/igneum-app/src/jobrun.rs | 44 ++++++++++++++++++++++++++++-- docs/plans/ember-tune.md | 8 ++++++ relay/playbooks/ember-tune-pc1.ps1 | 26 ++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/app/igneum-app/src/jobrun.rs b/app/igneum-app/src/jobrun.rs index bc833a21e..28db2c7da 100644 --- a/app/igneum-app/src/jobrun.rs +++ b/app/igneum-app/src/jobrun.rs @@ -1131,10 +1131,17 @@ fn run_script(shared: &Arc, job: &Job, sink: &Sink, jobs_dir: &Path, dat } cmd.current_dir(&dir); job_env(&mut cmd, shared, job, &dir, data_root); + // the elevated script's output reaches this side through a file: follow it while the script runs, so the + // 5-minute progress reports carry its lines (6 October 2026: a 35-minute run that never mined showed only + // "script running" until it ended; the lines that said why were in the file the whole time) + let follow = if elevated { Some(follow_file(sink, out_file.clone())) } else { None }; let ran = run_streamed(&mut cmd, sink, ctl, limit, shared, job, started, "script running")?; - if elevated { + if let Some(f) = follow { + f.stop.store(true, std::sync::atomic::Ordering::Relaxed); + let seen = f.handle.join().unwrap_or(0); + // the tail the follower had not read when the script ended if let Ok(t) = std::fs::read_to_string(&out_file) { - for l in t.lines() { + for l in t.lines().skip(seen) { sink.line(l); } } @@ -1142,6 +1149,39 @@ fn run_script(shared: &Arc, job: &Job, sink: &Sink, jobs_dir: &Path, dat finish_ran(ran, "script") } +/// Follows a file another process writes (the elevated script's output), feeding each new complete line to the +/// sink every 2 s until stopped; returns how many lines it delivered, so the caller can hand over the remainder. +struct Follow { + stop: Arc, + handle: std::thread::JoinHandle, +} + +fn follow_file(sink: &Sink, path: PathBuf) -> Follow { + let stop = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop2 = stop.clone(); + let s = Sink { shared: sink.shared.clone(), id: sink.id.clone(), dir: sink.dir.clone(), log_path: sink.log_path.clone(), file: Mutex::new(std::fs::OpenOptions::new().append(true).open(&sink.log_path).ok()), results: Mutex::new(Vec::new()) }; + let handle = std::thread::spawn(move || { + let mut seen = 0usize; + loop { + if let Ok(t) = std::fs::read_to_string(&path) { + let lines: Vec<&str> = t.lines().collect(); + // only complete lines (the writer may be mid-line): keep the last one for the next pass unless the + // text ends with a newline + let complete = if t.ends_with('\n') { lines.len() } else { lines.len().saturating_sub(1) }; + for l in lines.iter().take(complete).skip(seen) { + s.line(l); + } + seen = seen.max(complete); + } + if stop2.load(std::sync::atomic::Ordering::Relaxed) { + break seen; + } + std::thread::sleep(Duration::from_secs(2)); + } + }); + Follow { stop, handle } +} + fn finish_ran(ran: Ran, what: &str) -> Result { match ran.code { Some(0) => Ok(Done { status: "done".into(), exit: 0, summary: format!("{what} finished, exit 0"), extra: json!({}) }), diff --git a/docs/plans/ember-tune.md b/docs/plans/ember-tune.md index 6d38b142f..275e21442 100644 --- a/docs/plans/ember-tune.md +++ b/docs/plans/ember-tune.md @@ -161,6 +161,14 @@ maximum, 14,001 MHz memory; 9070 XT present on bus 98 with OFFSET ranges `gmax_r 10`). The offset finding changed the AMD mapping (054e041): an offset clock range closes the clock knob and the power ladder runs on a percent scale bounded by `plimit_range`. The re-run follows the 0.3.11 rollout. +## 8a. Next-cut notes (for the 0.3.12 shipper) + +| Commit | What | Where | +|---|---|---| +| b671c8b | every `quit:` names its source; Power control alone decides; no cap at start under `--sweep` | main.rs, server.rs, engine.rs (separable) | +| e600e63 | a second engine never runs the updater (`IGNEUM_APP_NO_OTA`, implied by `--sweep`) | engine.rs (6 lines, separable) | +| 6a8297c | the elevated job path's output file is followed while the script runs, so the 5-minute progress reports carry its lines (a 35-minute run that never mined showed only "script running" on 6 October 2026); the tune playbook's watchdog fails a run that mines nothing within 120 s of its first status line, with the engine's last log line in the RESULT | jobrun.rs `follow_file`, relay/playbooks/ember-tune-pc1.ps1 | + ## 9. Open - The NVIDIA clock readback: `nvidia-smi -lgc` is confirmed only through the core clock during the hold (a mean over diff --git a/relay/playbooks/ember-tune-pc1.ps1 b/relay/playbooks/ember-tune-pc1.ps1 index a50b018f3..f6ca8898d 100644 --- a/relay/playbooks/ember-tune-pc1.ps1 +++ b/relay/playbooks/ember-tune-pc1.ps1 @@ -132,8 +132,34 @@ function EndTree([int] $procId, [string] $why) { } $seen = 0 $rows = 0 +# the watchdog (coordinator, 6 October 2026): a tune engine that mines nothing for 120 s after its first status line +# (every card "waiting" or 0.00 MH/s: no payout address, no worker, no node) fails the job at once with the engine's +# last log line in the RESULT, its tree ended, mining restored by the job runner; a job that cannot mine never burns +# its budget silently again +$firstStatusAt = $null +$lastMining = $null +$lastEngineLine = '' +function EngineTail() { $t = Get-ChildItem -Path $sLogs -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1; if ($t) { $l = Get-Content -LiteralPath $t.FullName -Tail 1 -ErrorAction SilentlyContinue; if ($l) { return [string]$l } }; return '' } while (-not $p.HasExited) { Start-Sleep -Seconds 5 + $tailLine = EngineTail + if ($tailLine) { $lastEngineLine = $tailLine } + if ($lastEngineLine -match ' status: ') { + if (-not $firstStatusAt) { $firstStatusAt = Get-Date } + if ($lastEngineLine -match ', mining \|' -or ($lastEngineLine -match '(\d+\.\d+) MH/s' -and [double]$Matches[1] -gt 0)) { $lastMining = Get-Date } + } + if ($firstStatusAt -and -not $lastMining -and ((Get-Date) - $firstStatusAt).TotalSeconds -gt 120) { + Write-Output ('RESULT TUNE error=not_mining reason=no_card_mined_within_120_s_of_the_first_status_line last_log_line=' + ($lastEngineLine -replace '\s+', '_')) + EndTree $p.Id 'watchdog: not mining' + Write-Output 'RESULT TUNE error=no_rows' + exit 3 + } + if ($lastMining -and ((Get-Date) - $lastMining).TotalSeconds -gt 300) { + Write-Output ('RESULT TUNE error=stopped_mining reason=every_card_idle_for_300_s last_log_line=' + ($lastEngineLine -replace '\s+', '_')) + EndTree $p.Id 'watchdog: stopped mining' + Write-Output 'RESULT TUNE error=no_rows' + exit 3 + } $all = @(); if (Test-Path -LiteralPath $outFile) { $all = @(Get-Content -LiteralPath $outFile -ErrorAction SilentlyContinue) } while ($seen -lt $all.Count) { $l = [string]$all[$seen]; $seen++