From 2ca1c58a62bd1a8680930863024bc7a4fbd3aeca Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 15:09:16 +0000 Subject: [PATCH] pc2-agg-cost.ps1, pc2-agg-cost-restore.ps1: no pause, resume or start of the installed app's miners (the 6 October ruling under the 5 October rule); the job is published with --stop-miners, the mining phases run the job's own miner from the app's install and settings.json, the idle and mining phases refuse while an installed miner is alive, the card switch kept for the identities step; tools/ci/playbook-quit-check.sh passes both Co-Authored-By: Claude Fable 5.1 --- docs/bench-log.md | 1 + tools/proving-v1/pc2-agg-cost-restore.ps1 | 6 +- tools/proving-v1/pc2-agg-cost.ps1 | 103 +++++++++------------- 3 files changed, 45 insertions(+), 65 deletions(-) diff --git a/docs/bench-log.md b/docs/bench-log.md index 819748536..0c618f9c4 100644 --- a/docs/bench-log.md +++ b/docs/bench-log.md @@ -1795,4 +1795,5 @@ Reading. Between 2^22 and 2^20 nothing moves: the card's time-slice scheduler al | Job 5 (`agg-cost-pc2-5`, 01:10:44Z) failed at PowerShell's parse in 1 s: `$RestoreIdentities:` inside a double-quoted string (a drive-qualified variable); no card or miner touched | `${RestoreIdentities}:`; the other `$name:` shapes are inside single-quoted bash here-strings | | Job 6's identities step found `settings.json` already at 8 identities under the active key `nvidia:0:NVIDIA GeForce RTX 5090` (a stale key `nvidia:NVIDIA GeForce RTX 5090` carries 2), so no change was sent; job 6's `/api/cards` with the 5090 disabled answered ok but the worker ran on for 120 s, `/api/pause` stopped it in 5 s, and at the end `/api/resume` brought it back in 5 s on 0.3.11 | the card switch keeps the pause as its fallback; the resume path works on 0.3.11 | | PC 2 ran three jobs at once from 01:24Z (`run-prover-on-pc2-20261006` at 01:24:21Z, the ledger suites build at 01:26:15Z, while agg-cost-pc2-6's closing report was still being uploaded): the app does not serialise jobs, "one job per machine at a time" holds only by the coordinator's word; job 6 had closed at 01:24:14Z, so its rows are clean | nothing of mine to fix; a rule for the job runner | +| The job scripts paused and resumed the installed app's miners through `/api/pause` and `/api/resume` (as the card-switch fallback, with a finally block); the coordinator's ruling of 6 October under the 5 October rule (a job never quits, pauses, resumes or restarts the installed app it did not start; `tools/ci/playbook-quit-check.sh`) forbids that even as a fallback | both scripts send no pause, resume or start any more; the job is published with `--stop-miners` (the runner stops the installed miners before the script and restarts them on any exit), the mining phases run the job's own miner from the app's install and `settings.json`, the idle and mining phases refuse to run while an installed miner is alive, the card switch stays for the identities step only; the gate passes both files | | The make-package gate ran the exporter's side files (`block-N.json.node-plan.json`) as fixtures and failed; its execute step took the exclusive `measure` lock for a cycle count and queued 25 min behind a packbench run | the glob skips `.node-plan.json`; the execute step runs under the `run` lock (a count, not a time) | diff --git a/tools/proving-v1/pc2-agg-cost-restore.ps1 b/tools/proving-v1/pc2-agg-cost-restore.ps1 index 9a735e591..d8f32fc4d 100644 --- a/tools/proving-v1/pc2-agg-cost-restore.ps1 +++ b/tools/proving-v1/pc2-agg-cost-restore.ps1 @@ -1,7 +1,9 @@ # Aggregation cost, the restore job (5 October 2026, night): what pc2-agg-cost.ps1's finally block does, for the case # where the app restarted under the job (the 0.3.10 rollout; a running job dies with the app and its finally block # never runs): stops any miner or CUDA worker the job started itself (a process whose parent is not igneum-app.exe), -# resumes the app's miners, switches the prover back on, resets the GPU compute-policy timeslice. Idempotent. +# kills a root GPU server and unlinks its socket, re-enables the NVIDIA card through /api/cards with the identities +# settings.json carries, switches the prover back on. It never pauses, resumes or starts the installed app's miners +# (the rule of 5 October 2026, 23:05 UTC; the job runner's --stop-miners restarts them on any exit). Idempotent. $ErrorActionPreference = 'Continue' function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') } $urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' } @@ -49,8 +51,6 @@ if ($c) { "RESULT card_on $(Stamp) $(Post '/api/cards' @{ cards = $list })" } "RESULT timeslice $(Stamp) $((& nvidia-smi compute-policy --set-timeslice=0 2>&1) -join ' ')" -"RESULT resume $(Stamp) $(Post '/api/resume' @{})" -"RESULT start $(Stamp) $(Post '/api/start' @{})" "RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})" Start-Sleep -Seconds 20 $st = State diff --git a/tools/proving-v1/pc2-agg-cost.ps1 b/tools/proving-v1/pc2-agg-cost.ps1 index 470199566..6568699a7 100644 --- a/tools/proving-v1/pc2-agg-cost.ps1 +++ b/tools/proving-v1/pc2-agg-cost.ps1 @@ -1,21 +1,24 @@ # Aggregation cost (5 October 2026, night; the project lead: "fix everything else in the numbers tonight"): what a per-block -# aggregation on PC 2's RTX 5090 spends and what each lever gives. A signed `run` job (shell powershell, not elevated). +# aggregation on PC 2's RTX 5090 spends and what each lever gives. A signed `run` job (shell powershell, not elevated), +# PUBLISHED WITH --stop-miners: the job runner stops the installed app's miners before the script and restarts them on +# any exit (the rule of 5 October 2026, 23:05 UTC, extended 6 October: a job never quits, pauses or resumes the +# installed app; tools/ci/playbook-quit-check.sh). The mining phases run the job's OWN miner (the app's binary and +# worker, the address and identities from settings.json) on the 5090, one at a time; the idle phases run with no miner. # The live prover is switched OFF for the run (its sp1-gpu-server would otherwise be shared, socket /tmp/sp1-cuda-0.sock) -# and ON again at the end; the app's miners are paused for the idle and own-miner phases and resumed at the end -# (try/finally). The live /opt/igneum host is untouched: this build lands in /opt/igneum-aggcost. +# and ON again at the end. The live /opt/igneum host is untouched: this build lands in /opt/igneum-aggcost. # Phases (the set at the top; every number is a RESULT line prefixed with the phase): -# A0 app miner mining, chain of 1 with RUST_LOG=info: the SP1 GPU server's own log lines (the profile) -# A app miner mining, chain of 4 (--save-shards), default knobs: tonight's baseline, shard proofs kept -# B0 app miner mining, aggregate-only over A's shard proofs, default knobs -# B app miner mining, aggregate-only, SP1_WORKER_VERIFY_INTERMEDIATES=false -# C miners paused: chain of 4, default knobs (the idle baseline tonight) -# C1 miners paused: aggregate-only over A's shard proofs -# C2 miners paused: block-344-shards4 (four full shards, one block, --save-shards): a 4-deferred aggregation idle +# A0 own miner (batch-log2 22) mining, chain of 1 with RUST_LOG=info: the SP1 GPU server's own log lines (the profile) +# A own miner mining, chain of 4 (--save-shards), default knobs: the baseline, shard proofs kept +# B0 own miner mining, aggregate-only over A's shard proofs, default knobs +# B own miner mining, aggregate-only, SP1_WORKER_VERIFY_INTERMEDIATES=false +# C no miner: chain of 4, default knobs (the idle baseline) +# C1 no miner: aggregate-only over A's shard proofs +# C2 no miner: block-344-shards4 (four full shards, one block, --save-shards): a 4-deferred aggregation idle # D own miner, the app's command line, batch-log2 22 (the worker's default): chain of 4; MH/s from its STATUS lines # E own miner at batch-log2 20, 18, 16: chain of 4 each; MH/s each # E0 own miner 22: aggregate-only over C2's four shard proofs (a 4-deferred aggregation under mining) # F own miner 22 with nvidia-smi compute-policy --set-timeslice=1 (SHORT), chain of 4; the policy restored after -# G0 miners paused: two host processes at once (chain of 2 each, disjoint blocks) on the one card +# G0 no miner: two host processes at once (chain of 2 each, disjoint blocks) on the one card # G own miner 22: two host processes at once (chain of 2 each, disjoint blocks) on the one GPU server # H own miner at $BestBatch with the best knobs: chain of 4 (the combination the plan adopts) $ErrorActionPreference = 'Continue' @@ -121,38 +124,25 @@ if ($st0 -and ($st0.mining.cards | Measure-Object).Count -eq 0) { "RESULT state_ "RESULT gpus $(Stamp) $(Smi)" $evmPort = $null foreach ($p in 26790, 26800, 26810) { if (Rpc $p 'igneum_getProvingStatus' @()) { $evmPort = $p; break } } -# the NVIDIA worker must be hashing (the app's own miner) before the mining phases mean anything -$waited = 0 -while ($waited -lt 180) { - $c = Card (State) - if ($c -and $c.hash_now -gt 10) { break } - if ($waited % 60 -eq 0) { "WAIT $(Stamp) nvidia worker state=$(if ($c) { $c.state } else { 'na' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' })" } - Start-Sleep -Seconds 15; $waited += 15 -} -$c = Card (State) -"RESULT miner $(Stamp) nvidia worker $(if ($c -and $c.hash_now -gt 10) { 'hashing' } else { 'NOT hashing: the app-miner phases are void' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' }) after $waited s" -# the app's miner command line, for the own-miner phases (the same binary, args, cwd and tuning file) -$mp = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' -and $_.CommandLine -like '*igneum-worker-cuda*' } | Select-Object -First 1 -$minerExe = $null; $minerArgs = $null -if ($mp) { - $minerExe = $mp.ExecutablePath - $cl = $mp.CommandLine - if ($cl -match '^\s*"[^"]*"\s*(.*)$') { $minerArgs = $Matches[1] } elseif ($cl -match '^\s*\S+\s+(.*)$') { $minerArgs = $Matches[1] } - $minerArgs = $minerArgs -replace '--status-secs \d+', '--status-secs 10' -} -if (-not $mp) { - # the 5090 miner is not running (job 2, 21:34Z: "off" after job 1's resume): the same command line as the app builds for - # it (engine.rs miner_args), synthesised from the iGPU miner's line when that one runs, else from the app's install - $amd = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' } | Select-Object -First 1 - $prog = Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner' - $cuda = Join-Path $prog 'igneum-worker-cuda.exe' - if ($amd -and $amd.CommandLine -match '--evm-address (0x[0-9a-fA-F]{40})') { - $evm = $Matches[1]; $minerExe = $amd.ExecutablePath - $rpc = if ($amd.CommandLine -match ' mine (\S+) ') { $Matches[1] } else { 'grpc://127.0.0.1:26610' } - $minerArgs = "mine $rpc 1 100000000 nvidia-$($st0.machine_id.Substring(0,8))-1 --worker `"$cuda`" --status-secs 10 --exit-on-seed-change --evm-address $evm --identities 8 --payout-label nvidia-$($st0.machine_id.Substring(0,8))-1 --prepare-packs packs\prepare --worker-args `"--device 0 --pack packs\devnet`"" - "RESULT miner_cmdline_synthesised $(Stamp) from the iGPU miner's line; exe=$minerExe cuda_worker_exists=$(Test-Path $cuda)" - } else { "RESULT miner_cmdline_synthesised $(Stamp) no running miner to copy from: the own-miner phases are void" } -} +# The installed app's miners were stopped by the job runner (--stop-miners) before this script; nothing here pauses, +# resumes or starts them. A miner still running means the job was published without --stop-miners: every mining +# phase is then refused (the double-mining guard in StartMiner), never run beside it. +$w0 = 0 +while ($w0 -lt 60 -and (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-miner*' -or $_.ProcessName -like 'igneum-worker*' } | Measure-Object).Count -gt 0) { Start-Sleep -Seconds 5; $w0 += 5 } +$installedAlive = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-miner*' -or $_.ProcessName -like 'igneum-worker*' } | Measure-Object).Count +"RESULT installed_miners $(Stamp) processes alive after $w0 s: $installedAlive $(if ($installedAlive -gt 0) { '(the job was published without --stop-miners: the mining phases are refused)' } else { '(stopped by the job runner)' })" +# the own miner's command line: what the app builds for the 5090 (engine.rs miner_args), from the app's install, the +# payout address in settings.json and the machine id (no running miner to copy under --stop-miners) +$prog = Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner' +$minerExe = Join-Path $prog 'igneum-miner.exe' +$cuda = Join-Path $prog 'igneum-worker-cuda.exe' +$settingsRaw = if (Test-Path (Join-Path $app 'settings.json')) { Get-Content (Join-Path $app 'settings.json') -Raw } else { '' } +$evm = if ($settingsRaw -match '"address":\s*"(0x[0-9a-fA-F]{40})"') { $Matches[1] } else { '' } +$id8 = if ($env:IGNEUM_MACHINE_ID) { $env:IGNEUM_MACHINE_ID.Substring(0, 8) } elseif ($settingsRaw -match '"machine_id":\s*"([0-9a-f]{8})') { $Matches[1] } else { '' } +$minerArgs = $null +if ((Test-Path $minerExe) -and (Test-Path $cuda) -and $evm -and $id8) { + $minerArgs = "mine grpc://127.0.0.1:26610 1 100000000 nvidia-$id8-1 --worker `"$cuda`" --status-secs 10 --exit-on-seed-change --evm-address $evm --identities 8 --payout-label nvidia-$id8-1 --prepare-packs packs\prepare --worker-args `"--device 0 --pack packs\devnet`"" +} else { "RESULT miner_cmdline_missing $(Stamp) exe=$(Test-Path $minerExe) worker=$(Test-Path $cuda) address=$($evm -ne '') machine_id=$($id8 -ne ''): the own-miner phases are void"; $minerExe = $null } $tuning = Join-Path $app 'tuning.json' if (Test-Path $tuning) { $env:IGNEUM_TUNING_FILE = $tuning } "RESULT miner_cmdline $(Stamp) exe=$minerExe tuning_file=$(Test-Path $tuning) args=$minerArgs" @@ -291,8 +281,8 @@ $script:miner = $null function StartMiner($label, $batchLog2) { $script:minerOk = $false if (-not $minerExe) { "RESULT $label own miner: no app miner command line was captured; phase void"; return } - $alive = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' } | Measure-Object).Count - if ($alive -gt 0) { "RESULT $label own miner: the app's CUDA worker is still running ($alive); the phase would mine twice on the card, so it is void (job 3, 22:48Z)"; return } + $alive = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' -or $_.ProcessName -like 'igneum-miner*' } | Measure-Object).Count + if ($alive -gt 0) { "RESULT $label own miner: an installed miner or CUDA worker is still running ($alive; publish with --stop-miners); the phase would mine twice on the card, so it is void (job 3, 22:48Z)"; return } $a = $minerArgs $extra = "--race off" if ($batchLog2 -and $batchLog2 -ne 22) { $extra = "$extra --batch-log2 $batchLog2" } @@ -329,12 +319,13 @@ function MinerRate($label, $skip) { $m = $h | Measure-Object -Average -Minimum -Maximum "n=$($h.Count) mean=$([math]::Round($m.Average,2)) min=$([math]::Round($m.Minimum,2)) max=$([math]::Round($m.Maximum,2)) MH/s wall (the now= field of the STATUS lines every 10 s, the first $skip skipped)" } -$paused = $false try { Wsl (Join-Path $job 'prep.sh') if (-not (Test-Path (Join-Path $job 'fixtures.txt'))) { "RESULT prep FAILED: no fixtures"; exit 1 } $list = Fixtures $first2 = ($list -split ',')[0..1] -join ','; $last2 = ($list -split ',')[2..3] -join ',' + $needOwnA = @('A0','A','B0','B') | Where-Object { $PhaseSet -contains $_ } + if ($needOwnA) { StartMiner 'A' 22; if (-not $script:minerOk) { "RESULT A phases refused $(Stamp) no own miner: A0, A, B0 and B are skipped"; $PhaseSet = @($PhaseSet | Where-Object { $_ -notin @('A0','A','B0','B') }) } } if ($PhaseSet -contains 'A0') { RunPhase 'A0' "SP1_PROVER=cuda RUST_LOG=info `$H --mode chain --chain $(($list -split ',')[0]) --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-A0.json" $errf = Join-Path $job 'host-A0.err' @@ -344,17 +335,9 @@ try { if ($PhaseSet -contains 'A') { RunPhase 'A' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --save-shards --out `$JOB/results-A.json" } if ($PhaseSet -contains 'B0') { RunPhase 'B0' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B0.json" } if ($PhaseSet -contains 'B') { RunPhase 'B' "SP1_WORKER_VERIFY_INTERMEDIATES=false SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B.json" } - $needIdle = @('C','C1','C2','G0','D','E','E0','F','G','H') | Where-Object { $PhaseSet -contains $_ } - if ($needIdle) { - # the 5090 switched off through /api/cards (and on again in the finally block): /api/pause then /api/resume left - # the worker off on 0.3.9 (21:25Z, an app defect in the resume path, fixed for 0.3.11) - "RESULT card_off $(Stamp) $(CardSwitch $false $null)"; $paused = $true - $w = 0; $left = 1 - while ($w -lt 120 -and $left -gt 0) { Start-Sleep -Seconds 5; $w += 5; $left = CudaWorkers } - if ($left -gt 0) { "RESULT card_off_failed $(Stamp) the app's CUDA worker is still running after $w s ($left); /api/pause as the fallback: $(Post '/api/pause' @{})"; $w2 = 0; while ($w2 -lt 60 -and (CudaWorkers) -gt 0) { Start-Sleep -Seconds 5; $w2 += 5 }; $left = CudaWorkers } - "RESULT paused $(Stamp) after $w s; CUDA workers left: $left; $(Smi)" - Start-Sleep -Seconds 5 - } + if ($needOwnA -and $script:miner) { "RESULT A miner rate $(MinerRate 'A' 2)"; StopMiner 'A' } + # the idle phases need the card to itself: the runner stopped the installed miners (checked above); nothing is paused here + if ((@('C','C1','C2','G0') | Where-Object { $PhaseSet -contains $_ }) -and (CudaWorkers) -gt 0) { "RESULT idle_refused $(Stamp) a CUDA worker is running (the job was published without --stop-miners): the idle phases C, C1, C2, G0 are skipped"; $PhaseSet = @($PhaseSet | Where-Object { $_ -notin @('C','C1','C2','G0') }) } if ($PhaseSet -contains 'C') { RunPhase 'C' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-C.json" } if ($PhaseSet -contains 'C1') { RunPhase 'C1' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-C1.json" } if ($PhaseSet -contains 'C2') { RunPhase 'C2' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain `$JOB/block-344-shards4.json --prover 0xCAfc6e74000000000000000000000000000000c2 --save-shards --out `$JOB/results-C2.json" } @@ -423,11 +406,7 @@ try { } finally { StopMiner 'final' & wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') } - if ($paused) { - "RESULT card_on $(Stamp) $(CardSwitch $true $null)"; "RESULT resume $(Stamp) $(Post '/api/resume' @{})" - $w3 = 0; while ($w3 -lt 90 -and (CudaWorkers) -eq 0) { Start-Sleep -Seconds 5; $w3 += 5 } - if ((CudaWorkers) -eq 0) { "RESULT miner_back_failed $(Stamp) no CUDA worker after $w3 s; /api/start: $(Post '/api/start' @{})" } else { "RESULT miner_back $(Stamp) CUDA worker running after $w3 s" } - } + # the installed app's miners come back through the job runner (--stop-miners restarts them on any exit), never from here "RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})" "RESULT end $(Stamp) $(Smi)" }