diff --git a/docs/bench-log.md b/docs/bench-log.md index 819748536..0c618f9c4 100644 --- a/docs/bench-log.md +++ b/docs/bench-log.md @@ -1795,4 +1795,5 @@ Reading. Between 2^22 and 2^20 nothing moves: the card's time-slice scheduler al | Job 5 (`agg-cost-pc2-5`, 01:10:44Z) failed at PowerShell's parse in 1 s: `$RestoreIdentities:` inside a double-quoted string (a drive-qualified variable); no card or miner touched | `${RestoreIdentities}:`; the other `$name:` shapes are inside single-quoted bash here-strings | | Job 6's identities step found `settings.json` already at 8 identities under the active key `nvidia:0:NVIDIA GeForce RTX 5090` (a stale key `nvidia:NVIDIA GeForce RTX 5090` carries 2), so no change was sent; job 6's `/api/cards` with the 5090 disabled answered ok but the worker ran on for 120 s, `/api/pause` stopped it in 5 s, and at the end `/api/resume` brought it back in 5 s on 0.3.11 | the card switch keeps the pause as its fallback; the resume path works on 0.3.11 | | PC 2 ran three jobs at once from 01:24Z (`run-prover-on-pc2-20261006` at 01:24:21Z, the ledger suites build at 01:26:15Z, while agg-cost-pc2-6's closing report was still being uploaded): the app does not serialise jobs, "one job per machine at a time" holds only by the coordinator's word; job 6 had closed at 01:24:14Z, so its rows are clean | nothing of mine to fix; a rule for the job runner | +| The job scripts paused and resumed the installed app's miners through `/api/pause` and `/api/resume` (as the card-switch fallback, with a finally block); the coordinator's ruling of 6 October under the 5 October rule (a job never quits, pauses, resumes or restarts the installed app it did not start; `tools/ci/playbook-quit-check.sh`) forbids that even as a fallback | both scripts send no pause, resume or start any more; the job is published with `--stop-miners` (the runner stops the installed miners before the script and restarts them on any exit), the mining phases run the job's own miner from the app's install and `settings.json`, the idle and mining phases refuse to run while an installed miner is alive, the card switch stays for the identities step only; the gate passes both files | | The make-package gate ran the exporter's side files (`block-N.json.node-plan.json`) as fixtures and failed; its execute step took the exclusive `measure` lock for a cycle count and queued 25 min behind a packbench run | the glob skips `.node-plan.json`; the execute step runs under the `run` lock (a count, not a time) | diff --git a/tools/proving-v1/pc2-agg-cost-restore.ps1 b/tools/proving-v1/pc2-agg-cost-restore.ps1 index 9a735e591..d8f32fc4d 100644 --- a/tools/proving-v1/pc2-agg-cost-restore.ps1 +++ b/tools/proving-v1/pc2-agg-cost-restore.ps1 @@ -1,7 +1,9 @@ # Aggregation cost, the restore job (5 October 2026, night): what pc2-agg-cost.ps1's finally block does, for the case # where the app restarted under the job (the 0.3.10 rollout; a running job dies with the app and its finally block # never runs): stops any miner or CUDA worker the job started itself (a process whose parent is not igneum-app.exe), -# resumes the app's miners, switches the prover back on, resets the GPU compute-policy timeslice. Idempotent. +# kills a root GPU server and unlinks its socket, re-enables the NVIDIA card through /api/cards with the identities +# settings.json carries, switches the prover back on. It never pauses, resumes or starts the installed app's miners +# (the rule of 5 October 2026, 23:05 UTC; the job runner's --stop-miners restarts them on any exit). Idempotent. $ErrorActionPreference = 'Continue' function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') } $urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' } @@ -49,8 +51,6 @@ if ($c) { "RESULT card_on $(Stamp) $(Post '/api/cards' @{ cards = $list })" } "RESULT timeslice $(Stamp) $((& nvidia-smi compute-policy --set-timeslice=0 2>&1) -join ' ')" -"RESULT resume $(Stamp) $(Post '/api/resume' @{})" -"RESULT start $(Stamp) $(Post '/api/start' @{})" "RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})" Start-Sleep -Seconds 20 $st = State diff --git a/tools/proving-v1/pc2-agg-cost.ps1 b/tools/proving-v1/pc2-agg-cost.ps1 index 470199566..6568699a7 100644 --- a/tools/proving-v1/pc2-agg-cost.ps1 +++ b/tools/proving-v1/pc2-agg-cost.ps1 @@ -1,21 +1,24 @@ # Aggregation cost (5 October 2026, night; the project lead: "fix everything else in the numbers tonight"): what a per-block -# aggregation on PC 2's RTX 5090 spends and what each lever gives. A signed `run` job (shell powershell, not elevated). +# aggregation on PC 2's RTX 5090 spends and what each lever gives. A signed `run` job (shell powershell, not elevated), +# PUBLISHED WITH --stop-miners: the job runner stops the installed app's miners before the script and restarts them on +# any exit (the rule of 5 October 2026, 23:05 UTC, extended 6 October: a job never quits, pauses or resumes the +# installed app; tools/ci/playbook-quit-check.sh). The mining phases run the job's OWN miner (the app's binary and +# worker, the address and identities from settings.json) on the 5090, one at a time; the idle phases run with no miner. # The live prover is switched OFF for the run (its sp1-gpu-server would otherwise be shared, socket /tmp/sp1-cuda-0.sock) -# and ON again at the end; the app's miners are paused for the idle and own-miner phases and resumed at the end -# (try/finally). The live /opt/igneum host is untouched: this build lands in /opt/igneum-aggcost. +# and ON again at the end. The live /opt/igneum host is untouched: this build lands in /opt/igneum-aggcost. # Phases (the set at the top; every number is a RESULT line prefixed with the phase): -# A0 app miner mining, chain of 1 with RUST_LOG=info: the SP1 GPU server's own log lines (the profile) -# A app miner mining, chain of 4 (--save-shards), default knobs: tonight's baseline, shard proofs kept -# B0 app miner mining, aggregate-only over A's shard proofs, default knobs -# B app miner mining, aggregate-only, SP1_WORKER_VERIFY_INTERMEDIATES=false -# C miners paused: chain of 4, default knobs (the idle baseline tonight) -# C1 miners paused: aggregate-only over A's shard proofs -# C2 miners paused: block-344-shards4 (four full shards, one block, --save-shards): a 4-deferred aggregation idle +# A0 own miner (batch-log2 22) mining, chain of 1 with RUST_LOG=info: the SP1 GPU server's own log lines (the profile) +# A own miner mining, chain of 4 (--save-shards), default knobs: the baseline, shard proofs kept +# B0 own miner mining, aggregate-only over A's shard proofs, default knobs +# B own miner mining, aggregate-only, SP1_WORKER_VERIFY_INTERMEDIATES=false +# C no miner: chain of 4, default knobs (the idle baseline) +# C1 no miner: aggregate-only over A's shard proofs +# C2 no miner: block-344-shards4 (four full shards, one block, --save-shards): a 4-deferred aggregation idle # D own miner, the app's command line, batch-log2 22 (the worker's default): chain of 4; MH/s from its STATUS lines # E own miner at batch-log2 20, 18, 16: chain of 4 each; MH/s each # E0 own miner 22: aggregate-only over C2's four shard proofs (a 4-deferred aggregation under mining) # F own miner 22 with nvidia-smi compute-policy --set-timeslice=1 (SHORT), chain of 4; the policy restored after -# G0 miners paused: two host processes at once (chain of 2 each, disjoint blocks) on the one card +# G0 no miner: two host processes at once (chain of 2 each, disjoint blocks) on the one card # G own miner 22: two host processes at once (chain of 2 each, disjoint blocks) on the one GPU server # H own miner at $BestBatch with the best knobs: chain of 4 (the combination the plan adopts) $ErrorActionPreference = 'Continue' @@ -121,38 +124,25 @@ if ($st0 -and ($st0.mining.cards | Measure-Object).Count -eq 0) { "RESULT state_ "RESULT gpus $(Stamp) $(Smi)" $evmPort = $null foreach ($p in 26790, 26800, 26810) { if (Rpc $p 'igneum_getProvingStatus' @()) { $evmPort = $p; break } } -# the NVIDIA worker must be hashing (the app's own miner) before the mining phases mean anything -$waited = 0 -while ($waited -lt 180) { - $c = Card (State) - if ($c -and $c.hash_now -gt 10) { break } - if ($waited % 60 -eq 0) { "WAIT $(Stamp) nvidia worker state=$(if ($c) { $c.state } else { 'na' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' })" } - Start-Sleep -Seconds 15; $waited += 15 -} -$c = Card (State) -"RESULT miner $(Stamp) nvidia worker $(if ($c -and $c.hash_now -gt 10) { 'hashing' } else { 'NOT hashing: the app-miner phases are void' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' }) after $waited s" -# the app's miner command line, for the own-miner phases (the same binary, args, cwd and tuning file) -$mp = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' -and $_.CommandLine -like '*igneum-worker-cuda*' } | Select-Object -First 1 -$minerExe = $null; $minerArgs = $null -if ($mp) { - $minerExe = $mp.ExecutablePath - $cl = $mp.CommandLine - if ($cl -match '^\s*"[^"]*"\s*(.*)$') { $minerArgs = $Matches[1] } elseif ($cl -match '^\s*\S+\s+(.*)$') { $minerArgs = $Matches[1] } - $minerArgs = $minerArgs -replace '--status-secs \d+', '--status-secs 10' -} -if (-not $mp) { - # the 5090 miner is not running (job 2, 21:34Z: "off" after job 1's resume): the same command line as the app builds for - # it (engine.rs miner_args), synthesised from the iGPU miner's line when that one runs, else from the app's install - $amd = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' } | Select-Object -First 1 - $prog = Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner' - $cuda = Join-Path $prog 'igneum-worker-cuda.exe' - if ($amd -and $amd.CommandLine -match '--evm-address (0x[0-9a-fA-F]{40})') { - $evm = $Matches[1]; $minerExe = $amd.ExecutablePath - $rpc = if ($amd.CommandLine -match ' mine (\S+) ') { $Matches[1] } else { 'grpc://127.0.0.1:26610' } - $minerArgs = "mine $rpc 1 100000000 nvidia-$($st0.machine_id.Substring(0,8))-1 --worker `"$cuda`" --status-secs 10 --exit-on-seed-change --evm-address $evm --identities 8 --payout-label nvidia-$($st0.machine_id.Substring(0,8))-1 --prepare-packs packs\prepare --worker-args `"--device 0 --pack packs\devnet`"" - "RESULT miner_cmdline_synthesised $(Stamp) from the iGPU miner's line; exe=$minerExe cuda_worker_exists=$(Test-Path $cuda)" - } else { "RESULT miner_cmdline_synthesised $(Stamp) no running miner to copy from: the own-miner phases are void" } -} +# The installed app's miners were stopped by the job runner (--stop-miners) before this script; nothing here pauses, +# resumes or starts them. A miner still running means the job was published without --stop-miners: every mining +# phase is then refused (the double-mining guard in StartMiner), never run beside it. +$w0 = 0 +while ($w0 -lt 60 -and (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-miner*' -or $_.ProcessName -like 'igneum-worker*' } | Measure-Object).Count -gt 0) { Start-Sleep -Seconds 5; $w0 += 5 } +$installedAlive = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-miner*' -or $_.ProcessName -like 'igneum-worker*' } | Measure-Object).Count +"RESULT installed_miners $(Stamp) processes alive after $w0 s: $installedAlive $(if ($installedAlive -gt 0) { '(the job was published without --stop-miners: the mining phases are refused)' } else { '(stopped by the job runner)' })" +# the own miner's command line: what the app builds for the 5090 (engine.rs miner_args), from the app's install, the +# payout address in settings.json and the machine id (no running miner to copy under --stop-miners) +$prog = Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner' +$minerExe = Join-Path $prog 'igneum-miner.exe' +$cuda = Join-Path $prog 'igneum-worker-cuda.exe' +$settingsRaw = if (Test-Path (Join-Path $app 'settings.json')) { Get-Content (Join-Path $app 'settings.json') -Raw } else { '' } +$evm = if ($settingsRaw -match '"address":\s*"(0x[0-9a-fA-F]{40})"') { $Matches[1] } else { '' } +$id8 = if ($env:IGNEUM_MACHINE_ID) { $env:IGNEUM_MACHINE_ID.Substring(0, 8) } elseif ($settingsRaw -match '"machine_id":\s*"([0-9a-f]{8})') { $Matches[1] } else { '' } +$minerArgs = $null +if ((Test-Path $minerExe) -and (Test-Path $cuda) -and $evm -and $id8) { + $minerArgs = "mine grpc://127.0.0.1:26610 1 100000000 nvidia-$id8-1 --worker `"$cuda`" --status-secs 10 --exit-on-seed-change --evm-address $evm --identities 8 --payout-label nvidia-$id8-1 --prepare-packs packs\prepare --worker-args `"--device 0 --pack packs\devnet`"" +} else { "RESULT miner_cmdline_missing $(Stamp) exe=$(Test-Path $minerExe) worker=$(Test-Path $cuda) address=$($evm -ne '') machine_id=$($id8 -ne ''): the own-miner phases are void"; $minerExe = $null } $tuning = Join-Path $app 'tuning.json' if (Test-Path $tuning) { $env:IGNEUM_TUNING_FILE = $tuning } "RESULT miner_cmdline $(Stamp) exe=$minerExe tuning_file=$(Test-Path $tuning) args=$minerArgs" @@ -291,8 +281,8 @@ $script:miner = $null function StartMiner($label, $batchLog2) { $script:minerOk = $false if (-not $minerExe) { "RESULT $label own miner: no app miner command line was captured; phase void"; return } - $alive = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' } | Measure-Object).Count - if ($alive -gt 0) { "RESULT $label own miner: the app's CUDA worker is still running ($alive); the phase would mine twice on the card, so it is void (job 3, 22:48Z)"; return } + $alive = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' -or $_.ProcessName -like 'igneum-miner*' } | Measure-Object).Count + if ($alive -gt 0) { "RESULT $label own miner: an installed miner or CUDA worker is still running ($alive; publish with --stop-miners); the phase would mine twice on the card, so it is void (job 3, 22:48Z)"; return } $a = $minerArgs $extra = "--race off" if ($batchLog2 -and $batchLog2 -ne 22) { $extra = "$extra --batch-log2 $batchLog2" } @@ -329,12 +319,13 @@ function MinerRate($label, $skip) { $m = $h | Measure-Object -Average -Minimum -Maximum "n=$($h.Count) mean=$([math]::Round($m.Average,2)) min=$([math]::Round($m.Minimum,2)) max=$([math]::Round($m.Maximum,2)) MH/s wall (the now= field of the STATUS lines every 10 s, the first $skip skipped)" } -$paused = $false try { Wsl (Join-Path $job 'prep.sh') if (-not (Test-Path (Join-Path $job 'fixtures.txt'))) { "RESULT prep FAILED: no fixtures"; exit 1 } $list = Fixtures $first2 = ($list -split ',')[0..1] -join ','; $last2 = ($list -split ',')[2..3] -join ',' + $needOwnA = @('A0','A','B0','B') | Where-Object { $PhaseSet -contains $_ } + if ($needOwnA) { StartMiner 'A' 22; if (-not $script:minerOk) { "RESULT A phases refused $(Stamp) no own miner: A0, A, B0 and B are skipped"; $PhaseSet = @($PhaseSet | Where-Object { $_ -notin @('A0','A','B0','B') }) } } if ($PhaseSet -contains 'A0') { RunPhase 'A0' "SP1_PROVER=cuda RUST_LOG=info `$H --mode chain --chain $(($list -split ',')[0]) --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-A0.json" $errf = Join-Path $job 'host-A0.err' @@ -344,17 +335,9 @@ try { if ($PhaseSet -contains 'A') { RunPhase 'A' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --save-shards --out `$JOB/results-A.json" } if ($PhaseSet -contains 'B0') { RunPhase 'B0' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B0.json" } if ($PhaseSet -contains 'B') { RunPhase 'B' "SP1_WORKER_VERIFY_INTERMEDIATES=false SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B.json" } - $needIdle = @('C','C1','C2','G0','D','E','E0','F','G','H') | Where-Object { $PhaseSet -contains $_ } - if ($needIdle) { - # the 5090 switched off through /api/cards (and on again in the finally block): /api/pause then /api/resume left - # the worker off on 0.3.9 (21:25Z, an app defect in the resume path, fixed for 0.3.11) - "RESULT card_off $(Stamp) $(CardSwitch $false $null)"; $paused = $true - $w = 0; $left = 1 - while ($w -lt 120 -and $left -gt 0) { Start-Sleep -Seconds 5; $w += 5; $left = CudaWorkers } - if ($left -gt 0) { "RESULT card_off_failed $(Stamp) the app's CUDA worker is still running after $w s ($left); /api/pause as the fallback: $(Post '/api/pause' @{})"; $w2 = 0; while ($w2 -lt 60 -and (CudaWorkers) -gt 0) { Start-Sleep -Seconds 5; $w2 += 5 }; $left = CudaWorkers } - "RESULT paused $(Stamp) after $w s; CUDA workers left: $left; $(Smi)" - Start-Sleep -Seconds 5 - } + if ($needOwnA -and $script:miner) { "RESULT A miner rate $(MinerRate 'A' 2)"; StopMiner 'A' } + # the idle phases need the card to itself: the runner stopped the installed miners (checked above); nothing is paused here + if ((@('C','C1','C2','G0') | Where-Object { $PhaseSet -contains $_ }) -and (CudaWorkers) -gt 0) { "RESULT idle_refused $(Stamp) a CUDA worker is running (the job was published without --stop-miners): the idle phases C, C1, C2, G0 are skipped"; $PhaseSet = @($PhaseSet | Where-Object { $_ -notin @('C','C1','C2','G0') }) } if ($PhaseSet -contains 'C') { RunPhase 'C' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-C.json" } if ($PhaseSet -contains 'C1') { RunPhase 'C1' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-C1.json" } if ($PhaseSet -contains 'C2') { RunPhase 'C2' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain `$JOB/block-344-shards4.json --prover 0xCAfc6e74000000000000000000000000000000c2 --save-shards --out `$JOB/results-C2.json" } @@ -423,11 +406,7 @@ try { } finally { StopMiner 'final' & wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') } - if ($paused) { - "RESULT card_on $(Stamp) $(CardSwitch $true $null)"; "RESULT resume $(Stamp) $(Post '/api/resume' @{})" - $w3 = 0; while ($w3 -lt 90 -and (CudaWorkers) -eq 0) { Start-Sleep -Seconds 5; $w3 += 5 } - if ((CudaWorkers) -eq 0) { "RESULT miner_back_failed $(Stamp) no CUDA worker after $w3 s; /api/start: $(Post '/api/start' @{})" } else { "RESULT miner_back $(Stamp) CUDA worker running after $w3 s" } - } + # the installed app's miners come back through the job runner (--stop-miners restarts them on any exit), never from here "RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})" "RESULT end $(Stamp) $(Smi)" }