433 lines
35 KiB
PowerShell
433 lines
35 KiB
PowerShell
# Aggregation cost (5 October 2026, night; the project lead: "fix everything else in the numbers tonight"): what a per-block
|
|
# aggregation on PC 2's RTX 5090 spends and what each lever gives. A signed `run` job (shell powershell, not elevated).
|
|
# The live prover is switched OFF for the run (its sp1-gpu-server would otherwise be shared, socket /tmp/sp1-cuda-0.sock)
|
|
# and ON again at the end; the app's miners are paused for the idle and own-miner phases and resumed at the end
|
|
# (try/finally). The live /opt/igneum host is untouched: this build lands in /opt/igneum-aggcost.
|
|
# Phases (the set at the top; every number is a RESULT line prefixed with the phase):
|
|
# A0 app miner mining, chain of 1 with RUST_LOG=info: the SP1 GPU server's own log lines (the profile)
|
|
# A app miner mining, chain of 4 (--save-shards), default knobs: tonight's baseline, shard proofs kept
|
|
# B0 app miner mining, aggregate-only over A's shard proofs, default knobs
|
|
# B app miner mining, aggregate-only, SP1_WORKER_VERIFY_INTERMEDIATES=false
|
|
# C miners paused: chain of 4, default knobs (the idle baseline tonight)
|
|
# C1 miners paused: aggregate-only over A's shard proofs
|
|
# C2 miners paused: block-344-shards4 (four full shards, one block, --save-shards): a 4-deferred aggregation idle
|
|
# D own miner, the app's command line, batch-log2 22 (the worker's default): chain of 4; MH/s from its STATUS lines
|
|
# E own miner at batch-log2 20, 18, 16: chain of 4 each; MH/s each
|
|
# E0 own miner 22: aggregate-only over C2's four shard proofs (a 4-deferred aggregation under mining)
|
|
# F own miner 22 with nvidia-smi compute-policy --set-timeslice=1 (SHORT), chain of 4; the policy restored after
|
|
# G0 miners paused: two host processes at once (chain of 2 each, disjoint blocks) on the one card
|
|
# G own miner 22: two host processes at once (chain of 2 each, disjoint blocks) on the one GPU server
|
|
# H own miner at $BestBatch with the best knobs: chain of 4 (the combination the plan adopts)
|
|
$ErrorActionPreference = 'Continue'
|
|
$PhaseSet = @('A0','A','B0','B','C','C1','C2','D','E')
|
|
$ShardSourceDir = '' # job 2: the WSL directory holding job 1's block-N-shard-i-compressed.bin and block-344 proofs ('' = this job's own)
|
|
$ShardSourceFirst = 0 # job 2: job 1's first block number (its four shard proofs are consecutive from it)
|
|
$ShardSourceParent = '' # job 2: the parent chain block hash of that first block
|
|
$BestBatch = 20
|
|
$BestKnobs = ''
|
|
function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') }
|
|
$urlFile = if ($env:IGNEUM_APP_DIR) { Join-Path $env:IGNEUM_APP_DIR 'app.url' } else { Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
if (-not (Test-Path $urlFile)) { $urlFile = Join-Path $env:LOCALAPPDATA 'igneum\app\app.url' }
|
|
$base = (Get-Content $urlFile -Raw).Trim().TrimEnd('/')
|
|
$app = if ($env:IGNEUM_APP_DIR) { $env:IGNEUM_APP_DIR } else { Join-Path $env:LOCALAPPDATA 'igneum\app' }
|
|
# The app's state as plain text parsed by regex (job 3, 22:41Z, app 0.3.10: Invoke-RestMethod's JSON conversion came back
|
|
# empty to PowerShell 5.1, so the card switch found no card and the own miner ran beside the app's). The card objects are
|
|
# serialised in struct order (src/state.rs CardState: key ... vendor ... enabled, state, hash_now), so the fields after a
|
|
# "vendor" belong to that card and the last "key" before it is its key.
|
|
$script:stateErr = ''
|
|
$script:nvidiaSeg = ''
|
|
function State {
|
|
try {
|
|
$raw = (Invoke-WebRequest -UseBasicParsing -Uri "$base/api/state" -TimeoutSec 20).Content
|
|
if (-not $raw) { $script:stateErr = 'empty body'; return $null }
|
|
$cards = @()
|
|
# one card = the text from its "index" key to the next card's (the fields come in any order inside it; the first
|
|
# version took the first "identities" AFTER "vendor", which belongs to the NEXT card: 23:16:53Z, the 5090 set to the
|
|
# iGPU's 2 identities)
|
|
$idx = @([regex]::Matches($raw, '\{"index":\d+,"key":"') | ForEach-Object { $_.Index })
|
|
for ($i = 0; $i -lt $idx.Count; $i++) {
|
|
$end = if ($i + 1 -lt $idx.Count) { $idx[$i + 1] } else { $raw.Length }
|
|
$seg = $raw.Substring($idx[$i], $end - $idx[$i])
|
|
$f = { param($re, $def) $x = [regex]::Match($seg, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
|
$v = (& $f '"vendor":"([^"]*)"' '')
|
|
if (-not $v) { continue }
|
|
if ($v -eq 'nvidia' -and -not $script:nvidiaSeg) { $script:nvidiaSeg = $seg.Substring(0, [Math]::Min(700, $seg.Length)) }
|
|
$cards += [pscustomobject]@{ key = (& $f '"key":"([^"]*)"' ''); name = (& $f '"name":"([^"]*)"' ''); vendor = $v; enabled = ((& $f '"enabled":(true|false)' 'true') -eq 'true'); identities = [int](& $f '"identities":(\d+)' '1'); state = (& $f '"state":"([^"]*)"' ''); hash_now = [double](& $f '"hash_now":([\d.eE+-]+)' '0') }
|
|
}
|
|
$g = { param($re, $def) $x = [regex]::Match($raw, $re); if ($x.Success) { $x.Groups[1].Value } else { $def } }
|
|
$script:stateErr = ''
|
|
return [pscustomobject]@{
|
|
version = (& $g '"version":"([^"]*)"' ''); machine_id = (& $g '"machine_id":"([^"]*)"' '')
|
|
settings = [pscustomobject]@{ prove = ((& $g '"settings":\{[^}]*?"prove":(true|false)' 'false') -eq 'true') }
|
|
mining = [pscustomobject]@{ paused = ((& $g '"mining":\{[^}]*?"paused":(true|false)' 'false') -eq 'true'); cards = $cards }
|
|
proving = [pscustomobject]@{ status = (& $g '"proving":\{[^}]*?"status":"([^"]*)"' '') }
|
|
raw_len = $raw.Length
|
|
}
|
|
} catch { $script:stateErr = "$_"; $null }
|
|
}
|
|
function Post($path, $body) { try { (Invoke-RestMethod -Method Post -Uri "$base$path" -ContentType 'application/json' -Body ($body | ConvertTo-Json -Compress -Depth 5) -TimeoutSec 15) | ConvertTo-Json -Compress } catch { "error: $_" } }
|
|
function Rpc($port, $method, $params) {
|
|
try { (Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$port" -ContentType 'application/json' -Body (@{jsonrpc='2.0'; id=1; method=$method; params=$params} | ConvertTo-Json -Compress -Depth 6) -TimeoutSec 180).result } catch { $null }
|
|
}
|
|
function Card($st) { if ($null -eq $st) { return $null }; $st.mining.cards | Where-Object { $_.vendor -eq 'nvidia' } | Select-Object -First 1 }
|
|
function Smi { try { (& nvidia-smi --query-gpu=index,name,memory.used,utilization.gpu,power.draw --format=csv,noheader,nounits 2>$null) -join ' | ' } catch { 'nvidia-smi failed' } }
|
|
# The card keys come from the app's own settings.json (`cards`: key -> {enabled, identities, ...}), never from /api/state:
|
|
# that reply is "{}" whenever the state fails to serialise (job 4, 00:18Z: raw_len=2; the app's state_json falls back
|
|
# to an empty object), which no parser can read. The state is still read for the hash rate when it answers.
|
|
function CardPrefs {
|
|
$f = Join-Path $app 'settings.json'
|
|
if (-not (Test-Path $f)) { return @{} }
|
|
$raw = Get-Content $f -Raw
|
|
$prefs = @{}
|
|
$cm = [regex]::Match($raw, '"cards":\s*\{')
|
|
if (-not $cm.Success) { return $prefs }
|
|
$i = $cm.Index + $cm.Length; $depth = 1; $start = $i
|
|
while ($i -lt $raw.Length -and $depth -gt 0) { $ch = $raw[$i]; if ($ch -eq '{') { $depth++ } elseif ($ch -eq '}') { $depth-- }; $i++ }
|
|
$body = $raw.Substring($start, $i - $start - 1)
|
|
foreach ($m in [regex]::Matches($body, '"([^"]+)":\s*\{([^}]*)\}')) {
|
|
$k = $m.Groups[1].Value; $v = $m.Groups[2].Value
|
|
$en = [regex]::Match($v, '"enabled":\s*(true|false)'); $id = [regex]::Match($v, '"identities":\s*(\d+)')
|
|
$prefs[$k] = @{ enabled = $(if ($en.Success) { $en.Groups[1].Value -eq 'true' } else { $true }); identities = $(if ($id.Success) { [int]$id.Groups[1].Value } else { 1 }) }
|
|
}
|
|
return $prefs
|
|
}
|
|
function NvidiaKey { $p = CardPrefs; ($p.Keys | Where-Object { $_ -like 'nvidia:*' } | Select-Object -First 1) }
|
|
function CardSwitch($on, $identities) {
|
|
$p = CardPrefs; $nk = NvidiaKey
|
|
if (-not $nk) { return "no nvidia card key in settings.json (keys: $($p.Keys -join ','))" }
|
|
$list = @(); foreach ($k in $p.Keys) { $list += @{ key = $k; enabled = $(if ($k -eq $nk) { [bool]$on } else { [bool]$p[$k].enabled }); identities = $(if ($k -eq $nk -and $identities) { [int]$identities } else { [int]$p[$k].identities }) } }
|
|
"prefs_before=$(($p.Keys | ForEach-Object { "$($_):enabled=$($p[$_].enabled):identities=$($p[$_].identities)" }) -join ',') " + (Post '/api/cards' @{ cards = $list })
|
|
}
|
|
function CudaWorkers { (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' } | Measure-Object).Count }
|
|
$job = $env:IGNEUM_JOB_DIR
|
|
if (-not $job) { $job = Join-Path $env:TEMP 'igneum-agg-cost' }
|
|
New-Item -ItemType Directory -Force -Path $job | Out-Null
|
|
$jobStart = Get-Date
|
|
$st0 = State
|
|
# the 5090 back to its 8 identities (the restore job of 23:16:53Z set it to 2 through the parser fault above)
|
|
$RestoreIdentities = 8
|
|
$p0 = CardPrefs; $nk0 = NvidiaKey
|
|
"RESULT card_prefs $(Stamp) settings.json cards: $(($p0.Keys | ForEach-Object { "$($_):enabled=$($p0[$_].enabled):identities=$($p0[$_].identities)" }) -join ',') nvidia_key='$nk0'"
|
|
if (-not $nk0) { "RESULT refused $(Stamp) no nvidia card key in settings.json: nothing is paused and nothing runs"; exit 1 }
|
|
if ($p0[$nk0].identities -ne $RestoreIdentities) {
|
|
"RESULT identities_set $(Stamp) $nk0 $($p0[$nk0].identities) -> ${RestoreIdentities}: $(CardSwitch ([bool]$p0[$nk0].enabled) $RestoreIdentities)"
|
|
Start-Sleep -Seconds 5; $p1 = CardPrefs
|
|
"RESULT identities_after $(Stamp) settings.json says identities=$($p1[$nk0].identities) enabled=$($p1[$nk0].enabled)"
|
|
}
|
|
"RESULT state_read $(Stamp) ok=$($null -ne $st0) raw_len=$($st0.raw_len) cards=$(($st0.mining.cards | Measure-Object).Count) err=$script:stateErr"
|
|
"RESULT nvidia_card_json $(Stamp) $script:nvidiaSeg"
|
|
if ($st0 -and ($st0.mining.cards | Measure-Object).Count -eq 0) { "RESULT state_head $(Stamp) no card parsed; the body starts: $((Invoke-WebRequest -UseBasicParsing -Uri "$base/api/state" -TimeoutSec 20).Content.Substring(0, 1500))" }
|
|
"RESULT start $(Stamp) app $($st0.version) machine $($st0.machine_id) prove_setting=$($st0.settings.prove) proving=$($st0.proving.status) paused=$($st0.mining.paused) phases=$($PhaseSet -join ',') shard_source='$ShardSourceDir' $ShardSourceFirst $ShardSourceParent"
|
|
"RESULT gpus $(Stamp) $(Smi)"
|
|
$evmPort = $null
|
|
foreach ($p in 26790, 26800, 26810) { if (Rpc $p 'igneum_getProvingStatus' @()) { $evmPort = $p; break } }
|
|
# the NVIDIA worker must be hashing (the app's own miner) before the mining phases mean anything
|
|
$waited = 0
|
|
while ($waited -lt 180) {
|
|
$c = Card (State)
|
|
if ($c -and $c.hash_now -gt 10) { break }
|
|
if ($waited % 60 -eq 0) { "WAIT $(Stamp) nvidia worker state=$(if ($c) { $c.state } else { 'na' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' })" }
|
|
Start-Sleep -Seconds 15; $waited += 15
|
|
}
|
|
$c = Card (State)
|
|
"RESULT miner $(Stamp) nvidia worker $(if ($c -and $c.hash_now -gt 10) { 'hashing' } else { 'NOT hashing: the app-miner phases are void' }) hash_now=$(if ($c) { $c.hash_now } else { 'na' }) after $waited s"
|
|
# the app's miner command line, for the own-miner phases (the same binary, args, cwd and tuning file)
|
|
$mp = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' -and $_.CommandLine -like '*igneum-worker-cuda*' } | Select-Object -First 1
|
|
$minerExe = $null; $minerArgs = $null
|
|
if ($mp) {
|
|
$minerExe = $mp.ExecutablePath
|
|
$cl = $mp.CommandLine
|
|
if ($cl -match '^\s*"[^"]*"\s*(.*)$') { $minerArgs = $Matches[1] } elseif ($cl -match '^\s*\S+\s+(.*)$') { $minerArgs = $Matches[1] }
|
|
$minerArgs = $minerArgs -replace '--status-secs \d+', '--status-secs 10'
|
|
}
|
|
if (-not $mp) {
|
|
# the 5090 miner is not running (job 2, 21:34Z: "off" after job 1's resume): the same command line as the app builds for
|
|
# it (engine.rs miner_args), synthesised from the iGPU miner's line when that one runs, else from the app's install
|
|
$amd = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' } | Select-Object -First 1
|
|
$prog = Join-Path $env:LOCALAPPDATA 'Programs\Igneum Miner'
|
|
$cuda = Join-Path $prog 'igneum-worker-cuda.exe'
|
|
if ($amd -and $amd.CommandLine -match '--evm-address (0x[0-9a-fA-F]{40})') {
|
|
$evm = $Matches[1]; $minerExe = $amd.ExecutablePath
|
|
$rpc = if ($amd.CommandLine -match ' mine (\S+) ') { $Matches[1] } else { 'grpc://127.0.0.1:26610' }
|
|
$minerArgs = "mine $rpc 1 100000000 nvidia-$($st0.machine_id.Substring(0,8))-1 --worker `"$cuda`" --status-secs 10 --exit-on-seed-change --evm-address $evm --identities 8 --payout-label nvidia-$($st0.machine_id.Substring(0,8))-1 --prepare-packs packs\prepare --worker-args `"--device 0 --pack packs\devnet`""
|
|
"RESULT miner_cmdline_synthesised $(Stamp) from the iGPU miner's line; exe=$minerExe cuda_worker_exists=$(Test-Path $cuda)"
|
|
} else { "RESULT miner_cmdline_synthesised $(Stamp) no running miner to copy from: the own-miner phases are void" }
|
|
}
|
|
$tuning = Join-Path $app 'tuning.json'
|
|
if (Test-Path $tuning) { $env:IGNEUM_TUNING_FILE = $tuning }
|
|
"RESULT miner_cmdline $(Stamp) exe=$minerExe tuning_file=$(Test-Path $tuning) args=$minerArgs"
|
|
# strays from an earlier run that died with an app restart (job 3, 23:03Z): a miner or worker whose parent is not the
|
|
# app (nor a miner) is ours and is stopped before anything is measured
|
|
$killed = 0
|
|
foreach ($p in (Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -or $_.Name -like 'igneum-worker*' })) {
|
|
$parent = Get-CimInstance Win32_Process -Filter "ProcessId = $($p.ParentProcessId)" -ErrorAction SilentlyContinue
|
|
if ((-not $parent) -or (($parent.Name -notlike 'igneum-app*') -and ($parent.Name -notlike 'igneum-miner*'))) { "RESULT stray $(Stamp) pid $($p.ProcessId) $($p.Name) parent=$($p.ParentProcessId): stopped"; Stop-Process -Id $p.ProcessId -Force -ErrorAction SilentlyContinue; $killed++ }
|
|
}
|
|
"RESULT strays_stopped $(Stamp) $killed"
|
|
# the live prover off for the whole run: its sp1-gpu-server would be shared with ours (same socket, its environment)
|
|
"RESULT prove_off $(Stamp) $(Post '/api/prove' @{on=$false})"
|
|
# WSL side: the package, the export, the fixtures, the phase runner
|
|
$pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-aggcost\igneum-prove-wsl2'
|
|
if (-not (Test-Path $pkg)) { $pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-aggcost' }
|
|
"RESULT package $(Stamp) $pkg exists=$(Test-Path (Join-Path $pkg 'package'))"
|
|
function WslPath($p) { $w = (& wsl.exe -d Ubuntu-24.04 -u root -- wslpath -a ($p -replace '\\', '/') 2>$null); if ($w) { ($w -replace "`0", '').Trim() } else { '/mnt/c' + ($p.Substring(2) -replace '\\', '/') } }
|
|
$pkgW = WslPath $pkg; $jobW = WslPath $job
|
|
$srcW = if ($ShardSourceDir) { $ShardSourceDir.TrimEnd('/') } else { $jobW }
|
|
$tipHex = Rpc $evmPort 'eth_blockNumber' @()
|
|
$tip = [Convert]::ToInt64($tipHex, 16)
|
|
$first = $tip - 30; $last = $first + 3
|
|
"RESULT chain $(Stamp) node_evm_port=$evmPort tip=$tip blocks $first..$last"
|
|
$t = Get-Date
|
|
$body = (@{jsonrpc='2.0'; id=1; method='igneum_exportSegments'; params=@('0x0', ('0x{0:x}' -f $last))} | ConvertTo-Json -Compress)
|
|
$seqFile = Join-Path $job 'seq.json'; $bodyFile = Join-Path $job 'export-request.json'
|
|
[IO.File]::WriteAllText($bodyFile, $body, (New-Object System.Text.UTF8Encoding $false))
|
|
& curl.exe -s -S -m 600 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bodyFile" -o $seqFile 2>&1 | ForEach-Object { "curl: $_" }
|
|
if (-not (Test-Path $seqFile) -or (Get-Item $seqFile).Length -lt 1000) { "RESULT export FAILED: no reply file"; exit 1 }
|
|
"RESULT export $(Stamp) $((Get-Item $seqFile).Length) bytes in $([math]::Round(((Get-Date) - $t).TotalSeconds,1)) s"
|
|
$vars = @"
|
|
export PKG='$pkgW'
|
|
export JOB='$jobW'
|
|
export SRC='$srcW'
|
|
export FIRST=$first
|
|
export LAST=$last
|
|
"@
|
|
[IO.File]::WriteAllText((Join-Path $job 'vars.sh'), ($vars -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
|
$prep = @'
|
|
set -uo pipefail
|
|
. "$(dirname "$0")/vars.sh"
|
|
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
|
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}"
|
|
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
DEST="$HOME/igneum-prove-aggcost"; LIVE_TARGET="$HOME/igneum-prove/proving/igneum-prove/target"
|
|
mkdir -p "$DEST"
|
|
rsync -a --delete --exclude target "$PKG/package/" "$DEST/"
|
|
find "$DEST" -name target -prune -o -type f -exec touch {} + 2>/dev/null
|
|
grep -o '"program_id": "0x[0-9a-f]*"' "$DEST/proving/igneum-prove/elf/manifest.json" | sed 's/^/RESULT manifest /'
|
|
cd "$DEST/proving/igneum-prove"
|
|
t0=$(date +%s)
|
|
if ! CARGO_TARGET_DIR="$LIVE_TARGET" cargo build --release -p igneum-prove-export -p igneum-prove-host --features igneum-prove-host/cuda 2>&1 | tail -3; then echo "RESULT build FAILED"; exit 1; fi
|
|
echo "RESULT build $(stamp) exit 0 in $(( $(date +%s) - t0 )) s"
|
|
mkdir -p /opt/igneum-aggcost && cp "$LIVE_TARGET/release/igneum-prove-host" "$LIVE_TARGET/release/igneum-prove-export" /opt/igneum-aggcost/
|
|
H=/opt/igneum-aggcost/igneum-prove-host; X=/opt/igneum-aggcost/igneum-prove-export
|
|
echo "RESULT installed $(sha256sum $H | cut -c1-16) host, $(sha256sum $X | cut -c1-16) export; live /opt/igneum untouched: $(sha256sum /opt/igneum/igneum-prove-host | cut -c1-16)"
|
|
$H --mode id | sed 's/^/RESULT aggcost-host /'
|
|
python3 -c "import json,sys; d=json.load(open('$JOB/seq.json')); json.dump(d['result'], open('$JOB/export.json','w'))"
|
|
LIST=""
|
|
for n in $(seq $FIRST $LAST); do
|
|
if ! $X "$JOB/export.json" $n "$JOB/block-$n.json" --source "PC 2 live devnet export, aggregation cost run" 2>&1 | tail -1 | sed "s/^/export $n: /"; then echo "RESULT cut $n FAILED"; exit 1; fi
|
|
$H "$JOB/block-$n.json" --mode native 2>&1 | grep -E "^RESULT (native|plan)" | sed "s/^/block $n /"
|
|
LIST="$LIST${LIST:+,}$JOB/block-$n.json"
|
|
done
|
|
echo "$LIST" > "$JOB/fixtures.txt"
|
|
cp "$DEST/proving/fixtures/block-344-shards4.json" "$JOB/"
|
|
# the parent hash of every fixture, for --mode aggregate
|
|
python3 - <<PY
|
|
import json
|
|
for n in range($FIRST, $LAST + 1):
|
|
f = json.load(open('$JOB/block-%d.json' % n))
|
|
print('RESULT parent', n, f['block']['env']['parent_hash'] if 'parent_hash' in f['block']['env'] else f['block']['env'].get('parentHash'))
|
|
open('$JOB/parent-%d.txt' % n, 'w').write(f['block']['env'].get('parent_hash') or f['block']['env'].get('parentHash'))
|
|
f = json.load(open('$JOB/block-344-shards4.json'))
|
|
open('$JOB/parent-344.txt', 'w').write(f['block']['env'].get('parent_hash') or f['block']['env'].get('parentHash'))
|
|
PY
|
|
echo "RESULT fixtures $(stamp) $LIST"
|
|
# the live prover's GPU server must be gone (its environment and socket would be ours otherwise); and this job runs as
|
|
# root, so the socket it leaves would be root-owned and the app's prover (another WSL user) could not connect (the
|
|
# 21:25Z fault of job 1): the server is killed and the socket removed here and again at the end (the rule of
|
|
# tools/ci/prover-socket-check.sh)
|
|
for i in $(seq 1 36); do pgrep -x sp1-gpu-server >/dev/null || break; sleep 5; done
|
|
pkill -f sp1-gpu-server 2>/dev/null; rm -f /tmp/sp1-cuda-*.sock # the root-socket class: the CI check (prover-socket-check.sh) wants pkill -f
|
|
echo "RESULT gpu_server_before $(stamp) running=$(pgrep -x sp1-gpu-server | wc -l) socket=$(ls /tmp/sp1-cuda-0.sock 2>/dev/null || echo none)"
|
|
'@
|
|
$phase = @'
|
|
# bash phase.sh <label> <cmd file>: runs the host command in the file (one line, env assignments first) with a 1-s
|
|
# nvidia-smi sampler underneath; every RESULT/STAGE line comes out prefixed "<label>:"; a gpu line closes the phase.
|
|
set -uo pipefail
|
|
. "$(dirname "$0")/vars.sh"
|
|
export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH"
|
|
CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}"
|
|
stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
L="$1"; CMD="$2"
|
|
H=/opt/igneum-aggcost/igneum-prove-host
|
|
export H
|
|
nvidia-smi --query-gpu=timestamp,index,memory.used,utilization.gpu,power.draw --format=csv,noheader,nounits -l 1 > "$JOB/smi-$L.csv" 2>/dev/null &
|
|
SMI=$!
|
|
t0=$(date +%s.%N); START=$(stamp)
|
|
echo "RESULT phase $L start $START cmd: $(cat "$CMD" | cut -c1-400)"
|
|
bash "$CMD" > "$JOB/host-$L.log" 2> "$JOB/host-$L.err"
|
|
RC=$?
|
|
WALL=$(python3 -c "import time; print(round(time.time() - $t0, 1))")
|
|
kill $SMI 2>/dev/null; sleep 1
|
|
grep -E "^(RESULT|STAGE|igneum-prove-host sources|segment proof written)" "$JOB/host-$L.log" | sed "s/^/$L: /"
|
|
awk -F', *' '{ if ($3+0 > max) max=$3+0; u+=$4; n++ } END { if (n) printf "RESULT phase_gpu LABEL samples=%d memory_used_max_mib=%d util_mean_pct=%.1f\n", n, max, u/n }' "$JOB/smi-$L.csv" | sed "s/LABEL/$L/"
|
|
echo "RESULT phase $L end $(stamp) exit $RC wall $WALL s (started $START); stderr $(wc -l < "$JOB/host-$L.err") lines"
|
|
if [ $RC -ne 0 ]; then tail -5 "$JOB/host-$L.err" | sed "s/^/$L stderr: /"; fi
|
|
'@
|
|
foreach ($pair in @(@('prep.sh', $prep), @('phase.sh', $phase))) {
|
|
[IO.File]::WriteAllText((Join-Path $job $pair[0]), ($pair[1] -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false))
|
|
}
|
|
function Wsl($file) { & wsl.exe -d Ubuntu-24.04 -u root -- bash (WslPath $file) 2>&1 | ForEach-Object { ($_ -replace "`0", '') } }
|
|
function Cmd($label, $line) {
|
|
$f = Join-Path $job "cmd-$label.sh"
|
|
[IO.File]::WriteAllText($f, ($line + "`n"), (New-Object System.Text.UTF8Encoding $false))
|
|
return (WslPath $f)
|
|
}
|
|
function RunPhase($label, $line) {
|
|
$el = ((Get-Date) - $jobStart).TotalMinutes
|
|
if ($el -gt 16.5) { "RESULT phase $label skipped: $([math]::Round($el,1)) min into the job (the 20-min window)"; return }
|
|
$cmdW = Cmd $label $line
|
|
& wsl.exe -d Ubuntu-24.04 -u root -- bash "$jobW/phase.sh" $label $cmdW 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
|
}
|
|
function Fixtures { (Get-Content (Join-Path $job 'fixtures.txt') -Raw).Trim() }
|
|
function Parent($n) { (Get-Content (Join-Path $job "parent-$n.txt") -Raw).Trim() }
|
|
# proof groups for --mode aggregate: the four blocks' shard 0 proofs from the source job (one shard a block tonight)
|
|
function SrcFirst { if ($ShardSourceFirst -gt 0) { $ShardSourceFirst } else { $first } }
|
|
function SrcParent { if ($ShardSourceParent) { $ShardSourceParent } else { Parent $first } }
|
|
function AggGroups($src) { $f = SrcFirst; (($f..($f + 3)) | ForEach-Object { "$src/block-$_-shard-0-compressed.bin" }) -join ';' }
|
|
# --- own miner ---
|
|
$script:miner = $null
|
|
# (job 1, 21:10Z: `if (StartMiner ...)` was always true because the function's RESULT strings are part of its output; the
|
|
# outcome is the script-scope flag $script:minerOk now)
|
|
function StartMiner($label, $batchLog2) {
|
|
$script:minerOk = $false
|
|
if (-not $minerExe) { "RESULT $label own miner: no app miner command line was captured; phase void"; return }
|
|
$alive = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker-cuda*' } | Measure-Object).Count
|
|
if ($alive -gt 0) { "RESULT $label own miner: the app's CUDA worker is still running ($alive); the phase would mine twice on the card, so it is void (job 3, 22:48Z)"; return }
|
|
$a = $minerArgs
|
|
$extra = "--race off"
|
|
if ($batchLog2 -and $batchLog2 -ne 22) { $extra = "$extra --batch-log2 $batchLog2" }
|
|
if ($a -match '--worker-args "([^"]*)"') { $a = $a -replace '--worker-args "([^"]*)"', ('--worker-args "$1 ' + $extra + '"') }
|
|
elseif ($a -match '--worker-args (\S+)') { $a = $a -replace '--worker-args (\S+)', ('--worker-args "$1 ' + $extra + '"') }
|
|
else { $a = $a + ' --worker-args "' + $extra + '"' }
|
|
$out = Join-Path $job "miner-$label.out"; $err = Join-Path $job "miner-$label.err"
|
|
$script:miner = Start-Process -FilePath $minerExe -ArgumentList $a -WorkingDirectory $app -RedirectStandardOutput $out -RedirectStandardError $err -NoNewWindow -PassThru
|
|
"RESULT $label own miner started $(Stamp) pid $($script:miner.Id) batch_log2=$batchLog2 args=$a"
|
|
$w = 0
|
|
while ($w -lt 150) {
|
|
Start-Sleep -Seconds 10; $w += 10
|
|
$s = Get-Content $out -ErrorAction SilentlyContinue | Where-Object { $_ -match 'STATUS' } | Select-Object -Last 1
|
|
if ($s -and $s -match 'hash=([\d.]+) MH/s' -and [double]$Matches[1] -gt 10) { "RESULT $label own miner hashing after $w s: $s"; $script:minerOk = $true; return }
|
|
if ($script:miner.HasExited) { "RESULT $label own miner EXITED (code $($script:miner.ExitCode)); err: $((Get-Content $err -ErrorAction SilentlyContinue | Select-Object -Last 3) -join ' / ')"; return }
|
|
}
|
|
"RESULT $label own miner not hashing after $w s; last lines: $((Get-Content $out -ErrorAction SilentlyContinue | Select-Object -Last 3) -join ' / ')"
|
|
}
|
|
function StopMiner($label) {
|
|
if (-not $script:miner) { return }
|
|
$pid0 = $script:miner.Id
|
|
Get-CimInstance Win32_Process | Where-Object { $_.ParentProcessId -eq $pid0 } | ForEach-Object { Stop-Process -Id $_.ProcessId -Force -ErrorAction SilentlyContinue }
|
|
Stop-Process -Id $pid0 -Force -ErrorAction SilentlyContinue
|
|
Start-Sleep -Seconds 3
|
|
$left = (Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.ProcessName -like 'igneum-worker*' -or $_.ProcessName -like 'igneum-miner*' } | Measure-Object).Count
|
|
"RESULT $label own miner stopped $(Stamp); miner/worker processes left: $left"
|
|
$script:miner = $null
|
|
}
|
|
function MinerRate($label, $skip) {
|
|
$out = Join-Path $job "miner-$label.out"
|
|
$h = @(); foreach ($l in (Get-Content $out -ErrorAction SilentlyContinue | Where-Object { $_ -match 'STATUS' })) { if ($l -match 'now=([\d.]+) MH/s wall') { $h += [double]$Matches[1] } }
|
|
if ($h.Count -gt $skip) { $h = $h[$skip..($h.Count - 1)] }
|
|
if ($h.Count -eq 0) { return "n=0" }
|
|
$m = $h | Measure-Object -Average -Minimum -Maximum
|
|
"n=$($h.Count) mean=$([math]::Round($m.Average,2)) min=$([math]::Round($m.Minimum,2)) max=$([math]::Round($m.Maximum,2)) MH/s wall (the now= field of the STATUS lines every 10 s, the first $skip skipped)"
|
|
}
|
|
$paused = $false
|
|
try {
|
|
Wsl (Join-Path $job 'prep.sh')
|
|
if (-not (Test-Path (Join-Path $job 'fixtures.txt'))) { "RESULT prep FAILED: no fixtures"; exit 1 }
|
|
$list = Fixtures
|
|
$first2 = ($list -split ',')[0..1] -join ','; $last2 = ($list -split ',')[2..3] -join ','
|
|
if ($PhaseSet -contains 'A0') {
|
|
RunPhase 'A0' "SP1_PROVER=cuda RUST_LOG=info `$H --mode chain --chain $(($list -split ',')[0]) --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-A0.json"
|
|
$errf = Join-Path $job 'host-A0.err'
|
|
"RESULT A0 profile: $((Get-Content $errf -ErrorAction SilentlyContinue | Measure-Object).Count) stderr lines; the first 160 with a level or a duration follow"
|
|
Get-Content $errf -ErrorAction SilentlyContinue | Where-Object { $_ -match 'INFO|WARN|DEBUG|ERROR|time\.|elapsed|took|ms\b|\bs\b' } | Select-Object -First 160 | ForEach-Object { "PROFILE $_" }
|
|
}
|
|
if ($PhaseSet -contains 'A') { RunPhase 'A' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --save-shards --out `$JOB/results-A.json" }
|
|
if ($PhaseSet -contains 'B0') { RunPhase 'B0' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B0.json" }
|
|
if ($PhaseSet -contains 'B') { RunPhase 'B' "SP1_WORKER_VERIFY_INTERMEDIATES=false SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-B.json" }
|
|
$needIdle = @('C','C1','C2','G0','D','E','E0','F','G','H') | Where-Object { $PhaseSet -contains $_ }
|
|
if ($needIdle) {
|
|
# the 5090 switched off through /api/cards (and on again in the finally block): /api/pause then /api/resume left
|
|
# the worker off on 0.3.9 (21:25Z, an app defect in the resume path, fixed for 0.3.11)
|
|
"RESULT card_off $(Stamp) $(CardSwitch $false $null)"; $paused = $true
|
|
$w = 0; $left = 1
|
|
while ($w -lt 120 -and $left -gt 0) { Start-Sleep -Seconds 5; $w += 5; $left = CudaWorkers }
|
|
if ($left -gt 0) { "RESULT card_off_failed $(Stamp) the app's CUDA worker is still running after $w s ($left); /api/pause as the fallback: $(Post '/api/pause' @{})"; $w2 = 0; while ($w2 -lt 60 -and (CudaWorkers) -gt 0) { Start-Sleep -Seconds 5; $w2 += 5 }; $left = CudaWorkers }
|
|
"RESULT paused $(Stamp) after $w s; CUDA workers left: $left; $(Smi)"
|
|
Start-Sleep -Seconds 5
|
|
}
|
|
if ($PhaseSet -contains 'C') { RunPhase 'C' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-C.json" }
|
|
if ($PhaseSet -contains 'C1') { RunPhase 'C1' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$(AggGroups $srcW)' --parent $(SrcParent) --out `$JOB/results-C1.json" }
|
|
if ($PhaseSet -contains 'C2') { RunPhase 'C2' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain `$JOB/block-344-shards4.json --prover 0xCAfc6e74000000000000000000000000000000c2 --save-shards --out `$JOB/results-C2.json" }
|
|
if ($PhaseSet -contains 'G0') {
|
|
$c1 = Cmd 'G01' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $first2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G01.json"
|
|
$c2 = Cmd 'G02' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $last2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G02.json"
|
|
$p1 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G01 $c1" -RedirectStandardOutput (Join-Path $job 'G01.out') -NoNewWindow -PassThru
|
|
Start-Sleep -Seconds 2
|
|
$p2 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G02 $c2" -RedirectStandardOutput (Join-Path $job 'G02.out') -NoNewWindow -PassThru
|
|
$p1.WaitForExit(); $p2.WaitForExit()
|
|
foreach ($f in 'G01.out', 'G02.out') { ((Get-Content (Join-Path $job $f) -Raw -ErrorAction SilentlyContinue) -replace "`0", '') -split "`n" | ForEach-Object { $_.TrimEnd("`r") } | Where-Object { $_ } }
|
|
}
|
|
if ($PhaseSet -contains 'D') {
|
|
StartMiner 'D' 22; if ($script:minerOk) { RunPhase 'D' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-D.json"; "RESULT D miner rate $(MinerRate 'D' 2)" }
|
|
StopMiner 'D'
|
|
}
|
|
if ($PhaseSet -contains 'E') {
|
|
foreach ($b in 20, 18, 16) {
|
|
$lab = "E$b"
|
|
StartMiner $lab $b; if ($script:minerOk) { RunPhase $lab "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-$lab.json"; "RESULT $lab miner rate $(MinerRate $lab 2)" }
|
|
StopMiner $lab
|
|
}
|
|
}
|
|
if ($PhaseSet -contains 'E0') {
|
|
$g = (0..3 | ForEach-Object { "$srcW/block-344-shard-$_-compressed.bin" }) -join ','
|
|
StartMiner 'E0' 22; if ($script:minerOk) { RunPhase 'E0' "SP1_PROVER=cuda RUST_LOG=off `$H --mode aggregate --proofs '$g' --parent $(Parent 344) --out `$JOB/results-E0.json"; "RESULT E0 miner rate $(MinerRate 'E0' 2)" }
|
|
StopMiner 'E0'
|
|
}
|
|
if ($PhaseSet -contains 'F') {
|
|
$pol = (& nvidia-smi compute-policy --set-timeslice=1 2>&1) -join ' '
|
|
"RESULT F timeslice set: $pol"
|
|
if ($pol -notmatch 'rror|nsufficient|not supported|Unknown') {
|
|
StartMiner 'F' 22; if ($script:minerOk) { RunPhase 'F' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-F.json"; "RESULT F miner rate $(MinerRate 'F' 2)" }
|
|
StopMiner 'F'
|
|
}
|
|
"RESULT F timeslice restored: $((& nvidia-smi compute-policy --set-timeslice=0 2>&1) -join ' ')"
|
|
}
|
|
if ($PhaseSet -contains 'G') {
|
|
StartMiner 'G' 22; if ($script:minerOk) {
|
|
$c1 = Cmd 'G1' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $first2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G1.json"
|
|
$c2 = Cmd 'G2' "SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $last2 --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-G2.json"
|
|
$p1 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G1 $c1" -RedirectStandardOutput (Join-Path $job 'G1.out') -NoNewWindow -PassThru
|
|
Start-Sleep -Seconds 2
|
|
$p2 = Start-Process -FilePath 'wsl.exe' -ArgumentList "-d Ubuntu-24.04 -u root -- bash $jobW/phase.sh G2 $c2" -RedirectStandardOutput (Join-Path $job 'G2.out') -NoNewWindow -PassThru
|
|
$p1.WaitForExit(); $p2.WaitForExit()
|
|
foreach ($f in 'G1.out', 'G2.out') { ((Get-Content (Join-Path $job $f) -Raw -ErrorAction SilentlyContinue) -replace "`0", '') -split "`n" | ForEach-Object { $_.TrimEnd("`r") } | Where-Object { $_ } }
|
|
"RESULT G miner rate $(MinerRate 'G' 2)"
|
|
}
|
|
StopMiner 'G'
|
|
}
|
|
if ($PhaseSet -contains 'H') {
|
|
# the combination from this job's own phases: the batch-log2 with the shortest chain among D and E (the miner's
|
|
# rate alongside), the knob when phase B beat B0 by 5% or more; skipped when the job is past 15 min (the window)
|
|
try {
|
|
$cands = @(); foreach ($lab in 'D', 'E20', 'E18', 'E16') { $f = Join-Path $job "results-$lab.json"; if (Test-Path $f) { $r = (Get-Content $f -Raw | ConvertFrom-Json); $cands += [pscustomobject]@{ lab = $lab; batch = $(if ($lab -eq 'D') { 22 } else { [int]$lab.Substring(1) }); secs = [double]$r.chain_seconds } } }
|
|
if ($cands.Count -gt 0) { $bestC = $cands | Sort-Object secs | Select-Object -First 1; $BestBatch = $bestC.batch; "RESULT H choice: $(($cands | ForEach-Object { "$($_.lab)=$([math]::Round($_.secs,1))s" }) -join ' ') -> batch_log2 $BestBatch" }
|
|
$b0 = Join-Path $job 'results-B0.json'; $b1 = Join-Path $job 'results-B.json'
|
|
if ((Test-Path $b0) -and (Test-Path $b1)) { $s0 = [double](Get-Content $b0 -Raw | ConvertFrom-Json).aggregate_seconds; $s1 = [double](Get-Content $b1 -Raw | ConvertFrom-Json).aggregate_seconds; if ($s1 -lt 0.95 * $s0) { $BestKnobs = 'SP1_WORKER_VERIFY_INTERMEDIATES=false' }; "RESULT H knobs: B0 $([math]::Round($s0,1)) s, B $([math]::Round($s1,1)) s -> '$BestKnobs'" }
|
|
} catch { "RESULT H choice error: $_" }
|
|
$elapsedMin = ((Get-Date) - $jobStart).TotalMinutes
|
|
if ($elapsedMin -gt 15) { "RESULT H skipped: $([math]::Round($elapsedMin,1)) min into the job (the 20-min window)" }
|
|
else { StartMiner 'H' $BestBatch }
|
|
if ($elapsedMin -le 15 -and $script:minerOk) { RunPhase 'H' "$BestKnobs SP1_PROVER=cuda RUST_LOG=off `$H --mode chain --chain $list --prover 0xCAfc6e74000000000000000000000000000000c2 --out `$JOB/results-H.json"; "RESULT H miner rate $(MinerRate 'H' 2)" }
|
|
StopMiner 'H'
|
|
}
|
|
} finally {
|
|
StopMiner 'final'
|
|
& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -x sp1-gpu-server; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') }
|
|
if ($paused) {
|
|
"RESULT card_on $(Stamp) $(CardSwitch $true $null)"; "RESULT resume $(Stamp) $(Post '/api/resume' @{})"
|
|
$w3 = 0; while ($w3 -lt 90 -and (CudaWorkers) -eq 0) { Start-Sleep -Seconds 5; $w3 += 5 }
|
|
if ((CudaWorkers) -eq 0) { "RESULT miner_back_failed $(Stamp) no CUDA worker after $w3 s; /api/start: $(Post '/api/start' @{})" } else { "RESULT miner_back $(Stamp) CUDA worker running after $w3 s" }
|
|
}
|
|
"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})"
|
|
"RESULT end $(Stamp) $(Smi)"
|
|
}
|