diff --git a/docs/bench-log.md b/docs/bench-log.md index 819748536..9e570ab68 100644 --- a/docs/bench-log.md +++ b/docs/bench-log.md @@ -1796,3 +1796,16 @@ Reading. Between 2^22 and 2^20 nothing moves: the card's time-slice scheduler al | Job 6's identities step found `settings.json` already at 8 identities under the active key `nvidia:0:NVIDIA GeForce RTX 5090` (a stale key `nvidia:NVIDIA GeForce RTX 5090` carries 2), so no change was sent; job 6's `/api/cards` with the 5090 disabled answered ok but the worker ran on for 120 s, `/api/pause` stopped it in 5 s, and at the end `/api/resume` brought it back in 5 s on 0.3.11 | the card switch keeps the pause as its fallback; the resume path works on 0.3.11 | | PC 2 ran three jobs at once from 01:24Z (`run-prover-on-pc2-20261006` at 01:24:21Z, the ledger suites build at 01:26:15Z, while agg-cost-pc2-6's closing report was still being uploaded): the app does not serialise jobs, "one job per machine at a time" holds only by the coordinator's word; job 6 had closed at 01:24:14Z, so its rows are clean | nothing of mine to fix; a rule for the job runner | | The make-package gate ran the exporter's side files (`block-N.json.node-plan.json`) as fixtures and failed; its execute step took the exclusive `measure` lock for a cycle count and queued 25 min behind a packbench run | the glob skips `.node-plan.json`; the execute step runs under the `run` lock (a count, not a time) | + +### 6 October 2026, 07:12Z to 07:17Z, the host's chain mode with --save-shards records and --prev, on the Mac's CPU + +`tools/lock/with-lock.sh run`, `SP1_PROVER=cpu igneum-prove-host --mode chain --chain proving/fixtures/chain/block-81046.json,block-81047.json --save-shards --out chain-a.json`, then `--chain block-81048.json --save-shards --prev segment-81047-aggregated.bin --out chain-b.json` (the app branch at ce8f34a, Apple M5 Max, CPU prover). The flags the app's segment path needs, before PC 2 (approximate figures: a CPU run, one sample each): + +| Step | Value | +|---|---| +| Shard proof, CPU, empty block | 34.7 s and 36.3 s | +| Aggregation, CPU, 1 then 2 deferred proofs | 39.1 s, 50.8 s | +| Chain of 2, end to end | 160.9 s | +| Per-shard records written | 2 (number, block_hash, shard, statement, proof_sha256, proof_bytes 1,272,897, proof_file, prove_seconds) | +| `--prev` run: base_chain_len, final chain_len | 2, 3 (the chain continued; a wrong previous proof is refused by number and parent hash) | + diff --git a/tools/proving-v1/pc2-segments.ps1 b/tools/proving-v1/pc2-segments.ps1 new file mode 100644 index 000000000..4944f3572 --- /dev/null +++ b/tools/proving-v1/pc2-segments.ps1 @@ -0,0 +1,337 @@ +# Proving v1, the segment-aligned prover on PC 2 (6 October 2026, the project lead: "proving v1 is active but has proven zero +# segments, and cannot with one prover; find a way to solve this"). A signed `run` job (shell powershell, not +# elevated; the app keeps mining; the app's own prover is switched OFF for the run through /api/prove and ON again +# at the end; the live /opt/igneum host is untouched: this build lands in /opt/igneum-segal). +# +# What it does, for RUN_MINUTES, exactly what the app's segment path (app/igneum-app/src/prover.rs, pick_segment and +# prove_segment) does: +# 1. the node's v1 status and the work list (igneum_getAssignedShards, lookback 600) grouped into whole untouched +# segments (every block present, every shard open, unpaid, not in our pool); the newest one inside its deadline by +# the margin (240 DAA, or 1.5x the last segment's time) is claimed; the app ranks by FNV of (first, key) for the +# multi-prover spread, which with one prover changes nothing but the order +# 2. the segment statement: executed, pending; fresh when the previous segment is not paid and no verified record of +# it waits in the pool, else chained to the previous proof (--prev) when the pool holds it +# 3. one export (igneum_exportSegments 0..last), one fixture per block, one host run (--mode chain --save-shards +# [--prev]) with a 1-s nvidia-smi sampler underneath +# 4. every shard record signed (igneum-miner.exe sign-record) and submitted (igneum_submitProofRecord), then the +# segment record (sign-segment-record, igneum_submitSegmentRecord) +# 5. the paid state of every submitted segment polled each pass (igneum_getSegmentRecords: carried, paid) +# Every number is a RESULT line. The miner's hash rate comes from the app log's "status: ... MH/s" lines (every 30 s), +# the 30 minutes before the job against the run. Card keys, when needed, come from settings.json, never /api/state +# (that reply is "{}" on 0.3.11 once paid_wei passes u64::MAX; fixed in 6714a45). +$ErrorActionPreference = 'Continue' +$RunMinutes = 30 +function Stamp { (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') } +$app = if ($env:IGNEUM_APP_DIR) { $env:IGNEUM_APP_DIR } else { Join-Path $env:LOCALAPPDATA 'igneum\app' } +$urlFile = Join-Path $app 'app.url' +$base = if (Test-Path $urlFile) { (Get-Content $urlFile -Raw).Trim().TrimEnd('/') } else { '' } +function Post($path, $obj) { try { (Invoke-RestMethod -Method Post -Uri "$base$path" -ContentType 'application/json' -Body ($obj | ConvertTo-Json -Compress) -TimeoutSec 10) | ConvertTo-Json -Compress } catch { "error: $_" } } +function Rpc($method, $params) { + try { (Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$evmPort" -ContentType 'application/json' -Body (@{jsonrpc='2.0'; id=1; method=$method; params=$params} | ConvertTo-Json -Compress -Depth 8) -TimeoutSec 120).result } catch { $script:rpcErr = "$_"; $null } +} +function RpcRaw($method, $params) { + try { Invoke-RestMethod -Method Post -Uri "http://127.0.0.1:$evmPort" -ContentType 'application/json' -Body (@{jsonrpc='2.0'; id=1; method=$method; params=$params} | ConvertTo-Json -Compress -Depth 8) -TimeoutSec 120 } catch { $script:rpcErr = "$_"; $null } +} +function Hex($v) { if ($null -eq $v) { return 0 }; if ($v -is [string] -and $v.StartsWith('0x')) { return [Convert]::ToInt64($v.Substring(2), 16) }; return [int64]$v } +$job = $env:IGNEUM_JOB_DIR +if (-not $job) { $job = Join-Path $env:TEMP 'igneum-segments' } +New-Item -ItemType Directory -Force -Path $job | Out-Null +$jobStart = Get-Date +"RESULT start $(Stamp) app_dir=$app base_len=$($base.Length) job=$job run_minutes=$RunMinutes" +$evmPort = $null +foreach ($p in 26790, 26800, 26810) { if (Rpc 'igneum_getProvingStatus' @()) { $evmPort = $p; break } } +if (-not $evmPort) { "RESULT refused $(Stamp) no node EVM RPC on 26790/26800/26810: $script:rpcErr"; exit 1 } +# the app's miner (the signer) and its label, the payout address +$mp = Get-CimInstance Win32_Process | Where-Object { $_.Name -like 'igneum-miner*' -and $_.CommandLine -like '* mine *' } | Select-Object -First 1 +$minerExe = if ($mp) { $mp.ExecutablePath } else { $null } +if (-not $minerExe -or -not (Test-Path $minerExe)) { + $cand = Get-ChildItem -Path (Split-Path $app -Parent) -Recurse -Filter 'igneum-miner.exe' -ErrorAction SilentlyContinue | Select-Object -First 1 + if ($cand) { $minerExe = $cand.FullName } +} +if (-not $minerExe) { "RESULT refused $(Stamp) no igneum-miner.exe found (the signer)"; exit 1 } +$settingsRaw = Get-Content (Join-Path $app 'settings.json') -Raw +$payout = [regex]::Match($settingsRaw, '"address":\s*"(0x[0-9a-fA-F]{40})"').Groups[1].Value +if ($payout.Length -ne 42) { "RESULT refused $(Stamp) no payout address in settings.json"; exit 1 } +$identities = 1; $im = [regex]::Match($settingsRaw, '"nvidia:[^"]*":\s*\{[^}]*"identities":\s*(\d+)'); if ($im.Success) { $identities = [int]$im.Groups[1].Value } +$machineId = ''; $mm = [regex]::Match($settingsRaw, '"machine_id":\s*"([0-9a-f]+)"'); if ($mm.Success) { $machineId = $mm.Groups[1].Value } +if (-not $machineId) { $mf = Join-Path $app 'machine-id'; if (Test-Path $mf) { $machineId = (Get-Content $mf -Raw).Trim() } } +if (-not $machineId -and $job -match '([0-9a-f]{8})') { $machineId = $Matches[1] } +$id8 = if ($machineId.Length -ge 8) { $machineId.Substring(0, 8) } else { '1ccfe586' } +$label = if ($identities -gt 1) { "win-$id8-1-1" } else { "win-$id8-1" } +$chain = 'igneum-devnet' # the app's chain_name with no devnet suffix (IGNEUM_APP_DEVNET_SUFFIX unset on the fleet) +$keyHash = (& $minerExe key-hash $label 2>$null | Select-Object -Last 1).Trim() +if ($keyHash -and -not $keyHash.StartsWith('0x')) { $keyHash = "0x$keyHash" } +"RESULT signer $(Stamp) miner=$minerExe label=$label key=$keyHash chain=$chain payout=$payout identities=$identities" +if (-not $keyHash -or $keyHash.Length -lt 64) { "RESULT refused $(Stamp) key-hash gave nothing"; exit 1 } +# the live prover off for the run: its sp1-gpu-server would be shared with ours (same socket, its environment) +"RESULT prove_off $(Stamp) $(Post '/api/prove' @{on=$false})" +Start-Sleep -Seconds 5 +# --- WSL side: the package built into /opt/igneum-segal (root, the warm target of the live build), the segment runner +$pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-segal\igneum-prove-wsl2' +if (-not (Test-Path $pkg)) { $pkg = Join-Path $env:LOCALAPPDATA 'igneum\prove\igneum-prove-wsl2-segal' } +"RESULT package $(Stamp) $pkg exists=$(Test-Path (Join-Path $pkg 'package'))" +function WslPath($p) { $w = (& wsl.exe -d Ubuntu-24.04 -u root -- wslpath -a ($p -replace '\\', '/') 2>$null); if ($w) { ($w -replace "`0", '').Trim() } else { '/mnt/c' + ($p.Substring(2) -replace '\\', '/') } } +$pkgW = WslPath $pkg; $jobW = WslPath $job +$prep = @' +set -uo pipefail +export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH" +CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}" +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +PKG="$1" +DEST="$HOME/igneum-prove-segal"; LIVE_TARGET="$HOME/igneum-prove/proving/igneum-prove/target" +mkdir -p "$DEST" +rsync -a --delete --exclude target "$PKG/package/" "$DEST/" +find "$DEST" -name target -prune -o -type f -exec touch {} + 2>/dev/null +grep -o '"program_id": "0x[0-9a-f]*"' "$DEST/proving/igneum-prove/elf/manifest.json" | sed 's/^/RESULT manifest /' +cd "$DEST/proving/igneum-prove" +t0=$(date +%s) +if ! CARGO_TARGET_DIR="$LIVE_TARGET" cargo build --release -p igneum-prove-export -p igneum-prove-host --features igneum-prove-host/cuda 2>&1 | tail -3; then echo "RESULT build FAILED"; exit 1; fi +echo "RESULT build $(stamp) exit 0 in $(( $(date +%s) - t0 )) s" +mkdir -p /opt/igneum-segal && cp "$LIVE_TARGET/release/igneum-prove-host" "$LIVE_TARGET/release/igneum-prove-export" /opt/igneum-segal/ +H=/opt/igneum-segal/igneum-prove-host; X=/opt/igneum-segal/igneum-prove-export +echo "RESULT installed $(sha256sum $H | cut -c1-16) host, $(sha256sum $X | cut -c1-16) export; live /opt/igneum untouched: $(sha256sum /opt/igneum/igneum-prove-host | cut -c1-16)" +$H --mode id | sed 's/^/RESULT segal-host /' +echo "RESULT gpu_server_before $(stamp) running=$(pgrep -x sp1-gpu-server | wc -l) socket=$(ls -la /tmp/sp1-cuda-0.sock 2>/dev/null || echo none)" +# the root-socket rule (tools/ci/prover-socket-check.sh): this run is root; the app's prover is off, so its server goes too +pkill -f sp1-gpu-server; sleep 1; rm -f /tmp/sp1-cuda-*.sock +echo "RESULT socket_start $(stamp) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)" +'@ +# bash seg.sh [prev file]: unwrap the export, cut the blocks, run the chain with the sampler +$seg = @' +set -uo pipefail +export PATH="$HOME/.cargo/bin:$HOME/.sp1/bin:$PATH" +CUDA_DIR="$(ls -d /usr/local/cuda-12.* 2>/dev/null | sort -V | tail -1 || true)"; [ -n "$CUDA_DIR" ] && export PATH="$CUDA_DIR/bin:$PATH" && export LD_LIBRARY_PATH="$CUDA_DIR/lib64:/usr/lib/wsl/lib:${LD_LIBRARY_PATH:-}" +stamp() { date -u +%Y-%m-%dT%H:%M:%SZ; } +JOB="$1"; FIRST="$2"; LAST="$3"; PAYOUT="$4"; PREV="${5:-}" +H=/opt/igneum-segal/igneum-prove-host; X=/opt/igneum-segal/igneum-prove-export +D="$JOB/seg-$FIRST"; mkdir -p "$D" +t0=$(date +%s.%N) +python3 -c "import json; d=json.load(open('$D/seq.json')); json.dump(d['result'], open('$D/export.json','w'))" || { echo "RESULT seg $FIRST unwrap FAILED"; exit 1; } +LIST="" +for n in $(seq $FIRST $LAST); do + if ! $X "$D/export.json" $n "$D/block-$n.json" --source "PC 2 live devnet, segment-aligned prover" >"$D/export-$n.log" 2>&1; then echo "RESULT seg $FIRST cut $n FAILED: $(tail -1 "$D/export-$n.log")"; exit 1; fi + LIST="$LIST${LIST:+,}$D/block-$n.json" +done +CUT=$(python3 -c "import time; print(round(time.time() - $t0, 1))") +echo "RESULT seg $FIRST cut $(stamp) $LAST blocks in $CUT s" +nvidia-smi --query-gpu=timestamp,index,memory.used,utilization.gpu,power.draw --format=csv,noheader,nounits -l 1 > "$D/smi.csv" 2>/dev/null & +SMI=$! +t1=$(date +%s.%N) +PREVARG=""; [ -n "$PREV" ] && PREVARG="--prev $PREV" +SP1_PROVER=cuda RUST_LOG=off $H --mode chain --chain "$LIST" --prover "$PAYOUT" --save-shards $PREVARG --out "$D/chain-results.json" >"$D/chain.log" 2>"$D/chain.err" +RC=$? +WALL=$(python3 -c "import time; print(round(time.time() - $t1, 1))") +kill $SMI 2>/dev/null; sleep 1 +grep -E "^RESULT (chain block|chain:|chain prev|setup)" "$D/chain.log" | sed "s/^/seg $FIRST: /" +awk -F', *' '{ if ($3+0 > max) max=$3+0; u+=$4; n++ } END { if (n) printf "RESULT seg FIRST gpu samples=%d memory_used_max_mib=%d util_mean_pct=%.1f\n", n, max, u/n }' "$D/smi.csv" | sed "s/FIRST/$FIRST/" +echo "RESULT seg $FIRST chain $(stamp) exit $RC wall $WALL s" +if [ $RC -ne 0 ]; then tail -4 "$D/chain.err" | sed "s/^/seg $FIRST stderr: /"; exit 1; fi +# the submit bodies: one JSON-RPC request per shard record and one for the segment, the proof bytes as hex +python3 - "$D" <<'PY' +import json, sys, os +d = sys.argv[1] +r = json.load(open(os.path.join(d, 'chain-results.json'))) +recs = [] +for b in r['blocks']: + for s in b.get('shard_records', []): + recs.append(s) +json.dump(recs, open(os.path.join(d, 'shard-records.json'), 'w')) +print('RESULT seg %s records %d shard records, segment chain_len %s, proof %s bytes' % (r['first'], len(recs), r['segment_chain_len'], r['segment_proof_bytes'])) +PY +'@ +# bash body.sh : the JSON-RPC body with the proof as hex +$body = @' +set -u +python3 - "$1" "$2" "$3" "$4" <<'PY' +import json, sys, binascii +proof = '0x' + binascii.hexlify(open(sys.argv[1], 'rb').read()).decode() +json.dump({'jsonrpc': '2.0', 'id': 1, 'method': sys.argv[3], 'params': [{'record': sys.argv[2], 'proof': proof}]}, open(sys.argv[4], 'w')) +PY +'@ +# bash unhex.sh : a hex string (0x...) to bytes +$unhex = @' +set -u +python3 -c "import sys,binascii; h=open(sys.argv[1]).read().strip(); h=h[2:] if h.startswith('0x') else h; open(sys.argv[2],'wb').write(binascii.unhexlify(h))" "$1" "$2" +'@ +foreach ($pair in @(@('prep.sh', $prep), @('seg.sh', $seg), @('body.sh', $body), @('unhex.sh', $unhex))) { + [IO.File]::WriteAllText((Join-Path $job $pair[0]), ($pair[1] -replace "`r`n", "`n"), (New-Object System.Text.UTF8Encoding $false)) +} +function Wsl { param([string[]]$a) & wsl.exe -d Ubuntu-24.04 -u root -- bash @a 2>&1 | ForEach-Object { ($_ -replace "`0", '') } } +Wsl @("$jobW/prep.sh", $pkgW) +if (-not (Test-Path (Join-Path $job 'prep.sh'))) { "RESULT refused $(Stamp) prep.sh missing"; exit 1 } +# --- the app log's miner rate before the run (baseline): "status: ... MH/s" every 30 s +$logDir = Join-Path $app 'logs' +function RateLines($since, $until) { + $out = @() + foreach ($f in (Get-ChildItem -Path $logDir -Filter 'app-*.log' -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 2)) { + foreach ($l in (Get-Content $f.FullName -ErrorAction SilentlyContinue | Where-Object { $_ -match '^(\d{10}) status: .* ([\d.]+) MH/s' })) { + $t = [int64]$Matches[1]; if ($t -ge $since -and $t -le $until) { $out += [double]$Matches[2] } + } + } + return $out +} +function Epoch($d) { [int64](($d.ToUniversalTime()) - (Get-Date '1970-01-01')).TotalSeconds } +$t0 = Epoch $jobStart +$before = RateLines ($t0 - 1800) $t0 +if ($before.Count) { $m = $before | Measure-Object -Average -Minimum -Maximum; "RESULT rate_before $(Stamp) n=$($before.Count) mean=$([math]::Round($m.Average,2)) min=$($m.Minimum) max=$($m.Maximum) MH/s (the app log's status lines, the 30 min before the job)" } else { "RESULT rate_before $(Stamp) no status lines found in $logDir" } +# --- the loop +$submittedSegments = @{} # first -> @{last; aggWei; at} +$attempted = @{} +$lastSegSecs = 0 +$n = 1; $start = 0; $unproven = 600 +$passes = 0; $claimed = 0; $submitted = 0; $shardsAccepted = 0; $shardsRefused = 0 +$runStart = Get-Date +function Candidates { + $st = Rpc 'igneum_getProvingStatus' @() + if (-not $st -or -not $st.v1 -or -not $st.v1.active -or -not $st.v1.start) { return @() } + $script:start = Hex $st.v1.start; $script:n = [math]::Max(1, (Hex $st.v1.segmentBlocks)); $script:unproven = Hex $st.v1.unprovenDaa + $tipDaa = Hex $st.tipDaa + $work = Rpc 'igneum_getAssignedShards' @(@($keyHash), 600) + if (-not $work) { return @() } + $by = @{} + foreach ($w in $work) { $num = Hex $w.number; if ($num -lt $script:start) { continue }; if (-not $by.ContainsKey($num)) { $by[$num] = @() }; $by[$num] += $w } + $need = [math]::Max(240, [math]::Ceiling($lastSegSecs * 1.5)) + $segs = @() + $seen = @{} + foreach ($num in ($by.Keys | Sort-Object)) { + $k = [math]::Floor(($num - $script:start) / $script:n); $first = $script:start + $k * $script:n; $last = $first + $script:n - 1 + if ($seen.ContainsKey($first)) { continue }; $seen[$first] = $true + if ($attempted.ContainsKey($first)) { continue } + $whole = $true; $shards = @(); $lastDaa = 0; $wei = 0 + for ($b = $first; $b -le $last; $b++) { + if (-not $by.ContainsKey($b)) { $whole = $false; break } + $es = $by[$b] | Sort-Object { $_.shard } -Unique + foreach ($e in $es) { + $inPool = ($e.pool -and (@($e.pool)).Count -gt 0) + if (-not $e.open -or $null -ne $e.paid -or $inPool) { $whole = $false; break } + $shards += @{ number = $b; hash = $e.hash; shard = [int]$e.shard } + if ($b -eq $last) { $lastDaa = Hex $e.daaScore } + } + if (-not $whole) { break } + } + if ($whole -and $shards.Count -gt 0) { + $deadline = $lastDaa + $script:unproven + if ($deadline -ge ($tipDaa + 1 + $need)) { $segs += @{ first = $first; last = $last; lastDaa = $lastDaa; deadline = $deadline; shards = $shards; margin = ($deadline - $tipDaa - 1) } } + } + } + # newest first (the most time before the deadline); the app ranks by FNV(first, key) for the spread + return @($segs | Sort-Object { $_.first } -Descending) +} +while (((Get-Date) - $runStart).TotalMinutes -lt $RunMinutes) { + $passes++ + # paid segments + foreach ($f in @($submittedSegments.Keys)) { + $rec = Rpc 'igneum_getSegmentRecords' @(('0x{0:x}' -f $f)) + if ($rec) { + $carried = (@($rec.carried)).Count + if ($rec.paid) { "RESULT paid $(Stamp) segment $f..$($submittedSegments[$f].last) paid wei=$($rec.paid.wei) ($([math]::Round((Hex $rec.paid.wei) / 1e18, 4)) IGN) carrier=$(Hex $rec.paid.carrierNumber) payout=$($rec.paid.payout) after $([math]::Round(((Get-Date) - $submittedSegments[$f].at).TotalSeconds)) s"; $submittedSegments.Remove($f) } + elseif ($carried -gt 0) { "RESULT carried $(Stamp) segment $f carried=$carried valid=$(@($rec.carried | ForEach-Object { $_.valid }) -join ',') rejected=$(@($rec.carried | ForEach-Object { $_.rejected }) -join ';')" } + } + } + $cands = Candidates + if (-not $cands -or $cands.Count -eq 0) { "RESULT pass $passes $(Stamp) no whole segment inside the margin (need=$([math]::Max(240, [math]::Ceiling($lastSegSecs * 1.5))) DAA); waiting 20 s"; Start-Sleep -Seconds 20; continue } + $picked = $null; $prevFile = $null; $expected = '' + $tries = 0 + foreach ($c in $cands) { + if ($tries -ge 3) { break }; $tries++ + $stmt = Rpc 'igneum_getSegmentStatement' @(('0x{0:x}' -f $c.first)) + if (-not $stmt -or -not $stmt.executed -or $stmt.status.status -ne 'pending') { $attempted[$c.first] = $true; "RESULT skip $(Stamp) segment $($c.first): executed=$($stmt.executed) status=$($stmt.status.status)"; continue } + if ($null -eq $stmt.previous) { + if ($c.first -ge ($start + $n)) { + $pr = Rpc 'igneum_getSegmentRecords' @(('0x{0:x}' -f ($c.first - $n))) + $waiting = $false + if ($pr -and $pr.pool) { foreach ($e in @($pr.pool)) { if ($e.verified -eq $true -and $null -eq $e.includedIn) { $waiting = $true } } } + if ($waiting -or ($pr -and $pr.paid)) { "RESULT skip $(Stamp) segment $($c.first): the previous segment has a record waiting or paid; a fresh chain would be refused"; continue } + } + $picked = $c; $expected = $stmt.publicValuesFresh; break + } + if ($stmt.previous.proofInPool -ne $true) { "RESULT skip $(Stamp) segment $($c.first): previous paid, its proof not in this pool"; continue } + $got = Rpc 'igneum_getSegmentProofBytes' @($stmt.previous.first, $stmt.previous.keyHash) + if (-not $got -or -not $got.proof) { continue } + $hexFile = Join-Path $job "prev-$($c.first).hex"; $binFile = Join-Path $job "prev-$($c.first).bin" + [IO.File]::WriteAllText($hexFile, $got.proof) + Wsl @("$jobW/unhex.sh", (WslPath $hexFile), (WslPath $binFile)) + if (-not (Test-Path $binFile)) { continue } + $picked = $c; $prevFile = WslPath $binFile; $expected = $stmt.publicValuesContinuing; break + } + if (-not $picked) { "RESULT pass $passes $(Stamp) $($cands.Count) candidates, none usable this pass; waiting 20 s"; Start-Sleep -Seconds 20; continue } + $first = $picked.first; $last = $picked.last + $attempted[$first] = $true; $claimed++ + $tipNow = Hex (Rpc 'eth_blockNumber' @()) + "RESULT claim $(Stamp) segment $first..$last ($($picked.shards.Count) shards, $(if ($prevFile) { 'continuing' } else { 'fresh' })), margin=$($picked.margin) DAA, tip=$tipNow, candidates=$($cands.Count)" + $segStart = Get-Date + $d = Join-Path $job "seg-$first"; New-Item -ItemType Directory -Force -Path $d | Out-Null + # 3. the export (curl.exe streams the reply to a file; Invoke-WebRequest's Content is a string there) + $t = Get-Date + $bodyTxt = (@{jsonrpc='2.0'; id=1; method='igneum_exportSegments'; params=@('0x0', ('0x{0:x}' -f $last))} | ConvertTo-Json -Compress) + $bodyFile = Join-Path $d 'export-request.json'; $seqFile = Join-Path $d 'seq.json' + [IO.File]::WriteAllText($bodyFile, $bodyTxt, (New-Object System.Text.UTF8Encoding $false)) + & curl.exe -s -S -m 600 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bodyFile" -o $seqFile 2>&1 | ForEach-Object { "curl: $_" } + if (-not (Test-Path $seqFile) -or (Get-Item $seqFile).Length -lt 1000) { "RESULT seg $first export FAILED"; continue } + "RESULT seg $first export $(Stamp) $((Get-Item $seqFile).Length) bytes in $([math]::Round(((Get-Date) - $t).TotalSeconds,1)) s" + $segArgs = @("$jobW/seg.sh", $jobW, "$first", "$last", $payout); if ($prevFile) { $segArgs += $prevFile } + Wsl $segArgs + Remove-Item $seqFile -ErrorAction SilentlyContinue + $resFile = Join-Path $d 'chain-results.json' + if (-not (Test-Path $resFile)) { "RESULT seg $first FAILED: no chain results"; continue } + $res = Get-Content $resFile -Raw | ConvertFrom-Json + $recs = Get-Content (Join-Path $d 'shard-records.json') -Raw | ConvertFrom-Json + # 4. the shard records + $okShards = 0 + foreach ($r in $recs) { + $sg = (& $minerExe sign-record $label $chain $r.block_hash "$($r.number)" "$($r.shard)" $payout $r.statement $r.proof_sha256 2>&1 | Select-Object -Last 1) + $signed = $null; try { $signed = $sg | ConvertFrom-Json } catch {} + if (-not $signed -or -not $signed.record) { "RESULT seg $first shard $($r.number)/$($r.shard) sign FAILED: $sg"; $shardsRefused++; continue } + $proofWin = $r.proof_file -replace '^/mnt/c/', 'C:/' -replace '/', '\' + $bf = Join-Path $d "body-$($r.number)-$($r.shard).json" + Wsl @("$jobW/body.sh", $r.proof_file, $signed.record, 'igneum_submitProofRecord', (WslPath $bf)) + $reply = (& curl.exe -s -S -m 120 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bf" 2>&1) + $rr = $null; try { $rr = ($reply | ConvertFrom-Json).result } catch {} + if ($rr -and $rr.accepted) { $okShards++; $shardsAccepted++ } else { $shardsRefused++; "RESULT seg $first shard $($r.number)/$($r.shard) refused: $(if ($rr) { $rr.reason } else { "$reply".Substring(0, [math]::Min(300, "$reply".Length)) })" } + Remove-Item $bf -ErrorAction SilentlyContinue + } + "RESULT seg $first shards $(Stamp) accepted $okShards of $($recs.Count) (prove $([math]::Round($res.shard_prove_seconds_total,1)) s, aggregation $([math]::Round($res.aggregate_prove_seconds_total,1)) s, chain $([math]::Round($res.chain_seconds,1)) s)" + if ($okShards -ne $recs.Count) { "RESULT seg $first FAILED: not every shard record accepted; the segment record is not submitted"; continue } + # 5. the segment record: the statement against the node's (every field but provers, bytes 236..268 of the 340) + $pv = $res.segment_public_values + function Strip($h) { $h = $h -replace '^0x', ''; if ($h.Length -eq 680) { $h.Substring(0, 472) + $h.Substring(536) } else { $h } } + if ((Strip $pv) -ne (Strip $expected)) { "RESULT seg $first FAILED: statement differs from the node's native one; ours $($pv.Substring(0,66)) node $($expected.Substring(0, [math]::Min(66, $expected.Length)))"; continue } + $lastHash = ($picked.shards | Where-Object { $_.number -eq $last } | Select-Object -First 1).hash + $sg = (& $minerExe sign-segment-record $label $chain "$first" "$last" $lastHash $payout $pv $res.segment_proof_sha256 2>&1 | Select-Object -Last 1) + $signed = $null; try { $signed = $sg | ConvertFrom-Json } catch {} + if (-not $signed -or -not $signed.record) { "RESULT seg $first segment sign FAILED: $sg"; continue } + $bf = Join-Path $d 'body-segment.json' + Wsl @("$jobW/body.sh", $res.segment_proof_file, $signed.record, 'igneum_submitSegmentRecord', (WslPath $bf)) + $reply = (& curl.exe -s -S -m 120 -X POST "http://127.0.0.1:$evmPort" -H 'Content-Type: application/json' --data-binary "@$bf" 2>&1) + $rr = $null; try { $rr = ($reply | ConvertFrom-Json).result } catch {} + Remove-Item $bf -ErrorAction SilentlyContinue + $segSecs = [math]::Round(((Get-Date) - $segStart).TotalSeconds, 1) + if ($rr -and $rr.accepted) { + $submitted++; $lastSegSecs = $segSecs + $stmt2 = Rpc 'igneum_getSegmentStatement' @(('0x{0:x}' -f $first)) + $aggWei = if ($stmt2) { Hex $stmt2.aggregatorWei } else { 0 } + $submittedSegments[$first] = @{ last = $last; aggWei = $aggWei; at = (Get-Date) } + "RESULT seg $first submitted $(Stamp) segment $first..$last record accepted (new=$($rr.new), chain_len $($res.segment_chain_len)), aggregator share $([math]::Round($aggWei / 1e18, 4)) IGN, end to end $segSecs s (export+cut+chain+sign+submit)" + } else { + "RESULT seg $first segment refused $(Stamp): $(if ($rr) { $rr.reason } else { "$reply".Substring(0, [math]::Min(300, "$reply".Length)) }); end to end $segSecs s" + } + foreach ($b in $first..$last) { Remove-Item (Join-Path $d "block-$b.json") -ErrorAction SilentlyContinue } + Remove-Item (Join-Path $d 'export.json') -ErrorAction SilentlyContinue +} +# --- the end: paid state once more, the rate during the run, the sockets, the prover back on +Start-Sleep -Seconds 30 +$paidTotal = 0; $paidCount = 0 +foreach ($f in @($submittedSegments.Keys)) { + $rec = Rpc 'igneum_getSegmentRecords' @(('0x{0:x}' -f $f)) + if ($rec -and $rec.paid) { $paidCount++; $paidTotal += (Hex $rec.paid.wei); "RESULT paid $(Stamp) segment $f..$($submittedSegments[$f].last) paid wei=$($rec.paid.wei) carrier=$(Hex $rec.paid.carrierNumber)"; $submittedSegments.Remove($f) } + else { "RESULT unpaid $(Stamp) segment $f..$($submittedSegments[$f].last) carried=$(if ($rec) { (@($rec.carried)).Count } else { '?' }) pool=$(if ($rec) { (@($rec.pool)).Count } else { '?' }) $(if ($rec -and $rec.pool) { ($rec.pool | ForEach-Object { "verified=$($_.verified) included=$($_.includedIn) note=$($_.note)" }) -join '; ' })" } +} +$st = Rpc 'igneum_getProvingStatus' @() +if ($st) { "RESULT node_v1 $(Stamp) paidSegments=$($st.v1.paidSegments) paidSegmentWei=$($st.v1.paidSegmentWei) window pending=$($st.v1.segmentsInWindow.pending) proven=$($st.v1.segmentsInWindow.proven) unproven=$($st.v1.segmentsInWindow.unproven) pool entries=$($st.v1.pool.entries) verified=$($st.v1.pool.verified) failed=$($st.v1.pool.failed)" } +$t1 = Epoch (Get-Date) +$during = RateLines (Epoch $runStart) $t1 +if ($during.Count) { $m = $during | Measure-Object -Average -Minimum -Maximum; "RESULT rate_during $(Stamp) n=$($during.Count) mean=$([math]::Round($m.Average,2)) min=$($m.Minimum) max=$($m.Maximum) MH/s (the app log's status lines over the run)" } else { "RESULT rate_during $(Stamp) no status lines found" } +"RESULT summary $(Stamp) passes=$passes claimed=$claimed submitted=$submitted paid=$paidCount paid_wei=$paidTotal shards_accepted=$shardsAccepted shards_refused=$shardsRefused last_segment_s=$lastSegSecs run_min=$([math]::Round(((Get-Date) - $runStart).TotalMinutes,1))" +& wsl.exe -d Ubuntu-24.04 -u root -- bash -c 'pkill -f sp1-gpu-server; sleep 1; rm -f /tmp/sp1-cuda-*.sock; echo "RESULT socket_cleanup $(date -u +%Y-%m-%dT%H:%M:%SZ) servers=$(pgrep -x sp1-gpu-server | wc -l) sockets=$(ls /tmp/sp1-cuda-*.sock 2>/dev/null | wc -l)"' 2>&1 | ForEach-Object { ($_ -replace "`0", '') } +"RESULT prove_on $(Stamp) $(Post '/api/prove' @{on=$true})" +"RESULT end $(Stamp)"