diff --git a/docs/design/live-proving/README.md b/docs/design/live-proving/README.md new file mode 100644 index 000000000..cdc1dbf1f --- /dev/null +++ b/docs/design/live-proving/README.md @@ -0,0 +1,13 @@ +# Live page: blocks, finality and proving strips (4 October 2026) + +Screenshots of `site/live.html` after the three-strip layout (one time axis, newest at the right: BLOCKS lanes, a FINALITY bar, a PROVING strip with one cell per shard of every chain block). Captured from the page served locally against two observers' tables, with the installed Chromium test build (never the owner's Chrome), device scale 2. + +| File | What it shows | +|---|---| +| `devnet-before-activation-desktop.png`, `devnet-before-activation-mobile.png` | The real devnet (a read-only second observer on the observer node 28640, proving data from the Mac app's node on the proving build, 26800): `proving_v0_activation_daa` unset, so the proving strip carries the one honest line "Proving layer: not yet activated on this devnet; activation not set". 1280 px and 375 px. | +| `testnet-active-desktop.png` | The private 3-node proving network (`tools/proving-v0/run.mjs --network-only`, activation at DAA 60) at the default adaptive scale: every chain block's planned shard cell, the lag label at the left when proofs land beyond the visible window. | +| `testnet-active-window300-desktop.png`, `testnet-active-window300-mobile.png` | The same network through the diagnostic `?window=300` view (300 s on the axis; the API returns the same span): paid cells in the prover's colour with the tick, the dashed "proofs land N behind the tip" line, the tooltip of a paid cell (block, shard, prover, lag, payout). | + +Frame budget (playwright, 6 s samples, `requestAnimationFrame` wrapped): 1280 px with 110 blocks on screen avg 0.63 ms, max 1.0 ms; 375 px avg 0.49 ms, max 0.9 ms. Budget 10 ms. + +What the CPU test prover showed (observer rows, `provtest_live_proofs`): block 155 shard 0 paid by chain block 405, lag 259 DAA (the compressed proof took 252 s on the loaded Mac), payout 634,116,040,000,000,000 wei; block 154 in `proving` while node 0's verifier queued behind the same CPU. Nothing here is a proving-time number. diff --git a/docs/design/live-proving/devnet-before-activation-desktop.png b/docs/design/live-proving/devnet-before-activation-desktop.png new file mode 100644 index 000000000..e1f65fbe5 Binary files /dev/null and b/docs/design/live-proving/devnet-before-activation-desktop.png differ diff --git a/docs/design/live-proving/devnet-before-activation-mobile.png b/docs/design/live-proving/devnet-before-activation-mobile.png new file mode 100644 index 000000000..606e9d6a9 Binary files /dev/null and b/docs/design/live-proving/devnet-before-activation-mobile.png differ diff --git a/docs/design/live-proving/testnet-active-desktop.png b/docs/design/live-proving/testnet-active-desktop.png new file mode 100644 index 000000000..f2165dc83 Binary files /dev/null and b/docs/design/live-proving/testnet-active-desktop.png differ diff --git a/docs/design/live-proving/testnet-active-window300-desktop.png b/docs/design/live-proving/testnet-active-window300-desktop.png new file mode 100644 index 000000000..4aa5b8b80 Binary files /dev/null and b/docs/design/live-proving/testnet-active-window300-desktop.png differ diff --git a/docs/design/live-proving/testnet-active-window300-mobile.png b/docs/design/live-proving/testnet-active-window300-mobile.png new file mode 100644 index 000000000..f212b0f31 Binary files /dev/null and b/docs/design/live-proving/testnet-active-window300-mobile.png differ diff --git a/site/api/live.mjs b/site/api/live.mjs index 411fdffa4..10b1221af 100644 --- a/site/api/live.mjs +++ b/site/api/live.mjs @@ -1,9 +1,12 @@ // Igneum live devnet feed. GET /api/live returns what tools/observer wrote to Neon: -// {now, state, blocks (last 90 s), miners (last 10 min), events (last 30), finality (checkpoints, newest lock)}. -// Four queries, all on indexed columns. Cached one second at the edge. +// {now, state, blocks (last 90 s, or ?window=N up to 300), miners (last 10 min), events (last 30), finality (checkpoints, newest lock), +// proving (activation and 10-minute counts; every chain block carries shards: [{i, state, prover, lag, payout}])}. +// Five queries, all on indexed columns. Cached one second at the edge. +// LIVE_TABLE_PREFIX (default empty) reads a test observer's tables (fintest_live_*, provtest_live_*) instead of the devnet's. // Zero dependencies: Neon's HTTP SQL endpoint over Node's built-in fetch. const STALE_AFTER_S = 30; +const T = (process.env.LIVE_TABLE_PREFIX || '').replace(/[^a-z0-9_]/gi, ''); function neon() { const url = process.env.DATABASE_URL; @@ -34,25 +37,33 @@ export default async function handler(req, res) { } try { const sql = neon(); - const [head, blockRows, minerRows, checkpointRows] = await Promise.all([ + // ?window=N seconds of blocks (30 to 300, default 90): the page's diagnostic long view; the row cap keeps a busy DAG bounded + const q = new URL(req.url || '/', 'http://x').searchParams; + const windowS = Math.max(30, Math.min(300, Number(q.get('window')) || 90)); + const [head, blockRows, minerRows, checkpointRows, proofRows] = await Promise.all([ sql(`SELECT now() AS now, - (SELECT row_to_json(s) FROM live_state s WHERE s.id = 1) AS state, - (SELECT json_agg(e) FROM (SELECT ts, kind, text FROM live_events ORDER BY ts DESC LIMIT 30) e) AS events`), + (SELECT row_to_json(s) FROM ${T}live_state s WHERE s.id = 1) AS state, + (SELECT json_agg(e) FROM (SELECT ts, kind, text FROM ${T}live_events ORDER BY ts DESC LIMIT 30) e) AS events`), sql(`SELECT hash, blue_score, daa_score, timestamp_ms, parent_hashes, is_chain_block, vote_key_hash, received_at, color - FROM live_blocks - WHERE received_at > now() - interval '90 seconds' + FROM ${T}live_blocks + WHERE received_at > now() - ($1 || ' seconds')::interval ORDER BY received_at DESC - LIMIT 600`), + LIMIT 2000`, [String(windowS)]), sql(`SELECT vote_key_hash, count(*)::int AS blocks, max(received_at) AS last_seen, max(engine) AS engine, max(miner_address) AS address - FROM live_blocks + FROM ${T}live_blocks WHERE received_at > now() - interval '10 minutes' AND vote_key_hash IS NOT NULL GROUP BY vote_key_hash ORDER BY blocks DESC, last_seen DESC`), // Finality v2: the most recent checkpoints; an older observer has no table yet, so this query may fail sql(`SELECT index, hash, blue_score, daa_score, state, signed_weight, active_weight, total_weight, fraction_active, fraction_total, votes_seen, voters, locked_at, updated_at - FROM live_checkpoints + FROM ${T}live_checkpoints ORDER BY index DESC LIMIT 40`).catch(() => []), + // Proving v0: the planned shards of the chain blocks in the window and what became of them (an older observer has no table) + sql(`SELECT block_hash, shard, shards, state, prover, lag_daa, payout_wei, pgas + FROM ${T}live_proofs + WHERE received_at > now() - ($1 || ' seconds')::interval + ORDER BY block_hash, shard`, [String(windowS + 30)]).catch(() => []), ]); const now = new Date(head[0].now).getTime(); @@ -106,6 +117,23 @@ export default async function handler(req, res) { checkpoints, }; + // proving v0: one cell per planned shard; payout in IGN (wei / 1e18) to four decimals, never an address (R4.6.8) + const shardsOf = new Map(); + for (const r of proofRows) { + const k = short(r.block_hash); + if (!shardsOf.has(k)) shardsOf.set(k, []); + let payout = null; + if (r.payout_wei) { try { payout = Number(BigInt(r.payout_wei) / 100000000000000n) / 10000; } catch { payout = null; } } + shardsOf.get(k).push({ i: num(r.shard), n: num(r.shards), state: r.state, prover: r.prover || null, lag: num(r.lag_daa), payout, pgas: num(r.pgas) }); + } + const pv = (s && s.proving) || null; + const proving = pv && pv.supported ? { + supported: true, active: !!pv.active, activation_daa: num(pv.activation_daa), tip_daa: num(pv.tip_daa), verifier: pv.verifier || null, pool: pv.pool || null, + paid_shards_total: num(pv.paid_shards_total), shard_budget_pgas: num(pv.shard_budget_pgas), + blocks_10m: num(pv.blocks_10m), blocks_fully_proven_10m: num(pv.blocks_fully_proven_10m), shards_proven_10m: num(pv.shards_proven_10m), shards_paid_10m: num(pv.shards_paid_10m), + median_proof_lag_s: num(pv.median_proof_lag_s), provers_10m: num(pv.provers_10m), + } : { supported: false, reason: pv ? (pv.reason || 'node without proving') : 'observer has not reported the proving layer' }; + const blocks = blockRows.reverse().map(b => ({ hash: short(b.hash), blue_score: num(b.blue_score), @@ -120,6 +148,9 @@ export default async function handler(req, res) { // Finality v2: a locked checkpoint block, and whether the block sits at or before the newest lock locked: lockedHashes.has(short(b.hash)), final: !!(newestLock && b.is_chain_block && num(b.blue_score) <= newestLock.blue_score), + // Proving v0: the chain block's shards (planned, proving, verified, paid) and whether every shard is proven + shards: shardsOf.get(short(b.hash)) || [], + proven: (shardsOf.get(short(b.hash)) || []).length > 0 && (shardsOf.get(short(b.hash)) || []).every(x => x.state === 'verified' || x.state === 'paid'), })); const miners = minerRows.map(m => ({ @@ -133,7 +164,7 @@ export default async function handler(req, res) { // peer events never carry an address on the public API (review round 4, R4.6.2); older stored rows are redacted here too const events = (head[0].events || []).map(e => ({ ts: e.ts, kind: e.kind, text: String(e.text).replace(/\b\d{1,3}(\.\d{1,3}){3}(:\d+)?\b/g, 'a peer') })); - return res.status(200).json({ ok: true, now: new Date(now).toISOString(), state, blocks, miners, events, finality }); + return res.status(200).json({ ok: true, now: new Date(now).toISOString(), state, blocks, miners, events, finality, proving }); } catch (e) { res.setHeader('Cache-Control', 'no-store'); return res.status(500).json({ ok: false, error: String(e.message || e) }); diff --git a/site/index.html b/site/index.html index edb6a1bf8..e6c9f9067 100644 --- a/site/index.html +++ b/site/index.html @@ -652,6 +652,8 @@ footer .wrap{padding-block:48px 32px} if(lk){x.strokeStyle='rgba(242,84,27,0.55)';x.lineWidth=1.5;x.setLineDash([5,6]);x.beginPath();x.moveTo(lk.x,6);x.lineTo(lk.x,H-6);x.stroke();x.setLineDash([]);var topBusy=blocks.some(function(q){return Math.abs(q.x-lk.x)0){var g=x.createRadialGradient(b.x,b.y,0,b.x,b.y,S*1.6);g.addColorStop(0,'rgba(242,84,27,'+(0.45*b.glow)+')');g.addColorStop(1,'rgba(242,84,27,0)');x.fillStyle=g;x.beginPath();x.arc(b.x,b.y,S*1.6,0,Math.PI*2);x.fill();} if(b.state==='proven'){x.fillStyle='#F2541B';rr(b.x-h,b.y-h,S,S,S*0.24);x.fill();} else{x.fillStyle='#0C0C0E';rr(b.x-h,b.y-h,S,S,S*0.24);x.fill();x.strokeStyle=b.state==='proving'?'rgba(255,179,92,0.9)':'rgba(58,58,66,1)';x.lineWidth=2;rr(b.x-h,b.y-h,S,S,S*0.24);x.stroke(); @@ -670,9 +672,11 @@ footer .wrap{padding-block:48px 32px} var label=document.getElementById('viz-label'),labelHTML=label?label.innerHTML:''; function laneFor(m){if(!(m in liveLanes))liveLanes[m]=Object.keys(liveLanes).length%lanes;return liveLanes[m];} function setLive(on){if(on===liveMode)return;liveMode=on;if(label)label.innerHTML=on?'live devnet':labelHTML;} - live.sub(function(fresh){setLive(live.on);if(!live.on)return; + live.sub(function(fresh,d){setLive(live.on);if(!live.on)return; fresh.forEach(function(b){liveQ.push({hash:b.hash,parents:b.parents||[],lane:laneFor(b.miner||'?')});}); - if(liveQ.length>6)liveQ=liveQ.slice(-6);}); + if(liveQ.length>6)liveQ=liveQ.slice(-6); + // the glint above: only the per-block shards data, only while the proving layer is active + if(d&&d.proving&&d.proving.supported&&d.proving.active)d.blocks.forEach(function(b){if(b.proven){var q=byHash[b.hash];if(q)q.sealed=true;}});}); })(); // one fetch on load; only if the observer is fresh does the 2 s poll start (it feeds the hero, the strip and the chain scene) pullLive(true).then(function(on){if(on)setInterval(function(){pullLive(false);},2000);}); diff --git a/site/live.html b/site/live.html index 1cbb86be4..345a89807 100644 --- a/site/live.html +++ b/site/live.html @@ -94,9 +94,12 @@ p{margin:0} .viz-head{display:flex;flex-wrap:wrap;justify-content:space-between;align-items:center;gap:10px 20px;margin-bottom:12px} .viz-stats{display:flex;flex-wrap:wrap;gap:14px 18px;font-size:12px;color:var(--ash)} .viz-stats b{color:var(--bone);font-weight:500;font-variant-numeric:tabular-nums} -#dag{display:block;width:100%;height:clamp(240px,38vw,380px);border-radius:12px;background:var(--obsidian);touch-action:pan-y} -.legend{display:flex;flex-wrap:wrap;gap:16px 20px;margin-top:16px;font-size:13px;color:var(--ink-2)} -.legend span{display:inline-flex;align-items:center;gap:8px} +#dag{display:block;width:100%;height:calc(clamp(240px,38vw,380px) + 18px + 64px);border-radius:12px;background:var(--obsidian);touch-action:pan-y} +@media (max-width:640px){#dag{height:calc(clamp(240px,38vw,380px) + 18px + 50px)}} +.legend{display:flex;flex-direction:column;gap:8px;margin-top:16px;font-size:13px;color:var(--ink-2)} +.legend div{display:flex;flex-wrap:wrap;align-items:center;gap:6px 10px} +.legend .lk{font-family:'IBM Plex Mono',monospace;font-size:11px;letter-spacing:.12em;text-transform:uppercase;color:var(--ash);min-width:64px} +.legend span{display:inline-flex;align-items:center;gap:6px;white-space:nowrap} .sw{width:14px;height:14px;border-radius:4px;display:inline-block} .two{display:grid;gap:var(--gap);grid-template-columns:1fr;margin-top:var(--gap)} @@ -160,7 +163,7 @@ footer{border-top:1px solid var(--line);padding-block:48px 32px}
devnet v0

Live devnet

-

A node is read every two seconds. Every block below is real: its parents, its miner, whether it sits on the selected chain, and whether it is a locked checkpoint. A checkpoint locks when the finality rule's quorum signs it. Proving is not in the devnet yet.

+

A node is read every two seconds. Every block below is real: its parents, its miner, whether it sits on the selected chain, and whether it is a locked checkpoint. A checkpoint locks when the finality rule's quorum signs it. Once the proving layer is activated, every chain block's shards show as they are planned, proven and paid.

@@ -174,23 +177,16 @@ footer{border-top:1px solid var(--line);padding-block:48px 32px}
Peers
0
mempool 0
-
+
-
block dag
-
on screen 0chain 0side 0newest blue 0locked none
+
blocks, finality, proving
+
on screen 0chain 0identities 0last lock noneproven n/a
- chain block - included (blue): paid - pending - excluded (red) - locked checkpoint, final - just arrived - selected chain - first parent - 5d3fd79cone lane per miner, colour from its id - hover or tap a block for its id, scores and parents +
blockschainincluded, paidpendingexcludedjust arrivedselected chainone lane per miner, colour from its id; hover or tap a block
+
finalityfinallocked checkpointproposedreads "paused" whenever under two thirds of the weight is signing
+
provingshard plannedprovingverified, prover colourpaidwhere proofs land behind the tiphover or tap a cell
@@ -264,35 +260,41 @@ footer{border-top:1px solid var(--line);padding-block:48px 32px} function rel(ms){var s=Math.max(0,Math.round(ms/1000));if(s<60)return s+' s ago';var m=Math.floor(s/60);if(m<60)return m+' min ago';var h=Math.floor(m/60);if(h<24)return h+' h '+(m%60)+' min ago';return Math.floor(h/24)+' d ago';} function esc(s){return String(s).replace(/&/g,'&').replace(//g,'>');} - // ---------- DAG scene ---------- - // One canvas, one requestAnimationFrame loop. Lanes: one per miner in the visible window, most blocks on top, - // the least active share an "others" lane. Labels live in a left gutter, never on blocks. Edges go under blocks. - var c=$('dag'),x=c.getContext('2d'),W=0,H=0,S=22,G=78,narrow=false,level=0,maxLanes=8; - var blocks=[],byHash={},miners={},lanes=[],laneN=1,othersN=0; + // ---------- three strips on one time axis ---------- + // One canvas, one requestAnimationFrame loop, newest at the right. BLOCKS (lanes: one per miner in the visible window, + // most blocks on top, the least active share an "others" lane; labels in a left gutter; edges under blocks), then a + // FINALITY bar, then a PROVING strip with one cell per shard of every chain block. All three share px-per-second. + var c=$('dag'),x=c.getContext('2d'),W=0,H=0,BH=0,S=22,G=78,narrow=false,level=0,maxLanes=8,FIN_H=18,PROV_H=64; + var blocks=[],byHash={},miners={},lanes=[],laneN=1,othersN=0,fin=null,prov=null,idents=0; var LV=[{S:22,s:14,pps:40,npps:14},{S:16,s:12,pps:26,npps:10},{S:12,s:10,pps:18,npps:8}]; var pxPerSec=40,targetPx=40,bps=0,live=false,serverNow=0,serverAt=0,frozenAt=null,note=''; - var running=false,needSize=true,inView=true,ptrX=-1,ptrY=-1,ptrTouch=false,pin=null,laneLast=[],laneRun=[],lagNote=''; + // ?window=N (30 to 300 s): a fixed long view for diagnosis, the API returns the same span; the default stays the adaptive scale + var fixedWindow=Math.max(0,Math.min(300,Number(new URLSearchParams(location.search).get('window'))||0));if(fixedWindow&&fixedWindow<30)fixedWindow=30; + var running=false,needSize=true,inView=true,ptrX=-1,ptrY=-1,pin=null,laneLast=[],laneRun=[],lagNote=''; var TOP=20,BOT=12; - // 12 hues that sit beside ember on obsidian; the index comes from the short id, so a miner keeps its colour across reloads + // 12 hues that sit beside ember on obsidian; the index comes from the short id, so a miner (or a prover) keeps its colour across reloads var PAL=['#5CB8FF','#3FD39A','#F5C542','#C49BFF','#FF7AA8','#8FD15F','#FFB08A','#6F8CFF','#D9E25A','#FF8FD6','#4FD8D8','#E0A96D']; function colour(id){var v=0;for(var i=0;i>>0;return PAL[v%PAL.length];} function jitter(h){var v=0;for(var i=0;i<8;i++)v=(v*31+h.charCodeAt(i))%1000;return v/1000-0.5;} function vnow(){if(frozenAt!==null)return frozenAt;return serverNow+(performance.now()-serverAt);} - function laneY(l){var u=H-TOP-BOT,p=u/Math.max(1,laneN);return TOP+(l+0.5)*p;} + function laneY(l){var u=BH-TOP-BOT,p=u/Math.max(1,laneN);return TOP+(l+0.5)*p;} function size(){ needSize=false;W=c.clientWidth;H=c.clientHeight;if(!W||!H)return; c.width=Math.round(W*dpr);c.height=Math.round(H*dpr);x.setTransform(dpr,0,0,dpr,0,0); - narrow=W<560;G=narrow?36:80;applyLevel(); + narrow=W<560;G=narrow?36:80;PROV_H=narrow?50:64;BH=H-FIN_H-PROV_H;applyLevel(); + for(var id in miners)miners[id].y=laneY(miners[id].lane); // a resize is a layout change: lanes snap, nothing eases from a stale position } function applyLevel(){ var L=LV[level];S=narrow?L.s:L.S;targetPx=narrow?L.npps:L.pps; - maxLanes=Math.max(2,Math.min(14,Math.floor((H-TOP-BOT)/(S*1.7)))); + if(fixedWindow){targetPx=(W-G)/fixedWindow;S=fixedWindow>120?(narrow?8:10):(narrow?L.s:LV[1].S);} + maxLanes=Math.max(2,Math.min(14,Math.floor((BH-TOP-BOT)/(S*1.7)))); layoutLanes(); } function pickLevel(){ // projected blocks on screen at the base scale drives the step, with hysteresis so it never flickers var proj=bps*(W-G)/(narrow?LV[0].npps:LV[0].pps),up=[48,110],down=[36,85],next=level; if(level<2&&proj>up[level])next=level+1;else if(level>0&&proj120?2:1; if(next!==level){level=next;applyLevel();} } function layoutLanes(){ @@ -302,16 +304,18 @@ footer{border-top:1px solid var(--line);padding-block:48px 32px} laneN=Math.max(1,Math.min(maxLanes,ids.length)); othersN=ids.length>laneN?ids.length-(laneN-1):0; lanes=[]; - ids.forEach(function(id,i){var m=miners[id],l=othersN&&i>=laneN-1?laneN-1:i;m.lane=l;if(m.y===undefined)m.y=laneY(l);if(!othersN||l=laneN-1?laneN-1:i;m.lane=l;if(m.y===undefined&&BH>0)m.y=laneY(l);if(!othersN||lcut;if(!keep)delete byHash[b.hash];return keep;}); blocks.sort(function(a,b){return a.ts-b.ts;}); for(var id in miners)miners[id].n=0; @@ -319,42 +323,53 @@ footer{border-top:1px solid var(--line);padding-block:48px 32px} for(id in miners)if(!miners[id].n)delete miners[id]; bps=d.state.blocks_per_second_60s||0; pickLevel();layoutLanes(); - var vis=vnow()-((W-G)/Math.max(8,targetPx))*1000,on=0,chain=0; - blocks.forEach(function(b){if(b.ts>vis){on++;if(b.chain)chain++;}}); - $('c-screen').textContent=on;$('c-chain').textContent=chain;$('c-side').textContent=on-chain; - $('c-newest').textContent=blocks.length?compact(blocks[blocks.length-1].blue_score):'0'; - var f=d.finality;$('c-locked').textContent=f&&f.latest_locked_index?('#'+f.latest_locked_index+' at blue '+compact(f.latest_locked_blue_score)):(f&&f.supported?'none yet':'n/a'); + var vis=vnow()-((W-G)/Math.max(8,targetPx))*1000,on=0,chain=0,planned=0,proven=0; + blocks.forEach(function(b){if(b.ts>vis){on++;if(b.chain){chain++;if(b.shards.length){planned++;if(b.proven)proven++;}}}}); + $('c-screen').textContent=on;$('c-chain').textContent=chain;$('c-idents').textContent=idents; + var f=fin;$('c-locked').textContent=f&&f.latest_locked_index?('#'+f.latest_locked_index+(f.checkpoints&&f.checkpoints.length?', '+lockAge(f):'')):(f&&f.supported?'none yet':'n/a'); + $('c-proven').textContent=prov&&prov.supported&&prov.active?(proven+'/'+planned):(prov&&prov.supported?'not active':'n/a'); if(!running&&!document.hidden)kick(); } + function lockAge(f){var cps=f.checkpoints||[],lk=null;for(var i=cps.length-1;i>=0;i--)if(cps[i].state==='locked'){lk=cps[i];break;} + if(!lk||!lk.locked_at)return '';return rel(vnow()-new Date(lk.locked_at).getTime());} function rrect(px,py,w,h,r){x.beginPath();x.moveTo(px+r,py);x.lineTo(px+w-r,py);x.quadraticCurveTo(px+w,py,px+w,py+r);x.lineTo(px+w,py+h-r);x.quadraticCurveTo(px+w,py+h,px+w-r,py+h);x.lineTo(px+r,py+h);x.quadraticCurveTo(px,py+h,px,py+h-r);x.lineTo(px,py+r);x.quadraticCurveTo(px,py,px+r,py);x.closePath();} // a cubic with horizontal tangents; the control offset is clamped so a link across many lanes reads as a gentle S, never a loop function link(a,b){var d=Math.max(S*0.6,Math.min(48,Math.abs(b.px-a.px)*0.5));x.moveTo(a.px,a.py);x.bezierCurveTo(a.px+d,a.py,b.px-d,b.py,b.px,b.py);} + // shard cells: a column under each chain block, shard 0 at the top; sizes from the zoom level, never wider than the block + function cellGeom(n){var avail=PROV_H-12,gap=2,ch=Math.min(narrow?7:9,Math.floor((avail-(n-1)*gap)/Math.max(1,n))),cw=Math.max(3,Math.min(narrow?10:12,Math.round(S*0.7),Math.round(pxPerSec*0.85)));return {ch:ch,cw:cw,gap:gap,top:BH+FIN_H+6+Math.max(0,Math.floor((avail-(n*ch+(n-1)*gap))/2))};} function hit(){ - var best=null,bd=S*0.9,qx=ptrX,qy=ptrY; - if(pin){var p=byHash[pin];return p&&p.on?p:null;} + // returns {b, i}: a block (i = -1) in the blocks strip, or a shard cell (i = shard index) in the proving strip + var qx=ptrX,qy=ptrY; + if(pin){var p=byHash[pin.h];return p&&p.on?{b:p,i:pin.i}:null;} if(qx<0)return null; - for(var i=0;i=BH+FIN_H&&provOn()){var bb=null,bdx=1e9; + for(i=0;iMath.max(g.cw,S*0.5))return null; + var row=Math.floor((qy-g.top)/(g.ch+g.gap));if(row<0||row>=bb.shards.length)return null;return {b:bb,i:row};} + return null; } + function provOn(){return !!(prov&&prov.supported&&prov.active);} function draw(now){ if(needSize)size();if(!W||!H)return; pxPerSec+=(targetPx-pxPerSec)*0.06; var t=vnow(),xr=W-S,i,b,m,n=blocks.length,id; - for(id in miners){m=miners[id];var ty=laneY(m.lane);m.y+=(ty-m.y)*0.12;if(Math.abs(ty-m.y)<0.05)m.y=ty;} + for(id in miners){m=miners[id];var ty=laneY(m.lane);if(m.y===undefined)m.y=ty;m.y+=(ty-m.y)*0.12;if(Math.abs(ty-m.y)<0.05)m.y=ty;} for(i=0;iG-S;} x.clearRect(0,0,W,H); - // final region: everything left of the newest locked checkpoint + // the newest locked checkpoint on screen: its x bounds the final region of the finality bar var lk=null,fx=-1; for(i=0;ilk.ts))lk=b;if(b.final&&b.px>fx)fx=b.px;} if(lk)fx=lk.px; - if(fx>G){x.fillStyle='rgba(242,84,27,0.06)';x.fillRect(G,0,fx-G,H);x.strokeStyle='rgba(255,179,92,0.55)';x.lineWidth=1;x.setLineDash([4,5]);x.beginPath();x.moveTo(fx,4);x.lineTo(fx,H-4);x.stroke();x.setLineDash([]);} // lane rails x.strokeStyle='rgba(42,42,48,0.75)';x.lineWidth=1;x.beginPath();for(i=0;iG-S*2)link(q,b);} @@ -364,8 +379,8 @@ footer{border-top:1px solid var(--line);padding-block:48px 32px} for(i=0;iG-S*2)link(prev,b);prev=b;} x.stroke(); // hovered block: all of its parent edges, bright - var hv=hit(); - if(hv){x.strokeStyle='rgba(255,179,92,0.9)';x.lineWidth=1.5;x.beginPath();for(i=0;i0?'rgba(255,179,92,0.95)':(b.color==='red'?'rgba(154,154,158,0.6)':'rgba(154,154,158,0.85)');x.lineWidth=1.5;rrect(b.px-h,b.py-h,S,S,r);x.stroke(); if(b.color==='red'){x.strokeStyle='rgba(154,154,158,0.9)';x.lineWidth=1.5;x.beginPath();x.moveTo(b.px-h*0.5,b.py+h*0.5);x.lineTo(b.px+h*0.5,b.py-h*0.5);x.stroke();}} if(fl>0&&b.chain){x.strokeStyle='rgba(255,179,92,'+fl+')';x.lineWidth=2;rrect(b.px-h-3,b.py-h-3,S+6,S+6,r+2);x.stroke();} - if(b.locked){x.strokeStyle='#FFB35C';x.lineWidth=2.5;x.beginPath();x.arc(b.px,b.py,S*0.95,0,Math.PI*2);x.stroke();} - if(hv===b){x.strokeStyle='#F4F1EC';x.lineWidth=1.5;rrect(b.px-h-3,b.py-h-3,S+6,S+6,r+2);x.stroke();}} + if(hb===b){x.strokeStyle='#F4F1EC';x.lineWidth=1.5;rrect(b.px-h-3,b.py-h-3,S+6,S+6,r+2);x.stroke();}} x.font='500 10px IBM Plex Mono, monospace'; - // the chain tip carries its short id; nothing else is labelled unless hovered, pinned or locked + // the chain tip carries its short id; nothing else is labelled unless hovered or pinned if(tip){var lt=tip.hash.slice(0,8),lw=x.measureText(lt).width,lh=11,pad=3; // candidate label boxes around the tip: above-right, above-left, below-right, below-left, left, right; the first that - // stays inside the canvas and overlaps no other on-screen block wins, so the id is never drawn over a neighbour + // stays inside the strip and overlaps no other on-screen block wins, so the id is never drawn over a neighbour var cands=[[tip.px+h+2-lw,tip.py-h-6-lh],[tip.px-h-2-lw,tip.py-h-6-lh],[tip.px+h+2-lw,tip.py+h+4],[tip.px-h-2-lw,tip.py+h+4],[tip.px-h-6-lw,tip.py-lh/2],[tip.px+h+6,tip.py-lh/2]]; - var pick=null,ci;for(ci=0;ciW-2||cy<2||cy+lh>H-2)continue;var clash=false; + var pick=null,ci;for(ci=0;ciW-2||cy<2||cy+lh>BH-2)continue;var clash=false; for(var bi=0;biob.px-h&&cy-padob.py-h)clash=true;} if(!clash)pick=cands[ci];} if(!pick)pick=cands[0];x.fillStyle='rgba(244,241,236,0.92)';x.textAlign='left';x.textBaseline='top';x.fillText(lt,Math.max(G+2,Math.min(W-2-lw,pick[0])),Math.max(2,pick[1]));x.textBaseline='alphabetic';} - if(fx>G){var ft=lk?'final #'+lk.hash.slice(0,8):'final',fw=x.measureText(ft).width;x.fillStyle='#FFB35C'; - if(fx-G>fw+14){x.textAlign='right';x.fillText(ft,fx-6,12);}else{x.textAlign='left';x.fillText(ft,fx+6,12);}} - // tooltip on the canvas, no DOM - if(hv){var l1=hv.hash.slice(0,8)+' '+(hv.locked?'locked':hv.chain?'chain':hv.color==='blue'?'included, paid':hv.color==='red'?'excluded':'pending'),l2='blue '+compact(hv.blue_score)+' daa '+compact(hv.daa),l3=hv.parents.length+(hv.parents.length===1?' parent':' parents')+' '+hv.miner; - var tw=Math.max(x.measureText(l1).width,x.measureText(l2).width,x.measureText(l3).width)+20,th=52,tx=hv.px+h+10,tty=hv.py-th/2; - if(tx+tw>W-4)tx=hv.px-h-10-tw;if(txG){x.fillStyle='rgba(242,84,27,0.22)';x.fillRect(G,yb,fx-G,hb2);} + // proposed: from the newest lock to the newest proposed checkpoint on screen, faint; every checkpoint a tick + var cps=fin.checkpoints||[],px2=-1,ci2,cb; + for(ci2=0;ci2px2)px2=cb.px;} + if(px2>Math.max(G,fx)){x.fillStyle='rgba(154,154,158,0.10)';x.fillRect(Math.max(G,fx),yb,px2-Math.max(G,fx),hb2);} + for(ci2=0;ci2G){var tw0=x.measureText(ftxt).width,lx0=fx+8+tw0G){x.strokeStyle='rgba(255,179,92,0.7)';x.lineWidth=1;x.setLineDash([3,4]);x.beginPath();x.moveTo(lx,y1+3);x.lineTo(lx,H-3);x.stroke();x.setLineDash([]); + var ltx=narrow?'lag '+lagTxt:'proofs land '+lagTxt+' behind the tip',ltw=x.measureText(ltx).width;x.fillStyle='#FFB35C';x.textAlign=lx+6+ltwW-4)tx=hb.px-h-10-tw;if(tx=3){setStatus(false,'api unreachable');if(frozenAt===null)frozenAt=vnow();note='api unreachable, scene frozen';}});} + function poll(){fetch('/api/live'+(fixedWindow?'?window='+fixedWindow:''),{cache:'no-store'}).then(function(r){if(!r.ok)throw new Error(r.status);return r.json();}).then(function(d){if(!d.ok)throw new Error(d.error);failures=0;render(d);}).catch(function(){failures++;if(failures>=3){setStatus(false,'api unreachable');if(frozenAt===null)frozenAt=vnow();note='api unreachable, scene frozen';}});} poll();setInterval(poll,2000); })(); diff --git a/tools/observer/README.md b/tools/observer/README.md index 8104d787d..2e1c60a85 100644 --- a/tools/observer/README.md +++ b/tools/observer/README.md @@ -16,6 +16,7 @@ Environment, every value optional: | `DATABASE_URL` | read from `~/.config/igneum/env` | Neon connection string. Never commit it. | | `LIVE_RETAIN_HOURS` | `24` | Hours of blocks kept in `live_blocks`. Older rows are deleted once a minute. | | `LIVE_TABLE_PREFIX` | empty | Prefix for every table name, so a test observer against a test network can write `fintest_live_*` without touching the site. | +| `IGNEUM_EVM_RPC` | `http://127.0.0.1:26800` | The execution layer's JSON-RPC (http) of a node on the proving build, for `igneum_getShardPlan`, `igneum_getProofRecords` and `igneum_getProvingStatus`. The default is the Mac app's node; the observer node itself has no EVM listener yet (start it with `--evm-rpclisten=127.0.0.1:` once it runs the proving build and point this at it). A node without the RPCs (method not found) gives `proving = {supported: false}`, rechecked every 5 minutes; an unreachable endpoint is retried every 20 s. | A node started by another tool may listen on gRPC only. Then run your own non-mining peer with a JSON listener, on ports that do not clash with the devnet's (gRPC 26610, P2P 26611): @@ -32,6 +33,7 @@ IGNEUM_RPC=ws://127.0.0.1:28640 node tools/observer/observer.mjs - Decodes the miner address from the coinbase payload script (same bech32 variant as `crypto/addresses`). The fork's `vote_key_hash` header field is stored per block; the first 8 hex characters are the miner's short id on the site. - `engine` is the miner's tag in the coinbase extra data after the node's version prefix. The node exposes no engine name over RPC, so this is null on devnet v0. - When the node refuses the hash-rate estimate (it needs a 1,000-block window) the observer reports blue work added per second over the last 10 minutes instead. +- Proving v0 (4 Oct 2026, spec 7.7): every chain block (the `isChainBlock` flag of a new block, or a `virtualChainChanged` addition) has its shard plan read over `IGNEUM_EVM_RPC` as `{blockHash}` (the chain block hash is the same hash on both layers), one `live_proofs` row per shard in state `planned`; a plan the EVM node has not executed yet is retried with a growing delay (up to 30 tries). The proof records of the chain blocks of the last 10 minutes are polled in rotation (80 blocks per 2 s tick while active, 10 before activation, four calls in flight); a shard moves to `proving` (a record in the node's pool), `verified` (the SP1 proof verified by the node's verifier, or the record carried by a block and checked by consensus, which is what happens before activation) or `paid` (a carrying segment paid it). `lag_daa` is the carrier's DAA score minus the block's; `prover` is the first 8 hex characters of the record's vote key hash. A block whose every shard is paid, or older than 10 minutes, leaves the rotation. Events: `proving` (activation reached, first paid shard seen) and `prover_seen` (one per prover per run). A reorg drops the removed chain blocks from the rotation. - Finality v2 (3 Oct 2026): subscribes to `FinalityLock` (the node's lock event) and polls `getFinalityCheckpoints` every 2 s and `getFinalityWeights` every 10 s. Every checkpoint the node reports is upserted into `live_checkpoints`; a checkpoint turning `locked` writes the event `checkpoint N locked (xx% of weight, yy% of active, v votes of n voters) at block h`. The weights snapshot (total, active, per key) goes into `live_state.finality`. A node from before the finality layer answers the RPC with an error; the observer then logs once and skips finality. ## Tables @@ -44,8 +46,10 @@ Created on start if missing. | `live_state` | one row, updated every 2 s | `block_count`, `header_count`, `blue_score`, `difficulty`, `hashes_per_second_estimate`, `peers`, `mempool`, `node_version`, `network`, `blocks_60s`, `blocks_per_minute` (60 pairs of minute epoch ms and count), `observer_started_at`, `updated_at` | | `live_events` | one per event, kept 7 days | `ts`, `kind`, `text`. Kinds: `observer`, `miner_seen`, `miner_quiet`, `miner_back`, `peer_joined`, `peer_left`, `difficulty` (step over 5%), `checkpoint_locked`. | | `live_checkpoints` | one per checkpoint index, kept 7 days | `index`, `hash`, `blue_score`, `daa_score`, `state` (proposed, certified, locked), `signed_weight`, `active_weight`, `total_weight`, `fraction_active`, `fraction_total`, `votes_seen`, `voters`, `aggregators` (key hashes whose sortition proof made them aggregators), `locked_at`, `first_seen_at`, `updated_at`. | +| `live_proofs` | one per planned shard of a chain block, kept `LIVE_RETAIN_HOURS` | `block_hash`, `shard`, `shards` (in the plan), `block_number`, `block_daa`, `block_ts`, `pgas`, `state` (planned, proving, verified, paid), `prover` (id8), `verified`, `carried_by`, `carrier_number`, `carrier_daa`, `lag_daa`, `payout_wei`, `received_at`, `updated_at`. Primary key `(block_hash, shard)`, index on `received_at`. | +| `live_state.proving` | jsonb, updated every 2 s | `supported` (false with `reason` when the node has no proving RPCs or the endpoint is unreachable), `active`, `activation_daa`, `tip_daa`, `verifier`, `pool` (entries, pending, verified, failed), `paid_shards_total`, `shard_budget_pgas`, `blocks_10m`, `blocks_fully_proven_10m`, `shards_proven_10m`, `shards_paid_10m`, `median_proof_lag_s` (median `lag_daa` of the last 10 minutes; the devnet targets one DAA step per second), `provers_10m`, `open_blocks`, `pending_plans`, `evm_rpc`. | | `live_state.finality` | jsonb, updated every 2 s | `params`, `chain_id`, `next_index`, `finality_active`, `latest_locked_index`, `latest_locked_hash`, `latest_locked_blue_score`, `weights` (`total_weight`, `active_weight`, `voters`, `keys[]` with `id`, `blocks`, `voter`, `participation`, `stripped_until_daa`, `revealed`). | ## Reading it -`site/api/live.mjs` serves `/api/live` from these tables in three indexed queries. `site/live.html` polls it every 2 s. The site shows OFFLINE when `live_state.updated_at` is older than 30 s. +`site/api/live.mjs` serves `/api/live` from these tables in five indexed queries (`proving` from `live_state.proving`; every block carries `shards: [{i, n, state, prover, lag, payout, pgas}]` and `proven`). `LIVE_TABLE_PREFIX` on the API reads a test observer's tables. `site/live.html` polls it every 2 s. The site shows OFFLINE when `live_state.updated_at` is older than 30 s. diff --git a/tools/observer/observer.mjs b/tools/observer/observer.mjs index c593d6596..569be0f79 100644 --- a/tools/observer/observer.mjs +++ b/tools/observer/observer.mjs @@ -9,8 +9,17 @@ // DATABASE_URL Neon connection string default: read from ~/.config/igneum/env // LIVE_RETAIN_HOURS hours of blocks to keep default 24 // LIVE_TABLE_PREFIX prefix for every table name default '' (a test observer can write fintest_live_* instead) +// IGNEUM_EVM_RPC execution-layer JSON-RPC (http) of a node on the proving build, for the shard plans and proof +// records default http://127.0.0.1:26800 (the Mac app's node; the observer node has no EVM listener yet) // -// Tables (created on start if missing): live_blocks, live_state, live_events, live_checkpoints, live_certificates. See README.md. +// Tables (created on start if missing): live_blocks, live_state, live_events, live_checkpoints, live_certificates, +// live_proofs. See README.md. +// Proving v0 (4 Oct 2026, spec 7.7): every chain block's shard plan is read over the EVM RPC (igneum_getShardPlan) as +// the block joins the selected chain, one live_proofs row per shard (planned); the proof records of the chain blocks +// of the last 10 minutes are polled in rotation (igneum_getProofRecords, 80 blocks per 2 s tick, four calls in flight) and move a shard to +// proving (a record in the pool), verified (the SP1 proof verified, or the record carried and checked) or paid (a +// carrying block paid it). live_state.proving carries the activation state and the 10-minute counts. A node without +// the proving RPCs (method not found) gives proving = {supported: false}; the observer rechecks every 5 minutes. // Finality v2: subscribes to FinalityLock notifications and polls getFinalityCheckpoints and getFinalityWeights // every 2 s; writes the checkpoints table and emits "checkpoint N locked (xx% of weight)" events. // Light client (site/api/checkpoint.mjs, site/verify/): every certificate a block carries is stored with the voter @@ -31,7 +40,8 @@ import { homedir } from 'node:os'; const RPC = process.env.IGNEUM_RPC || 'ws://127.0.0.1:28610'; const RETAIN_HOURS = Number(process.env.LIVE_RETAIN_HOURS || 24); const T = (process.env.LIVE_TABLE_PREFIX || '').replace(/[^a-z0-9_]/gi, ''); -const TB = `${T}live_blocks`, TS = `${T}live_state`, TE = `${T}live_events`, TC = `${T}live_checkpoints`, TX = `${T}live_certificates`; +const TB = `${T}live_blocks`, TS = `${T}live_state`, TE = `${T}live_events`, TC = `${T}live_checkpoints`, TX = `${T}live_certificates`, TP = `${T}live_proofs`; +const EVM = process.env.IGNEUM_EVM_RPC || 'http://127.0.0.1:26800'; const STATE_EVERY_MS = 2000; const FLUSH_EVERY_MS = 500; const PRUNE_EVERY_MS = 60_000; @@ -91,6 +101,30 @@ async function setupSchema() { `ALTER TABLE ${TS} ADD COLUMN IF NOT EXISTS finality jsonb`, `ALTER TABLE ${TS} ADD COLUMN IF NOT EXISTS observer_lag_s double precision`, `ALTER TABLE ${TS} ADD COLUMN IF NOT EXISTS queue_depth int`, + // Proving v0 (4 Oct 2026, additive): {supported, active, activation_daa, tip_daa, blocks_fully_proven_10m, ...} + `ALTER TABLE ${TS} ADD COLUMN IF NOT EXISTS proving jsonb`, + // One row per planned shard of a chain block (spec 7.7 item 8): the plan as the block joins the chain, then the + // record's progress. prover is the first 8 hex characters of the record's vote key hash (never the full key, R4.6.2). + `CREATE TABLE IF NOT EXISTS ${TP} ( + block_hash text NOT NULL, + shard int NOT NULL, + shards int NOT NULL, + block_number bigint, + block_daa bigint, + block_ts bigint, + pgas bigint, + state text NOT NULL DEFAULT 'planned', + prover text, + verified boolean, + carried_by text, + carrier_number bigint, + carrier_daa bigint, + lag_daa int, + payout_wei text, + received_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (block_hash, shard))`, + `CREATE INDEX IF NOT EXISTS ${TP}_received_at ON ${TP} (received_at)`, `CREATE TABLE IF NOT EXISTS ${TE} ( id bigserial PRIMARY KEY, ts timestamptz NOT NULL DEFAULT now(), @@ -327,6 +361,7 @@ function onBlock(block) { if (mergesets.size > MERGESET_CACHE) mergesets.delete(mergesets.keys().next().value); if (vd.isChainBlock) pendingMerge.set(h.hash, ms); } + if (vd && vd.isChainBlock) queuePlan(h.hash); lastBlockAt = now; blockCountAtLastBlock = lastDagBlockCount; resubscribeAttempts = 0; pendingBlocks.push({ hash: h.hash, blue_score: h.blueScore, daa_score: h.daaScore, timestamp_ms: h.timestamp, @@ -456,20 +491,23 @@ async function tick(rpc) { // finality v2: checkpoints and weights (a node from before the finality layer answers with an error; then null) const finality = await finalityTick(rpc); + // proving v0: activation state and the 10-minute counts (a node without the RPCs gives {supported: false}) + await provingTick(); + const proving = await provingState(); try { await sql(`INSERT INTO ${TS} (id, block_count, header_count, blue_score, difficulty, hashes_per_second_estimate, peers, mempool, - node_version, network, blocks_60s, blocks_per_minute, observer_started_at, updated_at, finality, observer_lag_s, queue_depth) - VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb, $12, now(), $13::jsonb, $14, $15) + node_version, network, blocks_60s, blocks_per_minute, observer_started_at, updated_at, finality, observer_lag_s, queue_depth, proving) + VALUES (1, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11::jsonb, $12, now(), $13::jsonb, $14, $15, $16::jsonb) ON CONFLICT (id) DO UPDATE SET block_count = EXCLUDED.block_count, header_count = EXCLUDED.header_count, blue_score = EXCLUDED.blue_score, difficulty = EXCLUDED.difficulty, hashes_per_second_estimate = EXCLUDED.hashes_per_second_estimate, peers = EXCLUDED.peers, mempool = EXCLUDED.mempool, node_version = EXCLUDED.node_version, network = EXCLUDED.network, blocks_60s = EXCLUDED.blocks_60s, blocks_per_minute = EXCLUDED.blocks_per_minute, observer_started_at = EXCLUDED.observer_started_at, updated_at = now(), - finality = EXCLUDED.finality, observer_lag_s = EXCLUDED.observer_lag_s, queue_depth = EXCLUDED.queue_depth`, + finality = EXCLUDED.finality, observer_lag_s = EXCLUDED.observer_lag_s, queue_depth = EXCLUDED.queue_depth, proving = EXCLUDED.proving`, [dag.blockCount, dag.headerCount, (await rpc.call('getSinkBlueScore', {}).catch(() => ({}))).blueScore ?? null, diff, hps ? hps.networkHashesPerSecond : localHashesPerSecond(), peers.length, info.mempoolSize ?? 0, nodeVersion, network, blocks60s(now), JSON.stringify(blocksPerMinute(now)), STARTED_AT.toISOString(), - finality ? JSON.stringify(finality) : null, lagS === null ? null : Math.round(lagS * 10) / 10, queueDepth]); + finality ? JSON.stringify(finality) : null, lagS === null ? null : Math.round(lagS * 10) / 10, queueDepth, JSON.stringify(proving)]); } catch (e) { log('state write failed', e.message); } } @@ -667,9 +705,170 @@ async function backfillCertificate(rpc, report) { log(`no block within ${seen} of checkpoint ${newest.index} carries its certificate`); } +// ---------- Proving v0 (spec 7.7): shard plans and proof records over the execution layer's JSON-RPC ---------- +// The chain block hash is the same hash on both layers (the executor keys its segments by it), so every lookup is +// {blockHash}. The EVM node may trail the observer's node by a segment or two: a plan that is "not found" is retried +// with a growing delay, up to 30 attempts. A node without the RPCs answers -32601 once and the feed is marked +// unsupported (rechecked every 5 minutes, so an upgraded node is picked up without a restart). +const PLAN_BATCH = 24, RECORD_BATCH_ACTIVE = 80, RECORD_BATCH_INACTIVE = 10, PROOF_WINDOW_MS = 10 * 60_000; +const pendingPlans = new Map(); // chain block hash -> { tries, nextAt } +const openBlocks = new Map(); // chain block hash -> { number, daa, ts, shards: [state...], provers, polledAt, done } +const daaOfNumber = new Map(); // chain block number -> daa score, from the plans fetched (carriers are later chain blocks) +const provers = new Set(); // prover id8s seen this run (one event each) +let provingSupported = null; // null unknown, false = no RPC, true +let provingRecheckAt = 0; +let provingStatus = null; // the last igneum_getProvingStatus answer +let provingReason = null; // why supported is false +let provingWasActive = null; +let plansBusy = false, recordsBusy = false, firstPaidSeen = false; +let lastProvingStats = null, lastProvingStatsAt = 0; +const hx = v => (v === null || v === undefined ? null : (typeof v === 'string' && v.startsWith('0x') ? parseInt(v, 16) : Number(v))); +const weiStr = v => { if (v === null || v === undefined) return null; try { return BigInt(v).toString(); } catch { return null; } }; + +async function evm(method, params = []) { + const r = await fetch(EVM, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }), signal: AbortSignal.timeout(8000) }); + const j = await r.json(); + if (j.error) { const e = new Error(`${method}: ${j.error.message}`); e.code = j.error.code; throw e; } + return j.result; +} + +function queuePlan(hash) { if (!openBlocks.has(hash) && !pendingPlans.has(hash)) pendingPlans.set(hash, { tries: 0, nextAt: 0 }); } +function dropPlan(hash) { pendingPlans.delete(hash); openBlocks.delete(hash); } + +async function provingTick() { + const now = Date.now(); + if (provingSupported === false && now < provingRecheckAt) return; + try { + const s = await evm('igneum_getProvingStatus'); + if (provingSupported !== true) log(`proving RPCs available at ${EVM} (activation ${s.activationDaa === null ? 'not set' : hx(s.activationDaa)}, verifier ${s.verifier})`); + provingSupported = true; provingReason = null; provingStatus = s; + if (provingWasActive === false && s.active) recordEvent('proving', `Proving layer active: payouts start at DAA ${hx(s.activationDaa)}`); + provingWasActive = !!s.active; + } catch (e) { + const absent = e.code === -32601; + if (provingSupported !== false || !absent) log(absent ? `node at ${EVM} has no proving RPCs` : `proving status failed: ${e.message}`); + provingSupported = false; provingStatus = null; provingReason = absent ? 'node without proving' : `evm rpc unreachable (${e.message.slice(0, 60)})`; + provingRecheckAt = now + (absent ? 5 * 60_000 : 20_000); + } +} + +async function flushPlans() { + if (plansBusy || provingSupported !== true || !pendingPlans.size) return; + plansBusy = true; + try { + const now = Date.now(); + const due = [...pendingPlans].filter(([, p]) => p.nextAt <= now).slice(0, PLAN_BATCH).map(([h]) => h); + const rows = []; + for (let i = 0; i < due.length; i += 4) { + await Promise.all(due.slice(i, i + 4).map(async h => { + const p = pendingPlans.get(h); if (!p) return; + try { + const plan = await evm('igneum_getShardPlan', [{ blockHash: '0x' + h }]); + pendingPlans.delete(h); + const number = hx(plan.number), daa = hx(plan.daaScore); + daaOfNumber.set(number, daa); + const shards = (plan.shards || []).map(() => 'planned'); + openBlocks.set(h, { number, daa, ts: now, shards, provers: shards.map(() => null), polledAt: 0, done: false }); + (plan.shards || []).forEach((s, i) => rows.push({ hash: h, shard: i, shards: shards.length, number, daa, pgas: hx(s.pgas) })); + } catch (e) { + p.tries++; + if (p.tries > 30 || !/not found/.test(e.message)) { pendingPlans.delete(h); if (!/not found/.test(e.message)) log('shard plan failed', h.slice(0, 8), e.message); } + else p.nextAt = now + Math.min(30_000, 1000 * p.tries); + } + })); + } + if (rows.length) { + const params = [], values = []; + for (const r of rows) { + params.push(r.hash, r.shard, r.shards, r.number, r.daa, r.pgas); + const n = params.length; + values.push(`($${n - 5},$${n - 4},$${n - 3},$${n - 2},$${n - 1},$${n})`); + } + // block_ts comes from the block table (the header time the page's axis uses), one update for the batch + await sql(`INSERT INTO ${TP} (block_hash, shard, shards, block_number, block_daa, pgas) VALUES ${values.join(',')} ON CONFLICT (block_hash, shard) DO NOTHING`, params); + await sql(`UPDATE ${TP} p SET block_ts = b.timestamp_ms FROM ${TB} b WHERE b.hash = p.block_hash AND p.block_ts IS NULL AND p.block_hash = ANY($1::text[])`, [pgArray([...new Set(rows.map(r => r.hash))])]).catch(() => { }); + } + } catch (e) { log('plan flush failed', e.message); } + finally { plansBusy = false; } +} + +// Per shard: paid (a carrying segment paid it) > verified (the proof verified in the pool, or the record carried and +// checked by consensus, which is what happens before activation) > proving (a record in the pool) > planned. +function shardStates(block, r) { + const n = block.shards.length, states = block.shards.map(() => 'planned'), out = states.map(() => ({ prover: null, verified: null, carrier: null, carrierNumber: null, payout: null })); + for (const e of r.pool || []) { const i = e.shard; if (i >= n) continue; const rank = e.verified === true ? 2 : 1; if (rank > ['planned', 'proving', 'verified', 'paid'].indexOf(states[i])) { states[i] = rank === 2 ? 'verified' : 'proving'; out[i] = { ...out[i], prover: short(String(e.keyHash || '').replace(/^0x/, '')), verified: e.verified, carrier: e.includedIn ? String(e.includedIn).replace(/^0x/, '') : null }; } } + for (const c of r.carried || []) { const i = c.shard; if (i >= n || !c.valid) continue; if (states[i] !== 'paid') { states[i] = 'verified'; out[i] = { ...out[i], prover: short(String(c.keyHash || '').replace(/^0x/, '')), carrier: String(c.carrier || '').replace(/^0x/, ''), carrierNumber: hx(c.carrierNumber) }; } } + (r.paid || []).forEach((p, i) => { if (!p || i >= n) return; states[i] = 'paid'; out[i] = { ...out[i], prover: short(String(p.keyHash || '').replace(/^0x/, '')), carrierNumber: hx(p.carrierNumber), payout: weiStr(p.wei) }; }); + return { states, out }; +} + +async function pollRecords() { + if (recordsBusy || provingSupported !== true) return; + recordsBusy = true; + try { + const now = Date.now(); + for (const [h, b] of openBlocks) if (b.done || now - b.ts > PROOF_WINDOW_MS) openBlocks.delete(h); + const active = !!(provingStatus && provingStatus.active); + const batch = [...openBlocks].sort((a, b) => a[1].polledAt - b[1].polledAt).slice(0, active ? RECORD_BATCH_ACTIVE : RECORD_BATCH_INACTIVE); + const updates = []; + for (let i = 0; i < batch.length; i += 4) { + await Promise.all(batch.slice(i, i + 4).map(async ([h, b]) => { + b.polledAt = now; + let r; try { r = await evm('igneum_getProofRecords', [{ blockHash: '0x' + h }]); } catch (e) { if (!/not found/.test(e.message)) log('proof records failed', h.slice(0, 8), e.message); return; } + const { states, out } = shardStates(b, r); + for (let s = 0; s < states.length; s++) { + const key = `${states[s]}|${out[s].prover}|${out[s].carrierNumber}|${out[s].carrier}`; + if (b.provers[s] === key) continue; + b.provers[s] = key; b.shards[s] = states[s]; + let carrierDaa = out[s].carrierNumber !== null ? daaOfNumber.get(out[s].carrierNumber) ?? null : null; + if (carrierDaa === null && out[s].carrierNumber !== null) { try { const p = await evm('igneum_getShardPlan', ['0x' + out[s].carrierNumber.toString(16)]); carrierDaa = hx(p.daaScore); daaOfNumber.set(out[s].carrierNumber, carrierDaa); } catch { } } + updates.push({ hash: h, shard: s, state: states[s], prover: out[s].prover, verified: out[s].verified, carrier: out[s].carrier, carrierNumber: out[s].carrierNumber, carrierDaa, lag: carrierDaa !== null && b.daa !== null ? carrierDaa - b.daa : null, payout: out[s].payout }); + if (out[s].prover && !provers.has(out[s].prover)) { provers.add(out[s].prover); recordEvent('prover_seen', `Prover ${out[s].prover} seen (shard ${s} of block ${short(h)})`); } + if (states[s] === 'paid' && !firstPaidSeen) { firstPaidSeen = true; recordEvent('proving', `First paid shard seen: block ${short(h)} shard ${s}, carried ${updates[updates.length - 1].lag ?? '?'} DAA later`); } + } + if (states.every(s => s === 'paid')) b.done = true; + })); + } + for (const u of updates) { + await sql(`UPDATE ${TP} SET state = $3, prover = $4, verified = $5, carried_by = $6, carrier_number = $7, carrier_daa = $8, lag_daa = $9, payout_wei = COALESCE($10, payout_wei), updated_at = now() + WHERE block_hash = $1 AND shard = $2`, [u.hash, u.shard, u.state, u.prover, u.verified, u.carrier, u.carrierNumber, u.carrierDaa, u.lag, u.payout]).catch(e => log('proof row update failed', e.message)); + } + } catch (e) { log('records poll failed', e.message); } + finally { recordsBusy = false; } +} + +// live_state.proving: activation and the last 10 minutes (counts refreshed every 10 s; the status every tick) +async function provingState() { + if (provingSupported !== true) return { supported: false, reason: provingReason || 'not checked yet', evm_rpc: EVM }; + const s = provingStatus, now = Date.now(); + if (now - lastProvingStatsAt > 10_000) { + lastProvingStatsAt = now; + try { + const rows = await sql(`WITH w AS (SELECT * FROM ${TP} WHERE received_at > now() - interval '10 minutes') + SELECT (SELECT count(DISTINCT block_hash) FROM w)::int AS blocks, + (SELECT count(*) FROM (SELECT block_hash FROM w GROUP BY block_hash HAVING bool_and(state IN ('verified','paid'))) f)::int AS blocks_full, + (SELECT count(*) FROM w WHERE state IN ('verified','paid'))::int AS shards_proven, + (SELECT count(*) FROM w WHERE state = 'paid')::int AS shards_paid, + (SELECT count(DISTINCT prover) FROM w WHERE prover IS NOT NULL AND state IN ('verified','paid'))::int AS provers, + (SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY lag_daa) FROM w WHERE lag_daa IS NOT NULL)::float AS median_lag`); + lastProvingStats = rows[0] || null; + } catch (e) { log('proving stats failed', e.message); } + } + const st = lastProvingStats || {}; + return { + supported: true, active: !!s.active, activation_daa: s.activationDaa === null ? null : hx(s.activationDaa), tip_daa: hx(s.tipDaa), + verifier: s.verifier, pool: s.pool || null, paid_shards_total: s.paidShards ?? null, shard_budget_pgas: hx(s.shardBudget), + blocks_10m: st.blocks ?? 0, blocks_fully_proven_10m: st.blocks_full ?? 0, shards_proven_10m: st.shards_proven ?? 0, shards_paid_10m: st.shards_paid ?? 0, + // DAA score of the carrying chain block minus the proven block's; the devnet targets one DAA step per second + median_proof_lag_s: st.median_lag === null || st.median_lag === undefined ? null : Math.round(st.median_lag * 10) / 10, + provers_10m: st.provers ?? 0, open_blocks: openBlocks.size, pending_plans: pendingPlans.size, evm_rpc: EVM, + }; +} + async function prune() { try { await sql(`DELETE FROM ${TB} WHERE received_at < now() - ($1 || ' hours')::interval`, [String(RETAIN_HOURS)]); + await sql(`DELETE FROM ${TP} WHERE received_at < now() - ($1 || ' hours')::interval`, [String(RETAIN_HOURS)]); await sql(`DELETE FROM ${TE} WHERE ts < now() - interval '7 days'`); await sql(`DELETE FROM ${TC} WHERE updated_at < now() - interval '7 days'`); } catch (e) { log('prune failed', e.message); } @@ -690,6 +889,15 @@ async function seedFromDb() { for (const r of cps) checkpointStates.set(Number(r.index), r.state); const certs = await sql(`SELECT index, hash FROM ${TX}`); for (const r of certs) storedCerts.set(Number(r.index), r.hash); + // proving: keep polling the blocks of the last 10 minutes that are not fully paid, so a restart loses nothing + const open = await sql(`SELECT block_hash, block_number, block_daa, extract(epoch from received_at) * 1000 AS ts, array_agg(state ORDER BY shard) AS states, array_agg(prover ORDER BY shard) AS provers + FROM ${TP} WHERE received_at > now() - interval '10 minutes' GROUP BY 1, 2, 3, 4`); + for (const r of open) { + daaOfNumber.set(Number(r.block_number), Number(r.block_daa)); + openBlocks.set(r.block_hash, { number: Number(r.block_number), daa: Number(r.block_daa), ts: Number(r.ts), shards: r.states, provers: r.states.map(() => null), polledAt: 0, done: r.states.every(s => s === 'paid') }); + for (const p of r.provers) if (p) provers.add(p); + } + if (open.some(r => r.states.some(s => s === 'paid'))) firstPaidSeen = true; if (st.length && st[0].difficulty) lastDifficultyEvent = Number(st[0].difficulty); } catch (e) { log('seed failed', e.message); } } @@ -697,7 +905,7 @@ async function seedFromDb() { async function main() { await setupSchema(); await seedFromDb(); - log(`observer started, rpc ${RPC}, keeping ${RETAIN_HOURS} h of blocks, ${miners.size} miners known`); + log(`observer started, rpc ${RPC}, evm rpc ${EVM}, keeping ${RETAIN_HOURS} h of blocks, ${miners.size} miners known, ${openBlocks.size} blocks open for proofs`); const rpc = new Rpc(RPC); // The socket handler only enqueues; the drain loop does the work in bounded batches so the socket is always read rpc.onNotification = (method, params) => { inbox.push([method, params]); if (!draining) { draining = true; setImmediate(drain); } }; @@ -719,8 +927,8 @@ async function main() { } } else if (method === 'virtualChainChangedNotification' && inner) { - for (const h of inner.addedChainBlockHashes || []) { pendingChain.remove.delete(h); pendingChain.add.add(h); pendingUnmerge.delete(h); if (!pendingMerge.has(h)) pendingMerge.set(h, mergesets.get(h) || null); } - for (const h of inner.removedChainBlockHashes || []) { pendingChain.add.delete(h); pendingChain.remove.add(h); pendingMerge.delete(h); pendingUnmerge.add(h); } + for (const h of inner.addedChainBlockHashes || []) { pendingChain.remove.delete(h); pendingChain.add.add(h); pendingUnmerge.delete(h); if (!pendingMerge.has(h)) pendingMerge.set(h, mergesets.get(h) || null); queuePlan(h); } + for (const h of inner.removedChainBlockHashes || []) { pendingChain.add.delete(h); pendingChain.remove.add(h); pendingMerge.delete(h); pendingUnmerge.add(h); dropPlan(h); } } } let connected = false; @@ -746,6 +954,8 @@ async function main() { setInterval(async () => { if (flushBusy) return; flushBusy = true; try { await flushBlocks(); await flushChain(); } finally { flushBusy = false; } }, FLUSH_EVERY_MS); setInterval(() => flushColors(rpc), FLUSH_EVERY_MS); setInterval(() => flushCertificates(rpc), FLUSH_EVERY_MS); + setInterval(flushPlans, FLUSH_EVERY_MS); + setInterval(pollRecords, STATE_EVERY_MS); setInterval(() => tick(rpc), STATE_EVERY_MS); setInterval(prune, PRUNE_EVERY_MS); tick(rpc); prune();