igneum/tools/txgen/proving-watch.mjs

#!/usr/bin/env node
// The proving layer watched while tools/txgen/run.mjs fills blocks (5 October 2026). Two modes, no dependencies.
//
//   node tools/txgen/proving-watch.mjs watch [--rpc http://127.0.0.1:26800] [--interval 30] [--duration 1500]
//        [--node-log newest|<file>] [--live https://igneum.network/api/live] [--out <jsonl>]
//     Every interval: igneum_getProvingStatus (paid shards, paid wei, pool), the live page's proving object, the Mac
//     node log's proof-record lines since the start (accepted, verified, not verified, paid) and the live page's
//     per-block shard cells with content. One line per sample, every sample appended to --out as JSON.
//
//   node tools/txgen/proving-watch.mjs report --summary <txgen summary.json> [--rpc ...] [--node-log newest|<file>]
//        [--pc2-log <collected app log>] [--from <ISO time>] [--to <ISO time>] [--json <file>]
//     For every block the generator saw content in (and every block a proof record named in the window): the shard
//     plan (pgas, gas, transactions per shard, paid), the node log's record (accepted, verify time, verdict, payout,
//     carrying chain block) and PC 2's prover lines for that block (export, cut, prove, submit, with their times when
//     the line carries them). Prints Markdown tables for docs/bench-log.md and a JSON when --json is given.
//
// The node log lines this reads (vendor/igneum-node-036/igneum/exec, 0.3.8):
//   proof record for block N shard S by KEY (peer|local): accepted, verify Command(...)
//   proof of block N shard S by KEY: verified in X s: RESULT verify: VERIFIED in Y s; ... proof B bytes at T
//   proof of block N shard S by KEY: ... NOT VERIFIED ...   (a rejection names the reason on the same line)
//   chain block C: proof record for block N shard S by KEY paid W wei to ADDR
// PC 2's app log lines: "prover: block N shard S ..." (app/igneum-app, the proving job runner).

import { readFileSync, readdirSync, statSync, appendFileSync, writeFileSync, existsSync } from 'node:fs';
import { homedir } from 'node:os';
import { join } from 'node:path';

const args = process.argv.slice(2);
const mode = args[0];
const opt = (name, dflt) => { const i = args.indexOf(name); return i >= 0 && args[i + 1] !== undefined ? args[i + 1] : dflt; };
const RPC = opt('--rpc', 'http://127.0.0.1:26800');
const LIVE = opt('--live', 'https://igneum.network/api/live');
const log = (...a) => console.log(new Date().toISOString().slice(11, 19), ...a);
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const ign = (w) => (Number(BigInt(w)) / 1e18).toFixed(6);
let id = 0;
async function rpc(method, params = []) {
  const r = await fetch(RPC, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ jsonrpc: '2.0', id: ++id, method, params }), signal: AbortSignal.timeout(20000) });
  const j = await r.json(); if (j.error) throw new Error(j.error.message); return j.result;
}
const live = () => fetch(LIVE, { signal: AbortSignal.timeout(20000) }).then((r) => r.json()).catch((e) => ({ error: e.message }));
function newestNodeLog() {
  const dir = join(homedir(), 'Library', 'Logs', 'Igneum');
  if (!existsSync(dir)) return null;
  const files = readdirSync(dir).filter((f) => /^node-.*\.log$/.test(f)).map((f) => join(dir, f)).sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs);
  return files[0] || null;
}
const nodeLogPath = () => { const v = opt('--node-log', 'newest'); return v === 'newest' ? newestNodeLog() : v; };
// a log line's timestamp "2026-10-05 16:24:13.035+01:00" as epoch ms
const lineTime = (l) => { const m = /^(\d{4}-\d\d-\d\d) (\d\d:\d\d:\d\d\.\d+)([+-]\d\d:\d\d)/.exec(l); return m ? Date.parse(`${m[1]}T${m[2]}${m[3]}`) : null; };
// parses every proof-record line into {kind, block, shard, key, t, ...}
function parseNodeLog(text) {
  const out = [];
  for (const l of text.split('\n')) {
    let m;
    if ((m = /proof record for block (\d+) shard (\d+) by ([0-9a-f]{64}) \((peer|local)\): accepted/.exec(l))) out.push({ kind: 'accepted', block: +m[1], shard: +m[2], key: m[3].slice(0, 8), origin: m[4], t: lineTime(l) });
    else if ((m = /proof of block (\d+) shard (\d+) by ([0-9a-f]{64}): (?:verified in ([\d.]+) s: )?(.*)$/.exec(l))) {
      const rest = m[5];
      const verdict = /\bVERIFIED\b/.test(rest) && !/NOT VERIFIED/.test(rest) ? 'VERIFIED' : 'NOT VERIFIED';
      const inner = /VERIFIED in ([\d.]+) s/.exec(rest);
      const bytes = /proof (\d+) bytes/.exec(rest);
      out.push({ kind: verdict === 'VERIFIED' ? 'verified' : 'rejected', block: +m[1], shard: +m[2], key: m[3].slice(0, 8), t: lineTime(l), verify_wall_s: m[4] ? +m[4] : null, verify_s: inner ? +inner[1] : null, bytes: bytes ? +bytes[1] : null, text: verdict === 'VERIFIED' ? '' : rest.slice(0, 300) });
    } else if ((m = /chain block (\d+): proof record for block (\d+) shard (\d+) by ([0-9a-f]{64}) paid (\d+) wei to (0x[0-9a-fA-F]{40})/.exec(l))) out.push({ kind: 'paid', chain_block: +m[1], block: +m[2], shard: +m[3], key: m[4].slice(0, 8), wei: m[5], to: m[6], t: lineTime(l) });
    else if (/\[igneum-exec\].*(proof|prov|shard|export).*(error|fail|panic|refus|reject)/i.test(l)) out.push({ kind: 'error', t: lineTime(l), text: l.replace(/^\S+ \S+ /, '').slice(0, 300) });
  }
  return out;
}
const counts = (recs) => recs.reduce((a, r) => { a[r.kind] = (a[r.kind] || 0) + 1; return a; }, {});

// ---- watch ----
async function watch() {
  const INTERVAL = 1000 * +opt('--interval', 30), DURATION = 1000 * +opt('--duration', 1500);
  const OUT = opt('--out', null);
  const logFile = nodeLogPath();
  const startAt = Date.now();
  const startOffset = logFile ? statSync(logFile).size : 0;
  const cells = new Map(); // block hash prefix -> the live page's cell with shards
  log(`watching ${RPC}, live ${LIVE}, node log ${logFile || 'none'} (from byte ${startOffset}); every ${INTERVAL / 1000} s for ${DURATION / 1000} s`);
  let last = null;
  while (Date.now() - startAt < DURATION) {
    const [ps, lv] = await Promise.all([rpc('igneum_getProvingStatus').catch((e) => ({ error: e.message })), live()]);
    const tip = await rpc('eth_blockNumber').then((n) => Number(BigInt(n))).catch(() => null);
    let recs = [];
    if (logFile) { try { const t = readFileSync(logFile, 'utf8'); recs = parseNodeLog(t.slice(startOffset)); } catch { } }
    for (const b of lv.blocks || []) if (b.shards?.length) cells.set(b.hash, b);
    const c = counts(recs);
    const sample = {
      t: new Date().toISOString(), tip, paid_shards: ps.paidShards, paid_ign: ps.paidWei ? ign(ps.paidWei) : null, pool: ps.pool, tip_daa: ps.tipDaa ? Number(BigInt(ps.tipDaa)) : null,
      live: lv.proving || lv.error || null, log_since_start: c, cells_with_shards: cells.size,
      rejected: recs.filter((r) => r.kind === 'rejected').slice(-3), errors: recs.filter((r) => r.kind === 'error').slice(-3),
    };
    const d = last ? ` (+${(ps.paidShards ?? 0) - (last.paid_shards ?? 0)} paid, +${(c.accepted || 0) - (last.log_since_start.accepted || 0)} accepted, +${(c.verified || 0) - (last.log_since_start.verified || 0)} verified)` : '';
    log(`tip ${tip}: paid shards ${ps.paidShards} (${sample.paid_ign} IGN), pool verified ${ps.pool?.verified} pending ${ps.pool?.pending} failed ${ps.pool?.failed}; log since start: accepted ${c.accepted || 0}, verified ${c.verified || 0}, rejected ${c.rejected || 0}, paid ${c.paid || 0}, errors ${c.error || 0}${d}; live: shards 10m ${lv.proving?.shards_proven_10m}, fully proven blocks 10m ${lv.proving?.blocks_fully_proven_10m}, lag ${lv.proving?.median_proof_lag_s} s, provers ${lv.proving?.provers_10m}; cells with shards seen ${cells.size}`);
    for (const r of sample.rejected) log(`  rejected: block ${r.block} shard ${r.shard} by ${r.key}: ${r.text}`);
    for (const r of sample.errors) log(`  error: ${r.text}`);
    if (OUT) appendFileSync(OUT, JSON.stringify(sample) + '\n');
    last = sample;
    await sleep(INTERVAL);
  }
  if (OUT && cells.size) appendFileSync(OUT, JSON.stringify({ t: new Date().toISOString(), cells: [...cells.values()] }) + '\n');
  log(`done; ${cells.size} live-page blocks showed shard cells`);
}

// ---- report ----
const hex = (v) => (v == null ? null : Number(BigInt(v)));
async function report() {
  const summaryFile = opt('--summary', null);
  if (!summaryFile) throw new Error('--summary <txgen summary.json> is required');
  const s = JSON.parse(readFileSync(summaryFile, 'utf8'));
  const from = Date.parse(opt('--from', s.started)), to = Date.parse(opt('--to', s.ended)) + 10 * 60 * 1000; // records land up to minutes after the last send
  const logFile = nodeLogPath();
  const recs = logFile ? parseNodeLog(readFileSync(logFile, 'utf8')).filter((r) => r.t == null || (r.t >= from && r.t <= to)) : [];
  // PC 2's app log (collected with publish-jobs.sh --kind collect --glob "logs/app-*.log", read with tools/jobs.mjs <id> --all):
  //   <epoch> prover: block N shard S assigned to win-1ccfe586-1-1: export, cut, prove (CUDA), sign, submit
  //   <epoch> [proving] block N shard S proven and submitted in T s
  //   <epoch> [proving] block N shard S paid X IGN
  //   <epoch> prover: block N shard S: <stage>: failed ...      (any line naming the block with "failed" or "error")
  const pc2 = [];
  if (opt('--pc2-log', null)) for (const l of readFileSync(opt('--pc2-log'), 'utf8').split('\n')) {
    let m;
    if ((m = /^(\d+) prover: block (\d+) shard (\d+) assigned to (\S+):/.exec(l))) pc2.push({ kind: 'assigned', t: +m[1] * 1000, block: +m[2], shard: +m[3], prover: m[4] });
    else if ((m = /^(\d+) \[proving\] block (\d+) shard (\d+) proven and submitted in ([\d.]+) s/.exec(l))) pc2.push({ kind: 'proven', t: +m[1] * 1000, block: +m[2], shard: +m[3], prove_s: +m[4] });
    else if ((m = /^(\d+) \[proving\] block (\d+) shard (\d+) paid ([\d.]+) IGN/.exec(l))) pc2.push({ kind: 'paid', t: +m[1] * 1000, block: +m[2], shard: +m[3], ign: +m[4] });
    else if ((m = /^(\d+) .*block (\d+) shard (\d+).*(failed|error|panic|refused)/i.exec(l))) pc2.push({ kind: 'failed', t: +m[1] * 1000, block: +m[2], shard: +m[3], text: l.replace(/^\d+ /, '').slice(0, 300) });
  }
  const contentBlocks = Object.keys(s.blocks?.per_block || {}).map(Number);
  const recordBlocks = [...new Set(recs.filter((r) => r.block).map((r) => r.block))];
  const blocks = [...new Set([...contentBlocks, ...recordBlocks])].sort((a, b) => a - b);
  const rows = [];
  for (const n of blocks) {
    let plan = null;
    try { plan = await rpc('igneum_getShardPlan', ['0x' + n.toString(16)]); } catch (e) { plan = { error: e.message }; }
    const shards = (plan?.shards || []).map((sh) => ({ index: sh.index, pgas: hex(sh.pgas), gas: hex(sh.gas), txs: (sh.txEnd ?? 0) - (sh.txStart ?? 0), executed: sh.executed, skipped: sh.skipped, paid: sh.paid, over_budget: sh.overBudget }));
    const mine = recs.filter((r) => r.block === n);
    const byShard = {};
    for (const r of mine) {
      const b = (byShard[r.shard] ||= { accepted: [], verified: [], rejected: [], paid: [] });
      b[r.kind]?.push(r);
    }
    const pcLines = pc2.filter((e) => e.block === n);
    rows.push({ block: n, txs_included_by_txgen: s.blocks?.per_block?.[n] ?? 0, plan_error: plan?.error || null, shards, records: byShard, pc2: pcLines });
  }
  // tables
  const sum = { blocks: rows.length, content_blocks: rows.filter((r) => r.shards.some((x) => x.pgas > 0)).length, content_shards: 0, content_shards_paid: 0, content_shards_verified: 0, content_shards_rejected: 0, empty_shards_paid: 0, paid_ign_window: 0 };
  console.log(`\n## Blocks with content in the window ${new Date(from).toISOString()} to ${new Date(to).toISOString()} (node log ${logFile || 'none'})\n`);
  console.log('| Block | Txs (txgen) | Shard | Txs in shard | pgas | gas | Accepted (UTC) | Verify wall / inner | Verdict | Paid (IGN) at chain block | PC 2 prover lines |');
  console.log('|---|---|---|---|---|---|---|---|---|---|---|');
  for (const r of rows) {
    if (r.plan_error) { console.log(`| ${r.block} | ${r.txs_included_by_txgen} | plan error: ${r.plan_error} | | | | | | | | |`); continue; }
    for (const sh of r.shards) {
      const rec = r.records[sh.index] || { accepted: [], verified: [], rejected: [], paid: [] };
      const acc = rec.accepted[0]?.t ? new Date(rec.accepted[0].t).toISOString().slice(11, 19) : '';
      const v = rec.verified[0], rj = rec.rejected[0], pd = rec.paid[0];
      const verify = v ? `${v.verify_wall_s ?? '?'} s / ${v.verify_s ?? '?'} s` : rj ? `${rj.verify_wall_s ?? '?'} s` : '';
      const verdict = v ? 'VERIFIED' : rj ? `NOT VERIFIED: ${rj.text.slice(0, 120)}` : rec.accepted.length ? 'no verdict logged' : (sh.pgas > 0 ? 'NO RECORD' : '');
      const paid = pd ? `${ign(pd.wei)} at ${pd.chain_block}${rec.paid.length > 1 ? ` (x${rec.paid.length} lines)` : ''}` : sh.paid ? `plan says paid (${JSON.stringify(sh.paid).slice(0, 40)})` : '';
      const pcs = r.pc2.filter((e) => e.shard === sh.index);
      const asg = pcs.find((e) => e.kind === 'assigned'), prv = pcs.find((e) => e.kind === 'proven'), ppd = pcs.find((e) => e.kind === 'paid'), pfl = pcs.filter((e) => e.kind === 'failed');
      const pc = [asg ? `assigned ${new Date(asg.t).toISOString().slice(11, 19)}` : '', prv ? `proven+submitted ${prv.prove_s} s` : '', ppd ? `paid ${ppd.ign} IGN` : '', ...pfl.map((e) => `FAILED: ${e.text.slice(0, 120)}`)].filter(Boolean).join('; ');
      console.log(`| ${r.block} | ${r.txs_included_by_txgen} | ${sh.index} | ${sh.txs} | ${sh.pgas?.toLocaleString('en-GB')} | ${sh.gas?.toLocaleString('en-GB')} | ${acc} | ${verify} | ${verdict} | ${paid} | ${pc} |`);
      if (sh.pgas > 0) { sum.content_shards++; if (pd) sum.content_shards_paid++; if (v) sum.content_shards_verified++; if (rj) sum.content_shards_rejected++; }
      else if (pd) sum.empty_shards_paid++;
      if (pd) sum.paid_ign_window += Number(ign(pd.wei));
    }
  }
  const proveTimes = pc2.filter((e) => e.kind === 'proven' && e.t >= from && e.t <= to);
  const contentSet = new Set(rows.filter((r) => r.shards.some((x) => x.pgas > 0)).map((r) => r.block));
  const pt = (list) => { const v = list.map((e) => e.prove_s).sort((a, b) => a - b); return v.length ? `${v[Math.floor(v.length / 2)]} / ${v.at(-1)} (n=${v.length})` : '-'; };
  const allPaid = recs.filter((r) => r.kind === 'paid'), allVer = recs.filter((r) => r.kind === 'verified'), allRej = recs.filter((r) => r.kind === 'rejected');
  const vt = allVer.map((r) => r.verify_wall_s).filter((x) => x != null).sort((a, b) => a - b);
  console.log(`\n## Window totals (node log, ${new Date(from).toISOString().slice(11, 19)} to ${new Date(to).toISOString().slice(11, 19)} UTC)\n`);
  console.log('| What | Count |\n|---|---|');
  console.log(`| Blocks listed (txgen content or a proof record) | ${rows.length} |`);
  console.log(`| Blocks whose plan has a shard with pgas > 0 | ${sum.content_blocks} |`);
  console.log(`| Shards with pgas > 0: total / verified / rejected / paid | ${sum.content_shards} / ${sum.content_shards_verified} / ${sum.content_shards_rejected} / ${sum.content_shards_paid} |`);
  console.log(`| Empty shards paid in the window (listed blocks) | ${sum.empty_shards_paid} |`);
  console.log(`| Proof records accepted / verified / rejected / paid (all blocks) | ${recs.filter((r) => r.kind === 'accepted').length} / ${allVer.length} / ${allRej.length} / ${allPaid.length} |`);
  console.log(`| Verify wall time on the Mac, p50 / max (s) | ${vt.length ? `${vt[Math.floor(vt.length / 2)]} / ${vt.at(-1)}` : '-'} |`);
  console.log(`| Paid in the window (IGN, listed blocks) | ${sum.paid_ign_window.toFixed(6)} |`);
  if (pc2.length) {
    console.log(`| PC 2 prove+submit time, content shards, p50 / max (s) | ${pt(proveTimes.filter((e) => contentSet.has(e.block)))} |`);
    console.log(`| PC 2 prove+submit time, empty shards, p50 / max (s) | ${pt(proveTimes.filter((e) => !contentSet.has(e.block)))} |`);
    console.log(`| PC 2 failed lines in the window | ${pc2.filter((e) => e.kind === 'failed' && e.t >= from && e.t <= to).length} |`);
    for (const e of pc2.filter((x) => x.kind === 'failed' && x.t >= from && x.t <= to).slice(0, 10)) console.log(`  pc2 failed: ${e.text}`);
  }
  console.log(`| Exec-layer error lines in the window | ${recs.filter((r) => r.kind === 'error').length} |`);
  for (const r of recs.filter((x) => x.kind === 'error').slice(0, 10)) console.log(`  error: ${r.text}`);
  for (const r of allRej.slice(0, 10)) console.log(`  rejected: block ${r.block} shard ${r.shard} by ${r.key}: ${r.text}`);
  const out = { window: { from: new Date(from).toISOString(), to: new Date(to).toISOString() }, node_log: logFile, totals: sum, rows };
  if (opt('--json', null)) writeFileSync(opt('--json'), JSON.stringify(out, null, 2) + '\n');
}

try {
  if (mode === 'watch') await watch();
  else if (mode === 'report') await report();
  else { console.log(readFileSync(new URL(import.meta.url), 'utf8').split('\n').filter((l) => l.startsWith('//')).map((l) => l.slice(3)).join('\n')); process.exit(mode ? 2 : 0); }
} catch (e) { console.error('FAILED:', e.message); process.exit(1); }