473 lines
46 KiB
JavaScript
473 lines
46 KiB
JavaScript
#!/usr/bin/env node
|
|
// The ladder and class v5 together on the fast-time harness: the rehearsal for the Devnet 3 crossing (7 October 2026,
|
|
// evening; docs/design/class-v5-stored-state.md sections 6 and 10, docs/plans/counter-asic-3-node.md section 6, the
|
|
// class-v5-signal.mjs and latency-ladder.mjs shapes). A four-node private network on override-60x.json with CPU
|
|
// genesis bits, class v4 sub-version 3 FROM GENESIS (v3 and the v4 floor at 0) at ladder rung 0 (the ladder active from
|
|
// DAA 0), the class signal window at --class-window (60 DAA, one epoch) and the ladder window at --ladder-window (45 DAA),
|
|
// so the two seven-window tallies land on different epochs: the rung steps first and the class crosses two epochs later.
|
|
// rung 0 -> 1 at the first epoch whose seed block has seven full ladder windows below it: 60 e - 10 - 1 >= 7 x 45,
|
|
// epoch 6 (DAA 360), and no second step inside the run (the oldest window must begin after the step: epoch 12)
|
|
// v4 -> v5 at the first epoch whose seed block has seven full class windows below it: 60 e - 11 >= 7 x 60, epoch 8
|
|
// (DAA 480), by miner signal (byte 6 counted EXACTLY, 9,500 bps in each of seven windows), before the floor
|
|
// program_class_v5_activation_daa (--floor, default 900 = epoch 15: the backstop, which enables the object)
|
|
// Nodes n0, n1, n2 are honest (byte 6, ladder up). n3 is the STALE node: a class v5 node whose miner runs --freeze-state,
|
|
// so it keeps the first state stream it fetched (the flip epoch's) for every later epoch: from the first refresh after
|
|
// the flip every block it mines is rejected by its own node under the stale-chip rule (PoW rejected), and the honest
|
|
// nodes see none of them. n2 is RESTARTED across the v5 boundary on its own datadir (stopped --restart-before DAA before
|
|
// the expected boundary, started --restart-after DAA after it) and must resync with no lock fault.
|
|
//
|
|
// Every epoch's program id is read from the four miners' "program and 256 MiB cache ready" lines and compared with the
|
|
// CPU verifier's id for that seed (`igneum-pow show --epoch-hex <seed> --program-class v4|v5 --era-hex <era>
|
|
// [--shadow-reps 35] [--state <stream>]`), the state stream fetched DURING the run from n0's exec RPC at each epoch start
|
|
// (the executor keeps only the last three captures). Before the run the CLI must reproduce the first class v5 pack's id
|
|
// for Devnet 3's genesis seed (e5a4ac5978462156, proto-cuda/packs-ca3-v5/v5-dn3-epoch0) when the pack is present.
|
|
//
|
|
// The known-failed shape, run FIRST (the gate rule): `--signal 6,6,7,7 --ladder up,up,none,none --case failed-case`:
|
|
// two of four at byte 6 and two of four up are 50 percent, so no rung and no v5; the harness must report FAIL and name
|
|
// the checks.
|
|
//
|
|
// node infra/fast-time/class-v5-crossing.mjs [--case cross] [--signal 6,6,6,6] [--ladder up,up,up,up] [--class-window 60]
|
|
// [--ladder-window 45] [--floor 900|never] [--stale 3|none] [--restart 2|none] [--restart-before 25] [--restart-after 20] [--restart-miner-hold-s 300]
|
|
// [--epochs 11] [--secs 960] [--out <dir>] [--genesis-bits 0x1f010000]
|
|
// IGNEUMD, IGNEUM_MINER, IGNEUM_POW name the binaries (defaults: vendor/igneum-node-v5-fasttime/target/release and
|
|
// igneum-pow/target/release/igneum-pow, the layout on igneum-build-1 under /srv/builds/igneum-wt-v5-fasttime).
|
|
// IGNEUM_V5X_BASE moves the ports (default 29860, suffix 977); the data dir is /tmp/igneum-fast-time-v5x-<case>.
|
|
// The summary and this log land in --out (default docs/design/class-v5-harness/fasttime) as <case>.json and <case>.log.
|
|
|
|
import { spawn, spawnSync } from 'node:child_process';
|
|
import { mkdirSync, rmSync, writeFileSync, readFileSync, openSync, existsSync, appendFileSync, copyFileSync } from 'node:fs';
|
|
import { connectRpc } from '../../tools/finality-attacks/lib/rpc.mjs';
|
|
import { devAddress } from '../../tools/harness/lib/address.mjs';
|
|
import { mergeOverrideText, probeUnknownKeys } from './lib/override.mjs';
|
|
|
|
const ROOT = new URL('../../', import.meta.url).pathname;
|
|
const FILE = `${ROOT}infra/fast-time/override-60x.json`;
|
|
const BIN = process.env.IGNEUM_V5X_BIN || `${ROOT}vendor/igneum-node-v5-fasttime/target/release`;
|
|
const IGNEUMD = process.env.IGNEUMD || `${BIN}/igneumd`;
|
|
const CPU_MINER = process.env.IGNEUM_MINER || `${BIN}/igneum-miner`;
|
|
const IGNEUM_POW = process.env.IGNEUM_POW || `${ROOT}igneum-pow/target/release/igneum-pow`;
|
|
const DN3_PACK = `${ROOT}proto-cuda/packs-ca3-v5/v5-dn3-epoch0`;
|
|
const DN3_ID = 'e5a4ac5978462156';
|
|
const NEVER = '18446744073709551615';
|
|
const args = process.argv.slice(2);
|
|
const flag = (name, dflt) => { const i = args.indexOf(`--${name}`); return i >= 0 ? +args[i + 1] : dflt; };
|
|
const sflag = (name) => { const i = args.indexOf(`--${name}`); return i >= 0 ? args[i + 1] : null; };
|
|
const actflag = (name, dflt) => { const v = sflag(name); if (v == null) return dflt; return v === 'never' ? null : +v; };
|
|
const optidx = (name, dflt) => { const v = sflag(name); if (v == null) return dflt; return v === 'none' ? null : +v; };
|
|
const CASE = sflag('case') || 'cross';
|
|
const OUT = sflag('out') || `${ROOT}docs/design/class-v5-harness/fasttime`;
|
|
const TMP = process.env.IGNEUM_V5X_TMP || `/tmp/igneum-fast-time-v5x-${CASE}`;
|
|
const BASE = +(process.env.IGNEUM_V5X_BASE || 29860), SUFFIX = 977;
|
|
const GENESIS_BITS = flag('genesis-bits', 0x1f010000);
|
|
const SECS = flag('secs', 960);
|
|
const EPOCHS = flag('epochs', 11);
|
|
const CLASS_WINDOW = flag('class-window', 60);
|
|
const LADDER_WINDOW = flag('ladder-window', 45);
|
|
const FLOOR = actflag('floor', 900);
|
|
const STALE = optidx('stale', 3);
|
|
const RESTART = optidx('restart', 2);
|
|
const RESTART_BEFORE = flag('restart-before', 25), RESTART_AFTER = flag('restart-after', 20), MINER_HOLD = flag('restart-miner-hold-s', 300);
|
|
const SIGNAL = (sflag('signal') || '6,6,6,6').split(',').map(Number);
|
|
const LADDER = (sflag('ladder') || 'up,up,up,up').split(',').map(s => s.trim().toLowerCase());
|
|
const N = SIGNAL.length;
|
|
const WINDOWS = 7, CLASS_THRESHOLD = 9500, LADDER_THRESHOLD = 9000, RUNG0 = 27, RUNG1 = 35, V5_BYTE = 6;
|
|
if (N < 2 || LADDER.length !== N || !LADDER.every(s => ['up', 'down', 'none'].includes(s)) || (STALE != null && STALE >= N) || (RESTART != null && (RESTART >= N || RESTART === 0 || RESTART === STALE))) {
|
|
console.error('usage: --signal a,b,c,d --ladder up|down|none x4 [--stale <node index, not 0>|none] [--restart <node index, not 0, not the stale node>|none]'); process.exit(2);
|
|
}
|
|
mkdirSync(TMP, { recursive: true });
|
|
const LOG = `${TMP}/harness.log`;
|
|
const log = (...a) => { const line = `${new Date().toISOString().slice(11, 23)} ${a.join(' ')}`; console.log(line); try { appendFileSync(LOG, line + '\n'); } catch { } };
|
|
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
|
|
for (const b of [IGNEUMD, CPU_MINER, IGNEUM_POW]) if (!existsSync(b)) { console.error(`missing ${b}`); process.exit(2); }
|
|
|
|
// survivors of an earlier run of this case, by pid file (never by name)
|
|
const PIDS = `${TMP}/pids`;
|
|
try { for (const pid of readFileSync(PIDS, 'utf8').split('\n').map(Number).filter(Boolean)) { try { process.kill(pid, 'SIGKILL'); console.log(`killed survivor pid ${pid} of the previous run`); } catch { } } } catch { }
|
|
rmSync(TMP, { recursive: true, force: true }); mkdirSync(TMP, { recursive: true });
|
|
const recordPid = (p) => { try { appendFileSync(PIDS, `${p.pid}\n`); } catch { } };
|
|
|
|
// the override: merged as TEXT by lib/override.mjs (a never height does not survive a JavaScript number; duplicated
|
|
// top-level keys collapsed; a null field dropped); then the binary is probed and every key it does not know is dropped
|
|
const baseText = readFileSync(FILE, 'utf8');
|
|
const field = (name) => { const m = new RegExp(`"${name}":\\s*([0-9]+)`).exec(baseText); return m ? +m[1] : undefined; };
|
|
const EPOCH = field('pow_epoch_blocks');
|
|
const LEAD = field('pow_epoch_lead');
|
|
const DAY_MS = field('pow_day_ms');
|
|
const asText = (v) => v == null ? NEVER : String(v);
|
|
const override = `${TMP}/override.json`;
|
|
writeFileSync(override, mergeOverrideText(baseText, {
|
|
genesis_bits: GENESIS_BITS, skip_proof_of_work: false,
|
|
program_class_v3_activation_daa: '0', program_class_v4_activation_daa: '0', program_class_v4_signal_window_daa: String(CLASS_WINDOW),
|
|
program_class_v5_activation_daa: asText(FLOOR),
|
|
latency_ladder_activation_daa: '0', latency_ladder_window_daa: String(LADDER_WINDOW),
|
|
}));
|
|
const DROPPED = await probeUnknownKeys(IGNEUMD, override, { tmp: `${TMP}/probe`, port: BASE + 90, suffix: SUFFIX, log: (m) => console.log(`${new Date().toISOString().slice(11, 23)} ${m}`) });
|
|
// the first epoch whose seed block (the last chain block below L e - lead) can have seven full windows below it
|
|
const firstFull = (w) => { let e = 1; while (e * EPOCH - LEAD - 1 < WINDOWS * w) e++; return e; };
|
|
const FIRST_STEP_EPOCH = firstFull(LADDER_WINDOW);
|
|
const FIRST_FLIP_EPOCH = Math.max(firstFull(CLASS_WINDOW), 1); // the previous epoch must be v4: from genesis here
|
|
const FLOOR_EPOCH = FLOOR == null ? null : Math.ceil(FLOOR / EPOCH);
|
|
const BOUNDARY = FIRST_FLIP_EPOCH * EPOCH;
|
|
log(`override: ${DROPPED.length ? `${DROPPED.length} key(s) the binary does not know dropped (${DROPPED.join(', ')})` : 'every key known to the binary'}`);
|
|
log(`case ${CASE}: ${N} nodes, bytes ${SIGNAL.join('/')}, ladder ${LADDER.join('/')}; class v4 sub-version 3 from genesis at rung 0; class window ${CLASS_WINDOW} DAA x ${WINDOWS} (the first epoch that can cross to v5 is ${FIRST_FLIP_EPOCH}, DAA ${BOUNDARY}), ladder window ${LADDER_WINDOW} DAA x ${WINDOWS} (the first epoch that can step is ${FIRST_STEP_EPOCH}, DAA ${FIRST_STEP_EPOCH * EPOCH}); v5 floor ${FLOOR ?? 'never'} (epoch ${FLOOR_EPOCH ?? 'none'}); stale node ${STALE ?? 'none'}; restart node ${RESTART ?? 'none'} (stop at DAA ${BOUNDARY - RESTART_BEFORE}, start at ${BOUNDARY + RESTART_AFTER}); ${EPOCH} DAA per epoch, lead ${LEAD}; run ${SECS} s or ${EPOCHS} epochs; node ${IGNEUMD}`);
|
|
|
|
// the CPU verifier reproduces the first class v5 pack's id for Devnet 3's genesis seed (the pack carries its stream)
|
|
let dn3 = null;
|
|
if (existsSync(`${DN3_PACK}/state.igsd1`) && existsSync(`${DN3_PACK}/program.json`)) {
|
|
const pj = JSON.parse(readFileSync(`${DN3_PACK}/program.json`, 'utf8'));
|
|
const r = spawnSync(IGNEUM_POW, ['show', '--epoch-hex', pj.seed_bytes, '--program-class', 'v5', '--era-hex', pj.seed_bytes, '--state', `${DN3_PACK}/state.igsd1`], { encoding: 'utf8' });
|
|
const id = /program id ([0-9a-f]{16})/.exec(r.stdout || '')?.[1] ?? null;
|
|
dn3 = { seed: pj.seed_bytes.slice(0, 16), pack_id: String(pj.program_id).replace(/^0x/, ''), cli_id: id, ok: id === DN3_ID && String(pj.program_id).replace(/^0x/, '') === DN3_ID };
|
|
log(`CPU verifier on the Devnet 3 genesis pack: cli ${id} pack ${dn3.pack_id} expected ${DN3_ID}: ${dn3.ok ? 'OK' : 'MISMATCH'}`);
|
|
} else log(`no Devnet 3 genesis pack at ${DN3_PACK}: the e5a4ac5978462156 self-check is skipped`);
|
|
|
|
const started = [];
|
|
class Node {
|
|
constructor(i, connect = []) {
|
|
this.i = i; this.grpcPort = BASE + i * 10; this.p2pPort = BASE + i * 10 + 1; this.jsonPort = BASE + i * 10 + 2; this.execPort = BASE + i * 10 + 3;
|
|
this.connect = connect; this.dir = `${TMP}/n${i}`; this.logFile = `${this.dir}/node.log`; this.restartLine = 0; this.starts = 0;
|
|
}
|
|
get grpc() { return `grpc://127.0.0.1:${this.grpcPort}`; }
|
|
get execRpc() { return `http://127.0.0.1:${this.execPort}`; }
|
|
async start() {
|
|
mkdirSync(this.dir, { recursive: true });
|
|
if (this.starts > 0) this.restartLine = this.logLines().length;
|
|
this.starts++;
|
|
const a = ['--devnet', `--devnet-suffix=${SUFFIX}`, '--nodnsseed', '--disable-upnp', '--nologfiles', '--enable-unsynced-mining', '--utxoindex',
|
|
`--appdir=${this.dir}`, `--rpclisten=127.0.0.1:${this.grpcPort}`, `--rpclisten-json=127.0.0.1:${this.jsonPort}`, `--evm-rpclisten=127.0.0.1:${this.execPort}`,
|
|
`--listen=127.0.0.1:${this.p2pPort}`, `--override-params-file=${override}`, '--loglevel=info', '--yes'];
|
|
if (this.connect.length) a.push(`--connect=${this.connect.join(',')}`); else a.push('--outpeers=0');
|
|
const out = openSync(this.logFile, 'a');
|
|
this.proc = spawn(IGNEUMD, a, { stdio: ['ignore', out, out], env: { ...process.env, IGNEUM_CLASS_SIGNAL: String(SIGNAL[this.i]), IGNEUM_LADDER_SIGNAL: LADDER[this.i] } });
|
|
started.push(this.proc); recordPid(this.proc);
|
|
await sleep(1500);
|
|
if (this.proc.exitCode != null) throw new Error(`n${this.i} exited ${this.proc.exitCode}: ${this.grepLog(/ERROR|Error|error|refused|invalid|unknown field|duplicate/).slice(-3).join(' | ')}`);
|
|
if (this.grepLog(/Address already in use|AddrInUse/).length) throw new Error(`n${this.i} could not bind its ports (a previous run's node is alive): ${this.grepLog(/AddrInUse|Address already in use/)[0].slice(0, 160)}`);
|
|
this.rpc = await connectRpc(`ws://127.0.0.1:${this.jsonPort}`);
|
|
log(`n${this.i} ${this.starts > 1 ? 'RESTARTED' : 'up'} pid ${this.proc.pid} json ${this.jsonPort} p2p ${this.p2pPort} exec ${this.execPort}, byte ${SIGNAL[this.i]}, ladder ${LADDER[this.i]}${this.starts > 1 ? ' (own datadir kept)' : ''}`);
|
|
return this;
|
|
}
|
|
async stop() {
|
|
try { this.rpc?.close?.(); } catch { }
|
|
this.rpc = null;
|
|
const p = this.proc; if (!p) return;
|
|
try { p.kill('SIGINT'); } catch { }
|
|
for (let i = 0; i < 60 && p.exitCode == null && p.signalCode == null; i++) await sleep(250);
|
|
if (p.exitCode == null && p.signalCode == null) { try { p.kill('SIGKILL'); } catch { } await sleep(500); }
|
|
this.proc = null;
|
|
}
|
|
logLines() { try { return readFileSync(this.logFile, 'utf8').split('\n'); } catch { return []; } }
|
|
grepLog(re, from = 0) { return this.logLines().slice(from).filter(l => re.test(l)); }
|
|
async dag() { try { return await this.rpc.call('getBlockDagInfo'); } catch (e) { return { error: e.message }; } }
|
|
}
|
|
const miners = {};
|
|
function miner(i, node, secs) {
|
|
const out = openSync(`${TMP}/cpu${i}.log`, 'a');
|
|
const extra = ['--exec-rpc', node.execRpc];
|
|
if (STALE === i) extra.push('--freeze-state');
|
|
const p = spawn(CPU_MINER, ['mine', node.grpc, '1', String(secs), `cpu${i}`, '--engine', 'igneum-pow', '--payout-label', `cpu${i}`, '--status-secs', '30', '--no-vote', ...extra], { stdio: ['ignore', out, out], env: { ...process.env, IGNEUM_POW_DAY_MS: String(DAY_MS) } });
|
|
started.push(p); recordPid(p); miners[i] = p;
|
|
return p;
|
|
}
|
|
async function stopMiner(i) { const p = miners[i]; if (!p) return; try { p.kill('SIGINT'); } catch { } for (let k = 0; k < 40 && p.exitCode == null && p.signalCode == null; k++) await sleep(250); if (p.exitCode == null && p.signalCode == null) { try { p.kill('SIGKILL'); } catch { } } miners[i] = null; }
|
|
async function stopAll() {
|
|
for (const p of started.slice().reverse()) { try { p.kill('SIGINT'); } catch { } }
|
|
await sleep(1500);
|
|
for (const p of started) { try { p.kill('SIGKILL'); } catch { } }
|
|
}
|
|
async function finish(code) { try { mkdirSync(OUT, { recursive: true }); copyFileSync(LOG, `${OUT}/${CASE}.log`); } catch { } await stopAll(); process.exit(code); }
|
|
process.on('SIGINT', async () => { await finish(130); });
|
|
process.on('unhandledRejection', async (e) => { log(`FAILED: ${e?.stack || e}`); await finish(3); });
|
|
const minerLog = (i) => { try { return readFileSync(`${TMP}/cpu${i}.log`, 'utf8').split('\n'); } catch { return []; } };
|
|
// the miner stamps every line with epoch seconds (`1791374997.353 ACCEPTED block ...`)
|
|
const lineWall = (l) => { const m = /^(\d{10}\.\d+) /.exec(l); return m ? +m[1] * 1000 : null; };
|
|
const acceptedAfter = (i, wallMs, freshOnly = false) => wallMs == null ? null : minerLog(i).filter(l => /ACCEPTED block/.test(l) && (lineWall(l) ?? 0) >= wallMs && !(freshOnly && /gens old/.test(l))).length;
|
|
const STEP_LINE = /Latency ladder step by miner signal: epoch (\d+) moves to rung (\d+) \((\d+) shadow passes, from rung (\d+)\).*?weakest up (\d+) bps/;
|
|
const SIGNAL_LINE = /Program class v5 by miner signal: epoch (\d+) \(share (\d+) bps.*?weakest (\d+) bps/;
|
|
const FLOOR_LINE = /Program class v5 from the override file: enabled, the floor at epoch (\d+)/;
|
|
const LADDER_ACTIVE = /Latency ladder active: rungs/;
|
|
const LADDER_OWN = /Latency ladder signal from IGNEUM_LADDER_SIGNAL: this node signals (\w+)/;
|
|
const CLASS_OWN = /Program class signal from IGNEUM_CLASS_SIGNAL: this node signals object version (\d+)/;
|
|
const WINDOW_LINE = /Program class v4 signal window from the override file/;
|
|
const REJECT_LINE = /PoW rejected|Rejected block|rejected block/i;
|
|
// the restart step's fault class (the 0.3.20 line, 12:16Z: "While lock file .../meta/LOCK: Resource temporarily unavailable")
|
|
const LOCK_FAULT = /lock file|LOCK: Resource temporarily unavailable|panicked at|thread '[^']*' panicked/;
|
|
|
|
const t0 = Date.now();
|
|
const since = () => ((Date.now() - t0) / 1000).toFixed(1);
|
|
const n0 = await new Node(0).start();
|
|
const nodes = [n0];
|
|
for (let i = 1; i < N; i++) nodes.push(await new Node(i, [`127.0.0.1:${n0.p2pPort}`]).start());
|
|
const honest = nodes.filter(n => n.i !== STALE);
|
|
const stale = STALE == null ? null : nodes[STALE];
|
|
const restartNode = RESTART == null ? null : nodes[RESTART];
|
|
for (const n of nodes) log(`n${n.i}: ${n.grepLog(FLOOR_LINE).map(l => l.replace(/^.*?(Program class v5 from)/, '$1'))[0]?.slice(0, 140) || '(no v5 floor line)'} | ${n.grepLog(CLASS_OWN).map(l => l.replace(/^.*?(this node signals)/, '$1'))[0] || '(no class signal line)'} | ${n.grepLog(LADDER_OWN).map(l => l.replace(/^.*?(this node signals)/, '$1'))[0] || '(no ladder signal line)'}`);
|
|
log(`n0 digest: ${n0.grepLog(/Consensus params digest/).map(l => l.replace(/^.*?digest: /, '').slice(0, 16)).join(' ')}`);
|
|
nodes.forEach(n => miner(n.i, n, SECS));
|
|
const pay = devAddress('fast-time-v5x');
|
|
|
|
// per epoch: the template's class, rung, shares; the seed and era; the state stream of the seed block from every executing honest node
|
|
const epochs = new Map();
|
|
const streams = new Map(); // epoch -> { path, root, records, block, roots: { nodeIdx: root } }
|
|
let flip = null, step = null, lastEpoch = -1, lastReport = 0, lastDaa = 0, endAt = null, lastStreamTry = 0;
|
|
const restart = { stopped: null, started: null, resynced: null };
|
|
const samples = [];
|
|
async function fetchStream(node, seedHash) {
|
|
// five seconds: the restarted node's exec RPC hung a fetch at 19:18Z on 7 October and the end-of-run reads with it
|
|
const r = await fetch(node.execRpc, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'igneum_getPowStateLeaves', params: [seedHash] }), signal: AbortSignal.timeout(5000) });
|
|
const j = await r.json();
|
|
if (!j.result?.streamHex) throw new Error(JSON.stringify(j.error || j).slice(0, 160));
|
|
return j.result;
|
|
}
|
|
async function pollStreams() {
|
|
for (const [e, info] of epochs) {
|
|
if (!info.seed) continue;
|
|
const st = streams.get(e) || { path: null, root: null, records: null, block: null, roots: {} };
|
|
for (const n of honest) {
|
|
if (st.roots[n.i] != null || !n.rpc) continue;
|
|
try {
|
|
const res = await fetchStream(n, info.seed);
|
|
st.roots[n.i] = res.stateRoot;
|
|
if (!st.path) { st.path = `${TMP}/state-e${e}.bin`; writeFileSync(st.path, Buffer.from(res.streamHex.slice(2), 'hex')); st.root = res.stateRoot; st.records = res.records; st.block = res.block; log(`stream for epoch ${e} (seed ${info.seed.slice(0, 16)}): root ${String(res.stateRoot).slice(0, 18)} ${res.records} records, from n${n.i}`); }
|
|
} catch { }
|
|
}
|
|
streams.set(e, st);
|
|
}
|
|
}
|
|
while (Date.now() - t0 < SECS * 1000) {
|
|
await sleep(1000);
|
|
let daa = null, epoch = null, cls = null, nextCls = null, eraSeed = null, seed = null, bps5 = null, weak5 = null, sigEpoch = null, rung = null, reps = null, nextRung = null, upWeak = null, stepEpoch = null, sig = null;
|
|
try {
|
|
const t = await n0.rpc.call('getBlockTemplate', { payAddress: pay, extraData: [] });
|
|
const pe = t.powEpoch || t.pow_epoch || {};
|
|
daa = pe.virtualDaaScore ?? t.block?.header?.daaScore; epoch = pe.epochIndex; cls = pe.programClass; nextCls = pe.nextProgramClass;
|
|
eraSeed = pe.eraSeed; seed = pe.epochSeed; bps5 = pe.programClassV5SignalBps; weak5 = pe.programClassV5SignalWeakestBps; sigEpoch = pe.programClassV5SignalEpoch; sig = pe.programClassSignal;
|
|
rung = pe.latencyLadderStep; reps = pe.latencyLadderReps; nextRung = pe.nextLatencyLadderStep; upWeak = pe.latencyLadderUpWeakestBps; stepEpoch = pe.latencyLadderStepEpoch;
|
|
} catch (e) { log(`template: ${e.message}`); }
|
|
if (epoch != null && epoch !== lastEpoch) {
|
|
epochs.set(epoch, { class: cls, rung, reps, firstSeenDaa: daa, at: +since(), wall: Date.now(), eraSeed: eraSeed == null ? null : String(eraSeed), seed: seed == null ? null : String(seed), v5_bps: bps5, v5_weakest_bps: weak5, v5_signal_epoch: sigEpoch ?? null, up_weakest_bps: upWeak, step_epoch: stepEpoch ?? null });
|
|
log(`epoch ${lastEpoch} -> ${epoch} at daa ${daa}, ${since()} s: template class ${cls} (next ${nextCls}) rung ${rung} (${reps} passes, next rung ${nextRung}), v5 share at the sink ${bps5} bps (weakest of 7: ${weak5}), ladder up weakest ${upWeak} bps, this node signals byte ${sig}, v5 by signal at epoch ${sigEpoch ?? 'none'}, step at epoch ${stepEpoch ?? 'none'}`);
|
|
if (step == null && rung > 0) { step = { epoch, daa, rung, reps, at: +since(), wall: Date.now() }; log(`LADDER STEP: the template is rung ${rung} (${reps} shadow passes) from epoch ${epoch} (daa ${daa}) at ${since()} s wall`); }
|
|
if (flip == null && cls === 5) { flip = { epoch, daa, at: +since(), wall: Date.now(), rung, reps }; log(`CLASS SWITCH: the template is class v5 from epoch ${epoch} (daa ${daa}) at rung ${rung} (${reps} passes), ${since()} s wall`); }
|
|
lastEpoch = epoch;
|
|
}
|
|
lastDaa = daa ?? lastDaa;
|
|
if (Date.now() - lastStreamTry > 4000) { lastStreamTry = Date.now(); await pollStreams(); }
|
|
// the restart step: the node leaves before the expected boundary and comes back after it on its own datadir
|
|
if (restartNode && !restart.stopped && daa != null && daa >= BOUNDARY - RESTART_BEFORE) {
|
|
await stopMiner(RESTART); await restartNode.stop();
|
|
restart.stopped = { daa, at: +since(), wall: Date.now() };
|
|
log(`RESTART STEP: n${RESTART} and its miner stopped at daa ${daa} (${since()} s), ${BOUNDARY - daa} DAA before the expected v5 boundary ${BOUNDARY}`);
|
|
}
|
|
if (restartNode && restart.stopped && !restart.started && daa != null && daa >= BOUNDARY + RESTART_AFTER) {
|
|
try { await restartNode.start(); } catch (e) { log(`RESTART STEP FAILED: ${e.message}`); restart.started = { daa, at: +since(), wall: Date.now(), failed: e.message }; }
|
|
// the miner waits for the resync (--restart-miner-hold-s cap): on the 63524e28 run a miner restarted at once, with
|
|
// unsynced mining on, extended the node's own branch from the split, which outweighed the 24 honest blocks the node could
|
|
// validate (the v5 headers past the boundary deferred), so the executor followed the fork and the IBD catch-up waited forever
|
|
if (!restart.started) { restart.started = { daa, at: +since(), wall: Date.now() }; log(`RESTART STEP: n${RESTART} started on its own datadir at daa ${daa} (${since()} s), ${daa - BOUNDARY} DAA after the boundary; its miner waits for the resync (at most ${MINER_HOLD} s)`); }
|
|
}
|
|
if (restartNode && restart.started && !restart.started.failed && !restart.minerStarted && (restart.resynced || Date.now() - restart.started.wall > MINER_HOLD * 1000)) {
|
|
miner(RESTART, restartNode, Math.max(60, SECS - Math.floor((Date.now() - t0) / 1000)));
|
|
restart.minerStarted = { daa, at: +since(), wall: Date.now(), after_resync: !!restart.resynced };
|
|
log(`RESTART STEP: n${RESTART}'s miner restarted with its key at daa ${daa} (${since()} s), ${restart.resynced ? 'after the resync' : `without a resync inside ${MINER_HOLD} s`}`);
|
|
}
|
|
if (restartNode && restart.started && !restart.started.failed && !restart.resynced && restartNode.rpc) {
|
|
const [a, b] = await Promise.all([n0.dag(), restartNode.dag()]);
|
|
if (a.sink && a.sink === b.sink) { restart.resynced = { daa, at: +since(), after_s: +(((Date.now() - restart.started.wall) / 1000).toFixed(1)), blocks: b.blockCount }; log(`RESTART STEP: n${RESTART} at n0's sink ${String(a.sink).slice(0, 16)} (${b.blockCount} blocks) ${restart.resynced.after_s} s after its start`); }
|
|
}
|
|
if (Date.now() - lastReport > 15000) {
|
|
lastReport = Date.now();
|
|
const counts = await Promise.all(nodes.map(async n => { if (!n.rpc) return 'down'; const d = await n.dag(); return d.error ? '?' : `${d.blockCount}/${String(d.sink).slice(0, 8)}`; }));
|
|
log(`t=${since()} s daa ${daa} epoch ${epoch} class ${cls} rung ${rung} v5 ${bps5} bps (weakest ${weak5}) up weakest ${upWeak} blocks/sink per node ${counts.join(' ')}`);
|
|
samples.push({ t: +since(), daa, epoch, class: cls, rung, v5_bps: bps5, v5_weakest_bps: weak5, up_weakest_bps: upWeak, nodes: counts });
|
|
}
|
|
// the end: three epochs after the flip (two refreshes for the stale node, two epochs of v5 at the rung), or --epochs epochs
|
|
if (flip != null && daa != null && daa >= (flip.epoch + 3) * EPOCH) { endAt = +since(); break; }
|
|
if (flip == null && daa != null && daa >= EPOCHS * EPOCH) { endAt = +since(); break; }
|
|
}
|
|
await sleep(3000);
|
|
await pollStreams();
|
|
// the end-of-run reads under a cap: a hung RPC must not hold the box (the first cross run, 19:18Z)
|
|
setTimeout(async () => { log('FAILED: the end-of-run reads did not finish inside 180 s'); await finish(5); }, 180_000).unref();
|
|
|
|
// ---- the end-of-run reads ------------------------------------------------------------------------------------------
|
|
const dag = await Promise.all(nodes.map(n => n.rpc ? n.dag() : Promise.resolve({ error: 'down' })));
|
|
const genesis = dag[0].pruningPointHash;
|
|
async function allBlocks(n) {
|
|
const out = []; let low = genesis; const seen = new Set();
|
|
for (let round = 0; round < 500; round++) {
|
|
const r = await n.rpc.call('getBlocks', { lowHash: low, includeBlocks: true, includeTransactions: false });
|
|
const blocks = r.blocks || [];
|
|
let added = 0;
|
|
for (const b of blocks) { const h = b.verboseData?.hash || b.header?.hash; if (seen.has(h)) continue; seen.add(h); out.push({ hash: h, daa: +b.header.daaScore, version: +b.header.version, chain: !!b.verboseData?.isChainBlock }); added++; }
|
|
if (!blocks.length || added === 0) break;
|
|
low = (r.blockHashes || []).at(-1) || blocks.at(-1).verboseData?.hash; if (!low) break;
|
|
}
|
|
return out;
|
|
}
|
|
let blocks = [];
|
|
try { blocks = await allBlocks(n0); } catch (e) { log(`getBlocks: ${e.message}`); }
|
|
const before = blocks.filter(b => b.daa < BOUNDARY), after = blocks.filter(b => b.daa >= BOUNDARY);
|
|
const objectBytes = blocks.reduce((m, b) => { const v = (b.version >> 8) & 0x3f; m[v] = (m[v] || 0) + 1; return m; }, {});
|
|
const bitsOf = (v) => (v & 0x8000) ? 'up' : (v & 0x4000) ? 'down' : 'none';
|
|
const ladderBits = blocks.reduce((m, b) => { const k = bitsOf(b.version); m[k] = (m[k] || 0) + 1; return m; }, {});
|
|
const lowBytes = new Set(blocks.filter(b => b.daa > 0).map(b => b.version & 0xff));
|
|
const byte6Share = blocks.length ? Math.round(10000 * blocks.filter(b => ((b.version >> 8) & 0x3f) === V5_BYTE).length / blocks.length) : 0;
|
|
const upShare = blocks.length ? Math.round(10000 * blocks.filter(b => bitsOf(b.version) === 'up').length / blocks.length) : 0;
|
|
|
|
// program ids per epoch from every miner's ready line; the CLI's id for the same seed, class, era, rung and state
|
|
const programs = new Map();
|
|
for (const n of nodes) for (const l of minerLog(n.i)) {
|
|
const m = /epoch seed ([0-9a-f]{64}) day (\d+) \(daa (\d+)\): program and 256 MiB cache ready in ([\d.]+) ms; class (v\d) program id ([0-9a-f]{16})/.exec(l);
|
|
if (!m) continue;
|
|
const k = m[1]; const e = programs.get(k) || { seedHex: k, seed: k.slice(0, 16), epoch: Math.floor(+m[3] / EPOCH), class: m[5], id: m[6], miners: new Set() };
|
|
if (e.id !== m[6] || e.class !== m[5]) e.disagree = true;
|
|
e.miners.add(n.i); programs.set(k, e);
|
|
}
|
|
function cliId(seedHex, eraHex, cls, reps, statePath) {
|
|
const a = ['show', '--epoch-hex', seedHex, '--program-class', cls, '--era-hex', eraHex];
|
|
if (reps && reps !== RUNG0) a.push('--shadow-reps', String(reps));
|
|
if (statePath) a.push('--state', statePath);
|
|
const r = spawnSync(IGNEUM_POW, a, { encoding: 'utf8' });
|
|
return /program id ([0-9a-f]{16})/.exec(r.stdout || '')?.[1] ?? null;
|
|
}
|
|
const idRows = [];
|
|
for (const e of [...programs.values()].sort((a, b) => a.epoch - b.epoch)) {
|
|
const ep = epochs.get(e.epoch);
|
|
if (!ep || !ep.eraSeed) { idRows.push({ epoch: e.epoch, class: e.class, seed: e.seed, miners_id: e.id, miners: e.miners.size, reps: null, cli: null, cli_other: null, state_root: null, note: 'no template row for this epoch' }); continue; }
|
|
const st = streams.get(e.epoch);
|
|
const reps = ep.reps ?? 0;
|
|
// a seed that is not n0's template seed for the epoch is a FORK row (the first cross run: the restarted node's own chain
|
|
// printed its own epoch 8 and 9 seeds, "1 of 4"); listed, never counted as the main chain's
|
|
const fork = !!ep.seed && e.seedHex !== ep.seed;
|
|
const row = { epoch: e.epoch, class: e.class, rung: ep.rung, reps, seed: e.seed, fork, miners_id: e.id, miners: e.miners.size, disagree: !!e.disagree, state_root: fork ? null : (st?.root ?? null), state_records: fork ? null : (st?.records ?? null), roots_per_node: fork ? {} : (st?.roots ?? {}) };
|
|
if (e.class === 'v5') { row.cli = st?.path && !fork ? cliId(e.seedHex, ep.eraSeed, 'v5', reps, st.path) : null; row.cli_other = cliId(e.seedHex, ep.eraSeed, 'v4', reps, null); row.cli_rung0 = st?.path ? cliId(e.seedHex, ep.eraSeed, 'v5', 0, st.path) : null; }
|
|
else { row.cli = cliId(e.seedHex, ep.eraSeed, e.class, reps, null); row.cli_other = cliId(e.seedHex, ep.eraSeed, e.class, reps === RUNG0 ? RUNG1 : 0, null); }
|
|
idRows.push(row);
|
|
}
|
|
const mainRows = idRows.filter(r => !r.fork), forkRows = idRows.filter(r => r.fork);
|
|
const v4Rows = mainRows.filter(r => r.class === 'v4'), v5Rows = mainRows.filter(r => r.class === 'v5');
|
|
// the miners expected on a main-chain row: every honest node's, less the restarted node while it is off the chain
|
|
const MINERS_EXPECTED = Math.max(2, honest.length - (restartNode ? 1 : 0));
|
|
const rung0Rows = v4Rows.filter(r => r.reps === RUNG0 || r.reps === 0), rung1Rows = v4Rows.filter(r => r.reps === RUNG1);
|
|
|
|
// rejections, acceptance, the lines
|
|
// a chain refusal is `rejected nonce=`; a `submit error ... RPC request timeout` is the node's RPC under load (five on the
|
|
// first cross run at load 338 from another lane's build), counted apart and reported, never a PoW refusal
|
|
const rejectedMiner = nodes.map(n => minerLog(n.i).filter(l => /rejected nonce=/.test(l)).length);
|
|
const submitTimeouts = nodes.map(n => minerLog(n.i).filter(l => /submit error/.test(l)).length);
|
|
const rejectedNode = nodes.map(n => n.grepLog(REJECT_LINE).length);
|
|
const accepted = nodes.map(n => minerLog(n.i).filter(l => /ACCEPTED block/.test(l)).length);
|
|
const acceptedAfterFlip = nodes.map(n => acceptedAfter(n.i, flip?.wall));
|
|
const refreshEpoch = flip ? epochs.get(flip.epoch + 1) : null;
|
|
// a block the stale miner built on the previous epoch's template ("template N gens old") and submitted seconds after the
|
|
// boundary is that epoch's block, valid under that epoch's stream (one such on the first cross run); a block on a fresh
|
|
// template of the new epoch with the frozen stream is what the rule refuses
|
|
const staleAcceptedAfterRefresh = stale && refreshEpoch ? acceptedAfter(STALE, refreshEpoch.wall, true) : null;
|
|
const staleOldTemplateAfterRefresh = stale && refreshEpoch ? acceptedAfter(STALE, refreshEpoch.wall) - staleAcceptedAfterRefresh : null;
|
|
const stepLines = nodes.map(n => n.grepLog(STEP_LINE).map(l => l.replace(/^.*?(Latency ladder step by miner signal)/, '$1')));
|
|
const stepParsed = stepLines.map(ls => { const m = ls[0] && STEP_LINE.exec(ls[0]); return m ? { epoch: +m[1], rung: +m[2], reps: +m[3], weakest_up: +m[5] } : null; });
|
|
// the 0.3.23 line logs one step twice (the template's lead path and the boundary); a second STEP is a new (epoch, rung) pair
|
|
const distinctSteps = stepLines.map(ls => new Set(ls.map(l => { const m = STEP_LINE.exec(l); return m ? `${m[1]}:${m[2]}` : l; })).size);
|
|
// the rule allows the next step once the oldest of seven windows begins at or after the step's seed block (DAA L e - lead - 1),
|
|
// so the earliest second step is the first epoch whose seed block is at least seven windows above it; a step line for an
|
|
// earlier epoch is a fault, one at or after it is the rule at work (the 959b57c9 run: epoch 12 moves to rung 2, decided
|
|
// after the observation window closed at DAA 660 while the end-of-run reads ran)
|
|
const stepSeedDaa = step ? step.epoch * EPOCH - LEAD - 1 : null;
|
|
let EARLIEST_SECOND_STEP = null;
|
|
if (step) { EARLIEST_SECOND_STEP = step.epoch + 1; while (EARLIEST_SECOND_STEP * EPOCH - LEAD - 1 < stepSeedDaa + WINDOWS * LADDER_WINDOW) EARLIEST_SECOND_STEP++; }
|
|
const stepEpochsLogged = stepLines.map(ls => ls.map(l => { const m = STEP_LINE.exec(l); return m ? +m[1] : null; }).filter(e => e != null));
|
|
const earlySecondSteps = step ? stepEpochsLogged.flat().filter(e => e !== step.epoch && e < EARLIEST_SECOND_STEP) : [];
|
|
const laterSteps = step ? [...new Set(stepEpochsLogged.flat().filter(e => e !== step.epoch))] : [];
|
|
const signalLines = nodes.map(n => n.grepLog(SIGNAL_LINE).map(l => l.replace(/^.*?(Program class v5 by miner signal)/, '$1')));
|
|
const signalParsed = signalLines.map(ls => { const m = ls[0] && SIGNAL_LINE.exec(ls[0]); return m ? { epoch: +m[1], share: +m[2], weakest: +m[3] } : null; });
|
|
const floorLines = nodes.map(n => n.grepLog(FLOOR_LINE)[0]?.replace(/^.*?(Program class v5 from)/, '$1') ?? null);
|
|
const sinks = dag.map(d => String(d.sink || d.error || '?').slice(0, 16));
|
|
const counts = dag.map(d => d.blockCount ?? '?');
|
|
const honestIdx = honest.map(n => n.i);
|
|
const maxEpochSeen = Math.max(-1, ...epochs.keys());
|
|
const epochRows = [...epochs.entries()].sort((a, b) => a[0] - b[0]);
|
|
const afterStep = step ? epochRows.filter(([e]) => e > step.epoch).map(([, v]) => v.rung) : [];
|
|
const v5Epochs = epochRows.filter(([, v]) => v.class === 5);
|
|
const restartFaults = restartNode && restart.started && !restart.started.failed ? restartNode.grepLog(LOCK_FAULT, restartNode.restartLine) : (restartNode?.grepLog(LOCK_FAULT) ?? []);
|
|
// the first cross run (19:14Z): the restarted node's IBD aborted on every try with "proof-of-work check refused, class v5 needs
|
|
// the execution state after the epoch's seed block ... no published state stream" (headers-first IBD validates the v5 headers
|
|
// before its executor has passed the seed block it stopped short of), and with unsynced mining on it mined its own fork
|
|
const IBD_REFUSED = /IBD with peer .* completed with error|proof-of-work check refused|class v5 needs the execution state/;
|
|
const restartIbdRefusals = restartNode && restart.started && !restart.started.failed ? restartNode.grepLog(IBD_REFUSED, restartNode.restartLine) : [];
|
|
const restartAcceptedAfter = restartNode && restart.minerStarted ? acceptedAfter(RESTART, restart.minerStarted.wall) : null;
|
|
const CATCHUP_DONE = /IBD: class v5 catch-up done/;
|
|
const restartCatchupDone = restartNode && restart.started && !restart.started.failed ? restartNode.grepLog(CATCHUP_DONE, restartNode.restartLine) : [];
|
|
// the stream roots agree across the executing honest nodes wherever two or more answered
|
|
const rootRows = [...streams.entries()].map(([e, st]) => ({ epoch: +e, roots: Object.values(st.roots) })).filter(r => r.roots.length >= 2);
|
|
|
|
const checks = {
|
|
// the start: class v4 sub-version 3 at rung 0, every node signalling what it was told, the window and ladder on
|
|
started_on_class_v4_at_rung_0: epochs.get(0)?.class === 4 && (epochs.get(0)?.reps === RUNG0 || epochs.get(0)?.rung === 0),
|
|
every_node_signals_its_byte: nodes.every((n, i) => n.grepLog(CLASS_OWN).some(l => +CLASS_OWN.exec(l)[1] === SIGNAL[i])),
|
|
every_node_signals_its_ladder_bits: nodes.every((n, i) => n.grepLog(LADDER_OWN).some(l => LADDER_OWN.exec(l)[1] === LADDER[i])),
|
|
window_and_ladder_lines_on_every_node: nodes.every(n => n.grepLog(WINDOW_LINE).length > 0 && n.grepLog(LADDER_ACTIVE).length > 0),
|
|
v5_enabled_by_the_floor_on_every_node: FLOOR == null || floorLines.every(l => l && l.includes(`the floor at epoch ${FLOOR_EPOCH} `)),
|
|
chain_carries_the_bytes: blocks.length > 0 && [...lowBytes].every(v => v === 2) && Object.keys(objectBytes).every(v => SIGNAL.includes(+v) || (+v === 0 && objectBytes[v] === 1)) && Object.keys(ladderBits).every(k => LADDER.includes(k) || (k === 'none' && ladderBits[k] === 1)),
|
|
cli_reproduces_the_devnet_3_genesis_id: dn3 == null ? null : dn3.ok,
|
|
// the ladder: one rung by miner signal, at the first full-window epoch, on every node, and no second step
|
|
template_stepped_to_rung_1: step != null && step.rung === 1 && step.reps === RUNG1,
|
|
stepped_at_the_first_full_window_epoch: step != null && step.epoch === FIRST_STEP_EPOCH,
|
|
step_line_on_every_node_same_epoch: stepParsed.every(s => s != null && s.rung === 1) && new Set(stepParsed.map(s => s?.epoch)).size === 1 && stepParsed[0]?.epoch === step?.epoch,
|
|
step_weakest_up_at_or_above_threshold: stepParsed.every(s => s != null && s.weakest_up >= LADDER_THRESHOLD),
|
|
no_second_step_before_the_rule_allows: step != null && afterStep.length >= 2 && afterStep.every(r => r === 1) && earlySecondSteps.length === 0,
|
|
rung0_ids_equal_the_cli_rung0_id: rung0Rows.length > 0 && rung0Rows.every(r => r.cli != null && r.cli === r.miners_id && !r.disagree),
|
|
rung1_ids_equal_the_cli_rung1_id: rung1Rows.length > 0 && rung1Rows.every(r => r.cli != null && r.cli === r.miners_id && !r.disagree && r.cli_other !== r.miners_id),
|
|
// the crossing: v5 by signal at byte 6 exactly, at the first full-window epoch, before the floor, on every honest node
|
|
template_switched_to_v5: flip != null,
|
|
crossed_at_the_first_full_window_epoch: flip != null && flip.epoch === FIRST_FLIP_EPOCH,
|
|
crossed_after_the_rung_step: flip != null && step != null && flip.epoch > step.epoch,
|
|
crossed_before_the_floor: flip != null && (FLOOR_EPOCH == null || flip.epoch < FLOOR_EPOCH),
|
|
signal_line_on_every_honest_node_same_epoch: honestIdx.every(i => signalParsed[i] != null) && new Set(honestIdx.map(i => signalParsed[i]?.epoch)).size === 1 && signalParsed[honestIdx[0]]?.epoch === flip?.epoch,
|
|
signal_weakest_at_or_above_threshold: honestIdx.every(i => signalParsed[i] != null && signalParsed[i].weakest >= CLASS_THRESHOLD && signalParsed[i].share >= CLASS_THRESHOLD),
|
|
byte_6_counted_exactly_on_chain: flip != null && byte6Share >= CLASS_THRESHOLD,
|
|
blocks_on_both_sides: before.length > 0 && after.length > 0,
|
|
v5_rides_at_rung_1: flip != null && v5Epochs.length > 0 && v5Epochs.every(([, v]) => v.rung === 1 && v.reps === RUNG1),
|
|
// one program id per epoch on every node, equal to the CPU verifier's class v5 id for that seed, state and rung
|
|
a_stream_for_every_v5_epoch: v5Epochs.length > 0 && v5Epochs.every(([e]) => streams.get(e)?.path),
|
|
v5_ids_equal_the_cli_v5_id: v5Rows.length >= 2 && v5Rows.every(r => r.cli != null && r.cli === r.miners_id && r.miners >= MINERS_EXPECTED && !r.disagree),
|
|
v5_ids_differ_from_the_same_seed_v4_id: v5Rows.length > 0 && v5Rows.every(r => r.cli_other != null && r.cli_other !== r.miners_id),
|
|
v5_ids_differ_from_the_rung0_v5_id: v5Rows.length > 0 && v5Rows.every(r => r.cli_rung0 != null && r.cli_rung0 !== r.miners_id),
|
|
state_roots_agree_across_nodes: rootRows.length > 0 && rootRows.every(r => new Set(r.roots).size === 1),
|
|
a_v5_epoch_per_window_refresh: v5Rows.filter(r => r.state_root).length >= 2 && new Set(v5Rows.filter(r => r.state_root).map(r => r.state_root)).size === v5Rows.filter(r => r.state_root).length,
|
|
// 0 blocks rejected for PoW on the honest nodes; the honest miners kept mining after the flip (which proves the counter)
|
|
zero_rejected_on_honest_nodes: honestIdx.every(i => rejectedNode[i] === 0),
|
|
zero_rejected_by_honest_miners: honestIdx.every(i => rejectedMiner[i] === 0),
|
|
honest_miners_accepted_after_the_flip: flip != null && honestIdx.filter(i => i !== RESTART).every(i => acceptedAfterFlip[i] > 0),
|
|
sinks_agree: new Set(nodes.map(n => sinks[n.i])).size === 1,
|
|
block_counts_agree: new Set(nodes.map(n => String(counts[n.i]))).size === 1,
|
|
// the stale node: its blocks refused from the first refresh after the flip, by its own node, none reaching the honest set
|
|
stale_node_refused_from_the_first_refresh: stale == null ? null : (staleAcceptedAfterRefresh === 0 && rejectedNode[STALE] > 0 && rejectedMiner[STALE] > 0),
|
|
stale_node_mined_before_the_refresh: stale == null ? null : (accepted[STALE] > 0),
|
|
// the restart step across the v5 boundary on a kept datadir
|
|
restarted_across_the_boundary: restartNode == null ? null : (!!restart.stopped && !!restart.started && !restart.started.failed && restart.stopped.daa < BOUNDARY && restart.started.daa >= BOUNDARY),
|
|
no_lock_fault_after_the_restart: restartNode == null ? null : (!!restart.started && !restart.started.failed && restartFaults.length === 0),
|
|
restarted_node_resynced: restartNode == null ? null : (!!restart.resynced && sinks[RESTART] === sinks[0]),
|
|
restarted_node_ibd_not_refused: restartNode == null ? null : (!!restart.started && !restart.started.failed && restartIbdRefusals.length === 0),
|
|
restarted_node_mines_v5_after_the_restart: restartNode == null ? null : (restartAcceptedAfter > 0 && rejectedNode[RESTART] === 0 && !!restart.minerStarted?.after_resync),
|
|
restarted_node_catch_up_done_line: restartNode == null ? null : (restartCatchupDone.length > 0),
|
|
};
|
|
const pass = Object.values(checks).every(v => v === true || v === null);
|
|
const summary = {
|
|
pass, case: CASE, utc: new Date().toISOString(), checks, signals: SIGNAL, ladder: LADDER, class_window: CLASS_WINDOW, ladder_window: LADDER_WINDOW, windows: WINDOWS, class_threshold_bps: CLASS_THRESHOLD, ladder_threshold_bps: LADDER_THRESHOLD,
|
|
floor: FLOOR ?? 'never', floor_epoch: FLOOR_EPOCH, epoch_blocks: EPOCH, lead: LEAD, first_step_epoch: FIRST_STEP_EPOCH, first_flip_epoch: FIRST_FLIP_EPOCH, boundary_daa: BOUNDARY,
|
|
node: IGNEUMD, miner: CPU_MINER, pow: IGNEUM_POW, override_keys_dropped: DROPPED, earliest_second_step_epoch: EARLIEST_SECOND_STEP, later_step_epochs_logged: laterSteps, distinct_steps_per_node: distinctSteps, dn3_genesis_check: dn3, ladder_step: step, class_switch: flip, run_ended_at_s: endAt, final_daa: lastDaa, max_epoch_seen: maxEpochSeen,
|
|
epochs: Object.fromEntries(epochRows), streams: Object.fromEntries([...streams.entries()].map(([e, s]) => [e, { root: s.root, records: s.records, block: s.block, roots_per_node: s.roots }])),
|
|
blocks: { total: blocks.length, before_boundary: before.length, after_boundary: after.length, object_bytes: objectBytes, byte_6_share_bps: byte6Share, ladder_bits: ladderBits, up_share_bps: upShare },
|
|
program_id_rows: idRows, fork_rows: forkRows.length, miners_expected_on_a_row: MINERS_EXPECTED, accepted_per_miner: accepted, accepted_after_flip_per_miner: acceptedAfterFlip, rejected_by_miners: rejectedMiner, submit_timeouts_per_miner: submitTimeouts, rejected_by_nodes: rejectedNode,
|
|
stale: stale ? { node: STALE, accepted_total: accepted[STALE], accepted_after_first_refresh: staleAcceptedAfterRefresh, old_template_blocks_after_refresh: staleOldTemplateAfterRefresh, rejected_by_its_node: rejectedNode[STALE], rejected_lines: rejectedMiner[STALE], first_refresh_epoch: flip ? flip.epoch + 1 : null } : null,
|
|
restart: restartNode ? { node: RESTART, ...restart, lock_fault_lines: restartFaults.slice(0, 5), ibd_refusal_lines: restartIbdRefusals.length, catch_up_done_lines: restartCatchupDone.map(l => l.replace(/^.*?\] /, '').slice(0, 200)), ibd_refusal_example: restartIbdRefusals[0]?.replace(/^.*?\] /, '').slice(0, 300) ?? null, accepted_after_restart: restartAcceptedAfter, rejected_by_its_node: rejectedNode[RESTART] } : null,
|
|
sinks, block_counts: counts, step_lines: stepLines.map(ls => ls.map(l => l.slice(0, 300))), signal_lines: signalLines.map(ls => ls.map(l => l.slice(0, 300))), floor_lines: floorLines, samples,
|
|
};
|
|
writeFileSync(`${TMP}/summary.json`, JSON.stringify(summary, null, 2));
|
|
try { mkdirSync(OUT, { recursive: true }); writeFileSync(`${OUT}/${CASE}.json`, JSON.stringify(summary, null, 2)); } catch (e) { log(`could not write ${OUT}/${CASE}.json: ${e.message}`); }
|
|
log(`SUMMARY ${pass ? 'PASS' : 'FAIL'} (${CASE}) ${summary.utc}: ${step ? `rung ${step.rung} (${step.reps} passes) from epoch ${step.epoch} at DAA ${step.daa}` : 'no rung step'}; ${flip ? `v5 from epoch ${flip.epoch} at DAA ${flip.daa} at rung ${flip.rung}` : 'no v5 epoch'}; epochs ${epochRows.map(([e, v]) => `e${e}:v${v.class}:r${v.rung}:${v.v5_weakest_bps}bps`).join(' ')}; chain bytes ${JSON.stringify(objectBytes)} (${byte6Share} bps at byte 6), ladder bits ${JSON.stringify(ladderBits)}; blocks ${before.length} / ${after.length} across DAA ${BOUNDARY}; rejected miners ${rejectedMiner.join('/')} nodes ${rejectedNode.join('/')} (submit timeouts ${submitTimeouts.join('/')}); accepted after the flip ${acceptedAfterFlip.join('/')}; sinks ${sinks.join(' ')} (${checks.sinks_agree ? 'agree' : 'DIFFER'}) at ${counts.join('/')}; step lines ${stepParsed.filter(Boolean).length}/${N} (epochs ${stepParsed.map(s => s?.epoch ?? '-').join('/')}; later steps logged ${laterSteps.join(',') || 'none'}, earliest allowed ${EARLIEST_SECOND_STEP ?? '-'}), v5 signal lines ${signalParsed.filter(Boolean).length}/${N} (epochs ${signalParsed.map(s => s?.epoch ?? '-').join('/')}, weakest ${signalParsed.map(s => s?.weakest ?? '-').join('/')}); stale n${STALE ?? '-'} accepted after the first refresh ${staleAcceptedAfterRefresh ?? 'n/a'} rejected ${STALE != null ? rejectedNode[STALE] : 'n/a'}; restart n${RESTART ?? '-'} ${restart.stopped ? `stopped at DAA ${restart.stopped.daa}` : 'not stopped'}${restart.started ? `, started at DAA ${restart.started.daa}` : ''}${restart.resynced ? `, resynced ${restart.resynced.after_s} s later` : ''}, lock faults ${restartFaults.length}, IBD refusals ${restartIbdRefusals.length}, accepted after ${restartAcceptedAfter ?? 'n/a'}; dn3 id ${dn3 ? (dn3.ok ? 'OK' : 'MISMATCH') : 'skipped'}`);
|
|
for (const r of idRows) log(`PROGRAM ID epoch ${r.epoch}${r.fork ? ' FORK' : ''} ${r.class} rung ${r.rung ?? '?'} (${r.reps ?? '?'} passes) seed ${r.seed}: miners ${r.miners_id} (${r.miners} of ${N}${r.disagree ? ', DISAGREE' : ''}) cli ${r.cli} other-class ${r.cli_other}${r.cli_rung0 ? ` rung0 ${r.cli_rung0}` : ''}${r.state_root ? ` state ${String(r.state_root).slice(0, 18)} (${r.state_records} records)` : ''} ${r.cli === r.miners_id ? 'OK' : 'MISMATCH'}`);
|
|
for (const [k, v] of Object.entries(checks)) if (v === false) log(`FAILED CHECK ${k}`);
|
|
log(`summary: ${TMP}/summary.json and ${OUT}/${CASE}.json`);
|
|
await finish(pass ? 0 : 1);
|