igneum/tools/reliability/app-run.mjs

#!/usr/bin/env node
// The app engine's fault injector (docs/plans/miner-faults.md): a scratch Igneum Miner engine with its own private node
// (ports 29960 and 29961, devnet suffix 9960, no peers, unsynced mining allowed) and fake-worker.mjs standing in for
// the GPU worker (the Metal worker on macOS, the OpenCL worker on Linux and Windows), driven through the control file
// and with signals. One step per fault class, each stating what must happen and what must not. Never touches the live
// devnet. Runs on the box (Linux engine and node from target-remote/) or on a Mac.
//
//   node tools/reliability/app-run.mjs --app <igneum-app> --miner <igneum-miner> --node <igneumd> [--only a,b]
//
// Steps, in one engine run:
//   catch-up      MF-1, MF-2: the node is synced (private) but its execution layer holds no record: no worker starts
//                 for 120 s, the card says it waits for the executed tip, the node is NOT restarted by the watchdog;
//                 a CPU block producer then makes blocks, the record appears, the worker starts on its own
//   own-restart   the worker reports jobs done in 0.3 ms: the miner's guard restarts the worker; the app shows the
//                 fault and does NOT restart the miner (same pid, no app restart)
//   zero-ladder   MF-2: jobs complete with 0 hashes: the watchdog restarts the miner at 10 s, then 30 s, then 120 s
//                 (the ladder), the reason stays on the card in plain words, nothing says "restarted once already"
//                 and no card is ever "faulted"; the worker is healthy again: mining resumes with no tap
//   no-status     the miner process is stopped with SIGSTOP: no status line for 90 s, the app restarts it
//   card-appears  MF-3: a second card appears (plugged in, or driven after a driver install): its worker starts with
//                 no tap; it leaves: its row is marked removed; it comes back: mining again (Linux and Windows only)
//   node-silent   the node is stopped with SIGSTOP: no sign of life for 120 s, the app restarts the node in-process
//                 and the miner comes back once it is ready
//   kept-datadir  MF-8 (with --node-old): the previous release's node ran this datadir to a few hundred blocks and was
//                 stopped; the engine's new node must open it and sync (a node that dies inside 10 s is the class)
//   orphan-miner  MF-7: a stray igneum-miner on this engine's node, not started by it, is killed by the minute sweep;
//                 the engine's own miner is left alone
//   one-card-fails MF-4: two more cards appear, one failing its self-test for ever: the healthy two mine, the failing
//                 one is held 30 minutes with the reason on its row, the pack is not exported per failure (Linux)
// The app's own log is in the scratch directory. Every FAULT line the engine wrote is counted at the end.

import { spawn, spawnSync } from 'node:child_process';
import { mkdirSync, rmSync, writeFileSync, existsSync, symlinkSync, readFileSync, appendFileSync, chmodSync, readdirSync } from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname, join } from 'node:path';

const here = dirname(fileURLToPath(import.meta.url));
const args = process.argv.slice(2);
const opt = (n, d) => { const i = args.indexOf(n); return i >= 0 ? args[i + 1] : d; };
const APP = opt('--app');
const MINER = opt('--miner');
const NODE = opt('--node');
// MF-8: a previous release's igneumd; the kept-datadir pre-phase runs it first on the app's node dir, stops it, and the
// engine's own (new) node must come up on that datadir
const NODE_OLD = opt('--node-old');
const ONLY = opt('--only', '').split(',').filter(Boolean);
const SCRATCH = process.env.SCRATCH || `/tmp/igneum-reliability-app-${process.pid}`;
const CHAIN_ID_ENV = {};
const RPC = 29960, P2P = 29961, SUFFIX = 9960;
const MAC = process.platform === 'darwin';
for (const [k, v] of Object.entries({ APP, MINER, NODE })) if (!v || !existsSync(v)) { console.error(`missing ${k} (${v})`); process.exit(2); }

const t0 = Date.now();
const since = () => ((Date.now() - t0) / 1000).toFixed(1);
const log = (...a) => console.log(new Date().toISOString().slice(11, 23), `t=${since()}s`, ...a);
const sleep = (ms) => new Promise(r => setTimeout(r, ms));
const s = (ms) => (ms / 1000).toFixed(1) + ' s';

rmSync(SCRATCH, { recursive: true, force: true });
const bin = join(SCRATCH, 'stage', 'bin'); mkdirSync(bin, { recursive: true });
symlinkSync(NODE, join(bin, 'igneumd'));
symlinkSync(MINER, join(bin, 'igneum-miner'));
chmodSync(join(here, 'fake-worker.mjs'), 0o755);
// the fake worker under the name the engine's detection looks for on this platform
symlinkSync(join(here, 'fake-worker.mjs'), join(bin, MAC ? 'igneum-bench' : 'igneum-worker-opencl'));
const data = join(SCRATCH, 'data'); mkdirSync(join(data, 'app'), { recursive: true });
const logs = join(SCRATCH, 'logs'); mkdirSync(logs, { recursive: true });
const CTL = join(SCRATCH, 'ctl');
const setMode = (m) => { writeFileSync(CTL, m + '\n'); log(`fake worker mode -> ${m}`); };
const cardKey = MAC ? 'apple::Fake GPU' : 'other:0:Fake GPU';
setMode('ok');
writeFileSync(join(data, 'app', 'settings.json'), JSON.stringify({
  setup_done: true, address: '0x4242424242424242424242424242424242424242', address_source: 'pasted', key_saved: true, identities: 1,
  cards: { [cardKey]: { enabled: true, identities: 1, power_pct: 0 } }, vote: false, paused: false, accepted_total: 0,
  auto_update: false, remote_jobs: false, prove: false, dev_fee: false,
}, null, 2));

const env = {
  ...process.env, IGNEUM_APP_DATA: data, IGNEUM_APP_LOGS: logs, IGNEUM_APP_BIN: bin, IGNEUM_APP_RPC_PORT: String(RPC), IGNEUM_APP_P2P_PORT: String(P2P),
  IGNEUM_APP_PEERS: '', IGNEUM_APP_UNSYNCED: '1', IGNEUM_APP_DEVNET_SUFFIX: String(SUFFIX), IGNEUM_APP_STATUS_SECS: '10', FAKE_WORKER_CTL: CTL,
  // easy genesis bits so the CPU block producer of the catch-up step makes blocks on two threads
  IGNEUM_DEVNET_GENESIS_BITS: '0x1f100000',
};
// MF-8 pre-phase: the previous release writes the datadir first
let keptPre = null;
if (NODE_OLD) {
  if (!existsSync(NODE_OLD)) { console.error(`missing --node-old ${NODE_OLD}`); process.exit(2); }
  const ndir = join(data, `devnet-${SUFFIX}`); mkdirSync(ndir, { recursive: true });
  const nodeArgs = ['--devnet', `--devnet-suffix=${SUFFIX}`, '--nodnsseed', '--disable-upnp', '--nologfiles', '--enable-unsynced-mining', '--outpeers=0',
    `--appdir=${ndir}`, `--rpclisten=127.0.0.1:${RPC}`, `--evm-rpclisten=127.0.0.1:${RPC + 180}`, `--listen=127.0.0.1:${P2P}`, '--loglevel=warn', '--yes'];
  const oldLog = join(SCRATCH, 'node-old.log');
  const oldNode = spawn(NODE_OLD, nodeArgs, { stdio: ['ignore', 'pipe', 'pipe'], env: { ...env } });
  oldNode.stdout.on('data', d => appendFileSync(oldLog, d)); oldNode.stderr.on('data', d => appendFileSync(oldLog, d));
  log(`kept-datadir: the previous release's node pid ${oldNode.pid} on ${ndir}`);
  await sleep(5000);
  const producer = spawn(MINER, ['mine', `grpc://127.0.0.1:${RPC}`, '2', '600', 'cpu0', '--engine', 'igneum-pow', '--no-vote', '--payout-label', 'cpu0'], { stdio: ['ignore', 'pipe', 'ignore'] });
  let blocks = 0; producer.stdout.on('data', d => { blocks += (d.toString().match(/ACCEPTED block/g) || []).length; });
  const t0b = Date.now();
  while (blocks < 150 && Date.now() - t0b < 400000) await sleep(2000);
  try { producer.kill('SIGTERM'); } catch {}
  await sleep(3000);
  const tStop = Date.now();
  oldNode.kill('SIGTERM');
  let gone = false; for (let i = 0; i < 60 && !gone; i++) { await sleep(500); try { process.kill(oldNode.pid, 0); } catch { gone = true; } }
  if (!gone) { oldNode.kill('SIGKILL'); await sleep(1000); }
  keptPre = { blocks, stop_s: (Date.now() - tStop) / 1000, gone };
  log(`kept-datadir: ${blocks} blocks written by the previous release, its node stopped in ${keptPre.stop_s.toFixed(1)} s`);
}
const app = spawn(APP, ['--no-open'], { stdio: ['pipe', 'pipe', 'pipe'], env });
const appOut = join(SCRATCH, 'app.out');
app.stdout.on('data', d => appendFileSync(appOut, d)); app.stderr.on('data', d => appendFileSync(appOut, d));
let appExit = null;
app.on('exit', c => { appExit = c; log(`app exited ${c}`); });
const started = [app];
process.on('exit', () => { for (const p of started) { try { p.kill('SIGKILL'); } catch {} } });
process.on('SIGINT', () => process.exit(130));
log(`app pid ${app.pid}, scratch ${SCRATCH}, platform ${process.platform}`);

let url = null;
for (let i = 0; i < 150 && !url; i++) { try { url = readFileSync(join(data, 'app', 'app.url'), 'utf8').trim(); } catch { await sleep(200); } }
if (!url) { log('no app.url'); process.exit(1); }
async function state() {
  try { const r = await fetch(url + 'api/state'); return await r.json(); } catch { return null; }
}
// the fake cards only: a pod or a PC also lists its real GPU, which is switched off at start (through the app's own
// card path) and never read by a step
const fakes = (st) => ((st && st.mining && st.mining.cards) || []).filter(c => /^Fake GPU/.test(c.name || ''));
const card = (st) => fakes(st)[0] || {};
let realOff = false;
async function switchRealCardsOff(st) {
  if (realOff || !st || !st.mining) return;
  const real = (st.mining.cards || []).filter(c => !/^Fake GPU/.test(c.name || '') && c.enabled);
  if (!real.length) { realOff = true; return; }
  try {
    await fetch(url + 'api/cards', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ cards: real.map(c => ({ key: c.key, enabled: false, identities: c.identities || 1 })) }) });
    log(`real card(s) switched off for the run: ${real.map(c => c.name).join(', ')}`);
    realOff = true;
  } catch (e) { log(`api/cards: ${e}`); }
}
const events = [];
let last = '';
async function poll() {
  const st = await state();
  if (!st) return null;
  await switchRealCardsOff(st);
  const c = card(st);
  const key = [c.state, c.message, c.pid, c.restarts, c.faults, st.node.state, st.node.starts, c.hash_now > 0 ? '>0' : '0'].join('|');
  if (key !== last) { last = key; events.push({ t: Date.now(), key }); log(`card ${c.state || '(none)'} pid ${c.pid} restarts ${c.restarts} faults ${c.faults} hash ${Number(c.hash_now || 0).toFixed(1)} node ${st.node.state} starts ${st.node.starts} ${c.message ? '"' + c.message + '"' : ''}`); }
  return st;
}
async function until(pred, timeoutMs, what) {
  const end = Date.now() + timeoutMs;
  while (Date.now() < end) { const st = await poll(); if (st && pred(st, card(st))) return st; if (appExit !== null) return null; await sleep(1000); }
  log(`timeout waiting for: ${what}`);
  return null;
}
const mining = (st, c) => c.state === 'mining' && c.hash_now > 0 && st.node.synced;
const results = [];
function verdict(name, checks, metrics) {
  const pass = checks.every(c => c.ok);
  results.push({ name, pass, checks, metrics });
  log(`${pass ? 'PASS' : 'FAIL'} ${name}`);
  for (const c of checks) log(`  ${c.ok ? 'ok  ' : 'FAIL'} ${c.what}`);
  for (const [k, v] of Object.entries(metrics)) log(`  ${k}: ${v}`);
}
async function steady(ms) { const end = Date.now() + ms; while (Date.now() < end) { await poll(); await sleep(1000); } }
const engineLogLines = (re) => { const out = []; for (const f of readdirSync(logs).filter(f => f.startsWith('app-'))) { for (const l of readFileSync(join(logs, f), 'utf8').split('\n')) if (re.test(l)) out.push(l); } return out; };

const S = {};
S['catch-up'] = async () => {
  // the fake card is present for this step
  setMode('ok');
  const tStart = Date.now();
  // the private node reads synced within seconds; its execution layer holds no record until a block executes
  const synced = await until((st) => st.node.synced, 120000, 'node synced');
  const tSynced = Date.now();
  // 120 s: no worker starts, the card waits on the executed tip, the node is not restarted
  let workerStarted = false, nodeRestarts = 0, waitedForTip = false;
  const end = Date.now() + 120000;
  while (Date.now() < end) {
    const st = await poll(); if (!st) break;
    const c = card(st);
    if (c.pid > 0 || c.state === 'mining' || c.state === 'starting') workerStarted = true;
    if (/execute the tip/.test(c.message || '')) waitedForTip = true;
    nodeRestarts = Math.max(nodeRestarts, (st.node.starts || 1) - 1);
    await sleep(1000);
  }
  // now blocks: a CPU block producer against the app's node makes the first executed record
  // it stays up for the rest of the run: every later step needs an executed tip to exist
  const cpu = spawn(MINER, ['mine', `grpc://127.0.0.1:${RPC}`, '2', '36000', 'cpu', '--engine', 'igneum-pow', '--no-vote', '--payout-label', 'cpu'], { stdio: ['ignore', 'ignore', 'ignore'] });
  started.push(cpu);
  log(`cpu block producer pid ${cpu.pid}`);
  const tBlocks = Date.now();
  const back = await until(mining, 300000, 'the worker to start once the execution layer holds a record');
  const tMining = Date.now();
  const readyLines = engineLogLines(/node readiness: the execution layer reports an executed tip/);
  const gatedCalls = engineLogLines(/holds no record yet/);
  // after the kept-datadir pre-phase the datadir already holds executed blocks: the record exists from the first second
  // and the worker may start at once; the rule then reads as "the readiness line came before the first worker start"
  const readyLine = engineLogLines(/node readiness: the execution layer reports an executed tip/)[0];
  const startLine = engineLogLines(/: worker starting \(pid/)[0];
  const order = readyLine && startLine ? (readyLine.split(' ')[0] <= startLine.split(' ')[0]) : false;
  verdict('catch-up', [
    { ok: !!synced, what: 'the private node read synced' },
    keptPre
      ? { ok: order, what: 'the datadir carried records (kept): the readiness line preceded the first worker start' }
      : { ok: !workerStarted, what: 'no worker started while the execution layer held no record (120 s)' },
    keptPre ? { ok: true, what: 'the wait for the executed tip does not apply on a kept datadir' } : { ok: waitedForTip, what: 'the card said it waits for the executed tip' },
    { ok: nodeRestarts === 0, what: `the node watchdog did not restart the node during the catch-up (restarts ${nodeRestarts})` },
    { ok: readyLines.length >= 1, what: 'the engine logged the executed tip when it appeared' },
    { ok: !!back, what: 'the worker started on its own once the record existed' },
  ], { synced_after: s(tSynced - tStart), blocks_to_mining: s(tMining - tBlocks), gated_exec_calls_refused: gatedCalls.length });
};

S['own-restart'] = async () => {
  setMode('ok');
  const st0 = await until(mining, 300000, 'first mining');
  const pid0 = card(st0).pid, r0 = card(st0).restarts;
  const tInject = Date.now();
  setMode('fast');
  const faulted = await until((st, c) => c.faults >= 1, 90000, 'the worker fault to reach the card');
  const tFault = Date.now();
  setMode('ok');
  const back = await until((st, c) => mining(st, c) && c.faults >= 1, 90000, 'mining again after the worker restart');
  const tBack = Date.now();
  await steady(15000);
  const st1 = await poll();
  verdict('own-restart', [
    { ok: !!st0, what: 'the card mined with a rate above 0 on the fake worker' },
    { ok: !!faulted && /worker fault|killed by a guard/.test(card(faulted).message || ''), what: `the card showed the worker fault (${JSON.stringify(card(faulted || {}).message)})` },
    { ok: !!back, what: 'mining resumed after the miner restarted its own worker' },
    { ok: st1 && card(st1).pid === pid0 && card(st1).restarts === r0, what: `the app did not restart the miner (pid ${pid0} -> ${card(st1 || {}).pid}, app restarts ${r0} -> ${card(st1 || {}).restarts})` },
  ], { inject_to_fault_on_card: s(tFault - tInject), fault_to_mining_again: s(tBack - tFault) });
};

S['zero-ladder'] = async () => {
  setMode('ok');
  const st0 = await until(mining, 120000, 'mining');
  const r0 = card(st0).restarts;
  const tInject = Date.now();
  setMode('zero');
  // three rungs: the gaps between consecutive watchdog restarts must grow 10, 30, 120 s (plus the 60 s rule each time)
  const marks = [];
  let lastR = r0;
  const end = Date.now() + 600000;
  let faultedSeen = false, onceAlready = false;
  while (Date.now() < end && marks.length < 3) {
    const st = await poll(); if (!st) break;
    const c = card(st);
    if (c.state === 'faulted') faultedSeen = true;
    if (/restarted once already/.test(c.message || '')) onceAlready = true;
    if ((c.restarts || 0) > lastR) {
      lastR = c.restarts;
      // the row's countdown as it stands over the next two seconds (the first read can land on the same tick)
      let wait = c.restart_in_s || 0;
      for (let k = 0; k < 4; k++) { await sleep(500); const s2 = await state(); const c2 = s2 ? card(s2) : {}; wait = Math.max(wait, c2.restart_in_s || 0); }
      marks.push({ t: Date.now(), wait, msg: c.message });
      log(`rung ${marks.length}: restart_in_s ${wait} "${c.message}"`);
    }
    await sleep(500);
  }
  setMode('ok');
  const back = await until(mining, 400000, 'mining again on its own after the worker is healthy');
  const tBack = Date.now();
  const waits = marks.map(m => m.wait);
  const ladderOk = waits.length === 3 && waits[0] >= 8 && waits[0] <= 10 && waits[1] >= 28 && waits[1] <= 30 && waits[2] >= 118 && waits[2] <= 120;
  verdict('zero-ladder', [
    { ok: marks.length === 3, what: `three watchdog restarts observed (${marks.length})` },
    { ok: ladderOk, what: `the restart delays follow the ladder 10, 30, 120 s (saw ${waits.join(', ')})` },
    { ok: marks.every(m => /hash rate 0 for 60 s/.test(m.msg || '')), what: 'the reason stayed on the card in plain words' },
    { ok: !faultedSeen && !onceAlready, what: 'no "faulted" state and no "restarted once already" words' },
    { ok: !!back, what: 'mining resumed on its own once the worker was healthy' },
  ], { zero_to_rung1: marks[0] ? s(marks[0].t - tInject) : 'n/a', rung_gaps: marks.slice(1).map((m, i) => s(m.t - marks[i].t)).join(', '), healthy_to_mining: s(tBack - (marks[2] ? marks[2].t : tInject)) });
};

S['no-status'] = async () => {
  setMode('ok');
  const st0 = await until(mining, 400000, 'mining');
  const pid0 = card(st0).pid, r0 = card(st0).restarts;
  const tInject = Date.now();
  process.kill(pid0, 'SIGSTOP');
  log(`SIGSTOP miner pid ${pid0}`);
  const rs = await until((st, c) => c.restarts > r0 || /no status line/.test(c.message || ''), 150000, 'the watchdog restart');
  const tRestart = Date.now();
  const back = await until((st, c) => mining(st, c) && c.pid !== pid0 && c.pid > 0, 400000, 'mining on the restarted miner');
  const tBack = Date.now();
  let gone = false; try { process.kill(pid0, 0); } catch { gone = true; }
  if (!gone) { try { process.kill(pid0, 'SIGKILL'); } catch {} }
  verdict('no-status', [
    { ok: !!rs && /no status line/.test(card(rs).message || ''), what: `the watchdog restarted the miner for missing status lines (${JSON.stringify(card(rs || {}).message)})` },
    { ok: rs && tRestart - tInject >= 85000 && tRestart - tInject <= 130000, what: `between 85 and 130 s after the miner went quiet (${s(tRestart - tInject)})` },
    { ok: !!back, what: 'mining resumed on a new miner process' },
    { ok: gone, what: 'the stopped miner process was killed' },
  ], { quiet_to_restart: s(tRestart - tInject), restart_to_mining: s(tBack - tRestart) });
};

S['card-appears'] = async () => {
  if (MAC) { verdict('card-appears', [{ ok: true, what: 'skipped on macOS (Apple silicon has no GPU hot-plug; the Metal path enumerates once)' }], {}); return; }
  // MF-3: a second card appears in the enumeration (plugged in, or driven after a driver install): the hot-plug pass
  // starts its worker with no tap; it leaves (the tool still answers, with one device): its row is marked removed
  // and its worker stops; it comes back: revived in its slot, mining again. An EMPTY list is not used: the engine
  // reads a tool that lists nothing as "did not answer" and removes no card on it (the right call for a driver crash).
  await until(mining, 300000, 'mining on the first card');
  writeFileSync(CTL, 'ok\ndevices 2\n'); log('fake worker: a second device appears');
  const tAppear = Date.now();
  const listed = await until((st) => fakes(st).length >= 2, 200000, 'the second card to be listed by the hot-plug pass');
  const tListed = Date.now();
  const second = await until((st) => fakes(st).length >= 2 && fakes(st)[1].state === 'mining' && fakes(st)[1].hash_now > 0, 300000, 'the second card to mine with no tap');
  const tSecond = Date.now();
  writeFileSync(CTL, 'ok\ndevices 1\n'); log('fake worker: the second device leaves');
  const tGone = Date.now();
  const removed = await until((st) => fakes(st).length >= 2 && (fakes(st)[1].state === 'removed' || fakes(st)[1].removed === true), 200000, 'the hot-plug pass to mark the second card removed');
  const tMarked = Date.now();
  const stR = await poll();
  const firstStill = stR && fakes(stR)[0] && fakes(stR)[0].state === 'mining' && fakes(stR)[0].hash_now > 0;
  writeFileSync(CTL, 'ok\ndevices 2\n'); log('fake worker: the second device is back');
  const tBack = Date.now();
  const revived = await until((st) => fakes(st).length >= 2 && fakes(st)[1].state === 'mining' && fakes(st)[1].hash_now > 0, 300000, 'the second card to mine again with no tap');
  const tRevived = Date.now();
  verdict('card-appears', [
    { ok: !!listed, what: `the hot-plug pass listed the card that appeared (${s(tListed - tAppear)})` },
    { ok: !!second, what: `its worker started with no tap (${s(tSecond - tAppear)} after it appeared)` },
    { ok: !!removed, what: `the card that left was marked removed (${s(tMarked - tGone)})` },
    { ok: !!firstStill, what: 'the other card kept mining through it' },
    { ok: !!revived, what: `the card that came back mined again with no tap (${s(tRevived - tBack)})` },
  ], { appear_to_listed: s(tListed - tAppear), appear_to_mining: s(tSecond - tAppear), gone_to_marked: s(tMarked - tGone), back_to_mining: s(tRevived - tBack) });
  // leave one device for the steps after this one
  writeFileSync(CTL, 'ok\ndevices 1\n');
  await until((st) => fakes(st).length >= 2 && fakes(st)[1].state === 'removed', 150000, 'the second card gone again');
};

S['node-silent'] = async () => {
  setMode('ok');
  const st0 = await until(mining, 400000, 'mining');
  const npid = st0.node.pid, starts0 = st0.node.starts;
  const tInject = Date.now();
  process.kill(npid, 'SIGSTOP');
  log(`SIGSTOP node pid ${npid}`);
  const rs = await until((st) => st.node.state === 'restarting' || st.node.starts > starts0, 240000, 'the node restart');
  const tRestart = Date.now();
  const synced = await until((st) => st.node.starts > starts0 && st.node.synced, 180000, 'the restarted node to sync');
  const tSynced = Date.now();
  const back = await until(mining, 400000, 'mining again');
  const tBack = Date.now();
  let gone = false; try { process.kill(npid, 0); } catch { gone = true; }
  if (!gone) { try { process.kill(npid, 'SIGKILL'); } catch {} }
  verdict('node-silent', [
    { ok: !!rs, what: 'the app restarted the node' },
    { ok: rs && tRestart - tInject >= 115000 && tRestart - tInject <= 190000, what: `between 115 and 190 s after the node went quiet (${s(tRestart - tInject)})` },
    { ok: !!synced, what: `the new node synced (starts ${starts0} -> ${synced ? synced.node.starts : '?'})` },
    { ok: !!back, what: 'mining resumed once the node was ready' },
    { ok: gone, what: 'the stopped node process was killed' },
  ], { quiet_to_restart: s(tRestart - tInject), restart_to_synced: s(tSynced - tRestart), quiet_to_mining: s(tBack - tInject) });
};

S['one-card-fails'] = async () => {
  if (MAC) { verdict('one-card-fails', [{ ok: true, what: 'skipped on macOS (one Metal device)' }], {}); return; }
  // MF-4: two more cards appear, one of them failing its self-test for ever; the healthy two must mine on time,
  // the failing one is held with the reason on its row, and the pack is not exported once per failure
  const exportsBefore = engineLogLines(/export-pack: /).length;
  writeFileSync(CTL, 'ok\ndevices 3\ndev 2 selftest\n'); log('fake worker: 3 devices, device 2 fails its self-test');
  const tInject = Date.now();
  const listed = await until((st) => fakes(st).length >= 3, 200000, 'three fake cards listed');
  const tListed = Date.now();
  const twoMine = await until((st) => fakes(st).filter(c => c.state === 'mining' && c.hash_now > 0).length >= 2, 300000, 'two healthy cards mining');
  const tTwo = Date.now();
  const held = await until((st) => fakes(st).some(c => /not usable on this driver/.test(c.message || '')), 120000, 'the failing card held with the reason');
  await steady(90000);
  const st1 = await poll();
  const bad = st1 ? fakes(st1).find(c => /not usable on this driver/.test(c.message || '')) : null;
  const good = st1 ? fakes(st1).filter(c => !/not usable/.test(c.message || '')) : [];
  const exportsAfter = engineLogLines(/export-pack: /).length - exportsBefore;
  const reused = engineLogLines(/export-pack: reusing/).length;
  verdict('one-card-fails', [
    { ok: !!listed, what: 'the hot-plug pass listed the two new cards' },
    { ok: !!twoMine, what: 'the two healthy cards mined' },
    { ok: !!held && bad && bad.restart_in_s >= 1500, what: `the failing card is held 30 minutes with the reason on its row (restart_in_s ${bad ? bad.restart_in_s : '?'}, "${bad ? bad.message : ''}")` },
    { ok: good.length >= 2 && good.every(c => c.state === 'mining' && c.hash_now > 0), what: 'the healthy cards still mine 90 s later (no restart from the failing card)' },
    { ok: exportsAfter <= 6, what: `the pack was exported at most 6 times for three starts and the failures (${exportsAfter}, ${reused} reused)` },
  ], { listed_after: s(tListed - tInject), two_mining_after: s(tTwo - tInject), exports: exportsAfter, exports_reused: reused });
};

S['orphan-miner'] = async () => {
  // MF-7: an igneum-miner the engine did not start, on this engine's node (the fence), is killed by the minute sweep
  const st0 = await until(mining, 300000, 'mining');
  const stray = spawn(MINER, ['mine', `grpc://127.0.0.1:${RPC}`, '1', '3600', 'stray', '--worker', join(here, 'fake-worker.mjs'), '--status-secs', '10', '--no-vote', '--payout-label', 'stray'], { stdio: ['ignore', 'ignore', 'ignore'], env: { ...process.env, FAKE_WORKER_CTL: CTL } });
  started.push(stray);
  const tInject = Date.now();
  log(`stray miner pid ${stray.pid} on the engine's node`);
  let killedAt = null;
  const end = Date.now() + 150000;
  while (Date.now() < end) { await poll(); try { process.kill(stray.pid, 0); } catch { killedAt = Date.now(); break; } await sleep(1000); }
  const lines = engineLogLines(/orphan miner killed/);
  const st1 = await poll();
  const own = st1 ? card(st1) : {};
  let ownAlive = false; try { process.kill(own.pid, 0); ownAlive = true; } catch {}
  verdict('orphan-miner', [
    { ok: !!killedAt, what: `the stray miner was killed by the engine (${killedAt ? s(killedAt - tInject) : 'still alive after 150 s'})` },
    { ok: lines.length >= 1, what: `one log line per kill (${lines.length})` },
    { ok: ownAlive && own.pid > 0, what: `the engine's own miner (pid ${own.pid}) was left alone` },
  ], { inject_to_kill: killedAt ? s(killedAt - tInject) : 'n/a' });
};

S['kept-datadir'] = async () => {
  if (!NODE_OLD) { verdict('kept-datadir', [{ ok: false, what: 'not run: no --node-old given (a previous release\'s igneumd)' }], {}); return; }
  const tStart = Date.now();
  const synced = await until((st) => st.node.synced && st.node.starts >= 1, 240000, 'the new node to come up on the kept datadir');
  const tSynced = Date.now();
  const st1 = await poll();
  const died = engineLogLines(/igneumd exited at once|igneumd exited with code/);
  const rewrite = engineLogLines(/virtual state|rewrit|v1 row/i);
  verdict('kept-datadir', [
    { ok: keptPre && keptPre.blocks >= 100, what: `the previous release wrote the datadir (${keptPre ? keptPre.blocks : 0} blocks) and stopped in ${keptPre ? keptPre.stop_s.toFixed(1) : '?'} s` },
    { ok: !!synced, what: `the new node opened the kept datadir and read synced (${s(tSynced - tStart)} after the engine started)` },
    { ok: died.length === 0, what: `no node exit at start (${died.length} exit lines)` },
    { ok: st1 && st1.node.starts === 1, what: `one node start, no restart loop (starts ${st1 ? st1.node.starts : '?'})` },
  ], { blocks_by_old_node: keptPre ? keptPre.blocks : 0, old_node_stop_s: keptPre ? keptPre.stop_s.toFixed(1) : 'n/a', new_node_synced_after: s(tSynced - tStart), rewrite_lines: rewrite.length });
};

const order = ['kept-datadir', 'catch-up', 'card-appears', 'own-restart', 'zero-ladder', 'no-status', 'node-silent', 'one-card-fails', 'orphan-miner'];
const names = (ONLY.length ? order.filter(n => ONLY.includes(n)) : order).filter(n => n !== 'kept-datadir' || NODE_OLD);
for (const n of names) {
  log(`=== ${n}`);
  try { await S[n](); } catch (e) { log(`step ${n} threw: ${e.stack || e}`); results.push({ name: n, pass: false, checks: [{ ok: false, what: String(e) }], metrics: {} }); }
}
const faults = engineLogLines(/ FAULT class=/);
log(`FAULT lines the engine wrote: ${faults.length}`);
for (const f of faults.slice(0, 12)) log(`  ${f.slice(0, 200)}`);
app.stdin.write('quit\n');
await sleep(8000);
const report = { date: new Date().toISOString(), app: APP, miner: MINER, node: NODE, platform: process.platform, scratch: SCRATCH, results, fault_lines: faults.length, transitions: events.map(e => ({ t: new Date(e.t).toISOString(), key: e.key })) };
writeFileSync(join(SCRATCH, 'report.json'), JSON.stringify(report, null, 2));
console.log(JSON.stringify({ ...report, transitions: undefined }, null, 2));
process.exit(results.every(r => r.pass) ? 0 : 1);