igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

reliability injector: the stale duplicate step blocks removed (the card-appears rewrite had sliced before an earlier definition); the rung is read over two seconds

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
(cherry picked from commit 10badc02a3)
This commit is contained in:
igneum-labs 2026-10-07 11:55:12 +00:00
parent 313059f6bd
commit 942f537ff6
1 changed files with 8 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

104
tools/reliability/app-run.mjs
View file

@ -211,7 +211,14 @@ S['zero-ladder'] = async () => {
const c = card(st);
if (c.state === 'faulted') faultedSeen = true;
if (/restarted once already/.test(c.message || '')) onceAlready = true;
if ((c.restarts || 0) > lastR) { lastR = c.restarts; marks.push({ t: Date.now(), wait: c.restart_in_s, msg: c.message }); log(`rung ${marks.length}: restart_in_s ${c.restart_in_s} "${c.message}"`); }
if ((c.restarts || 0) > lastR) {
lastR = c.restarts;
// the row's countdown as it stands over the next two seconds (the first read can land on the same tick)
let wait = c.restart_in_s || 0;
for (let k = 0; k < 4; k++) { await sleep(500); const s2 = await state(); const c2 = s2 ? card(s2) : {}; wait = Math.max(wait, c2.restart_in_s || 0); }
marks.push({ t: Date.now(), wait, msg: c.message });
log(`rung ${marks.length}: restart_in_s ${wait} "${c.message}"`);
}
await sleep(500);
}
setMode('ok');
@ -284,101 +291,6 @@ S['card-appears'] = async () => {
await until((st) => fakes(st).length >= 2 && fakes(st)[1].state === 'removed', 150000, 'the second card gone again');
};
S['own-restart'] = async () => {
setMode('ok');
const st0 = await until(mining, 300000, 'first mining');
const pid0 = card(st0).pid, r0 = card(st0).restarts;
const tInject = Date.now();
setMode('fast');
const faulted = await until((st, c) => c.faults >= 1, 90000, 'the worker fault to reach the card');
const tFault = Date.now();
setMode('ok');
const back = await until((st, c) => mining(st, c) && c.faults >= 1, 90000, 'mining again after the worker restart');
const tBack = Date.now();
await steady(15000);
const st1 = await poll();
verdict('own-restart', [
{ ok: !!st0, what: 'the card mined with a rate above 0 on the fake worker' },
{ ok: !!faulted && /worker fault|killed by a guard/.test(card(faulted).message || ''), what: `the card showed the worker fault (${JSON.stringify(card(faulted || {}).message)})` },
{ ok: !!back, what: 'mining resumed after the miner restarted its own worker' },
{ ok: st1 && card(st1).pid === pid0 && card(st1).restarts === r0, what: `the app did not restart the miner (pid ${pid0} -> ${card(st1 || {}).pid}, app restarts ${r0} -> ${card(st1 || {}).restarts})` },
], { inject_to_fault_on_card: s(tFault - tInject), fault_to_mining_again: s(tBack - tFault) });
};
S['zero-ladder'] = async () => {
setMode('ok');
const st0 = await until(mining, 120000, 'mining');
const r0 = card(st0).restarts;
const tInject = Date.now();
setMode('zero');
// three rungs: the gaps between consecutive watchdog restarts must grow 10, 30, 120 s (plus the 60 s rule each time)
const marks = [];
let lastR = r0;
const end = Date.now() + 600000;
let faultedSeen = false, onceAlready = false;
while (Date.now() < end && marks.length < 3) {
const st = await poll(); if (!st) break;
const c = card(st);
if (c.state === 'faulted') faultedSeen = true;
if (/restarted once already/.test(c.message || '')) onceAlready = true;
if ((c.restarts || 0) > lastR) { lastR = c.restarts; marks.push({ t: Date.now(), wait: c.restart_in_s, msg: c.message }); log(`rung ${marks.length}: restart_in_s ${c.restart_in_s} "${c.message}"`); }
await sleep(500);
}
setMode('ok');
const back = await until(mining, 400000, 'mining again on its own after the worker is healthy');
const tBack = Date.now();
const waits = marks.map(m => m.wait);
const ladderOk = waits.length === 3 && waits[0] >= 8 && waits[0] <= 10 && waits[1] >= 28 && waits[1] <= 30 && waits[2] >= 118 && waits[2] <= 120;
verdict('zero-ladder', [
{ ok: marks.length === 3, what: `three watchdog restarts observed (${marks.length})` },
{ ok: ladderOk, what: `the restart delays follow the ladder 10, 30, 120 s (saw ${waits.join(', ')})` },
{ ok: marks.every(m => /hash rate 0 for 60 s/.test(m.msg || '')), what: 'the reason stayed on the card in plain words' },
{ ok: !faultedSeen && !onceAlready, what: 'no "faulted" state and no "restarted once already" words' },
{ ok: !!back, what: 'mining resumed on its own once the worker was healthy' },
], { zero_to_rung1: marks[0] ? s(marks[0].t - tInject) : 'n/a', rung_gaps: marks.slice(1).map((m, i) => s(m.t - marks[i].t)).join(', '), healthy_to_mining: s(tBack - (marks[2] ? marks[2].t : tInject)) });
};
S['no-status'] = async () => {
setMode('ok');
const st0 = await until(mining, 400000, 'mining');
const pid0 = card(st0).pid, r0 = card(st0).restarts;
const tInject = Date.now();
process.kill(pid0, 'SIGSTOP');
log(`SIGSTOP miner pid ${pid0}`);
const rs = await until((st, c) => c.restarts > r0 || /no status line/.test(c.message || ''), 150000, 'the watchdog restart');
const tRestart = Date.now();
const back = await until((st, c) => mining(st, c) && c.pid !== pid0 && c.pid > 0, 400000, 'mining on the restarted miner');
const tBack = Date.now();
let gone = false; try { process.kill(pid0, 0); } catch { gone = true; }
if (!gone) { try { process.kill(pid0, 'SIGKILL'); } catch {} }
verdict('no-status', [
{ ok: !!rs && /no status line/.test(card(rs).message || ''), what: `the watchdog restarted the miner for missing status lines (${JSON.stringify(card(rs || {}).message)})` },
{ ok: rs && tRestart - tInject >= 85000 && tRestart - tInject <= 130000, what: `between 85 and 130 s after the miner went quiet (${s(tRestart - tInject)})` },
{ ok: !!back, what: 'mining resumed on a new miner process' },
{ ok: gone, what: 'the stopped miner process was killed' },
], { quiet_to_restart: s(tRestart - tInject), restart_to_mining: s(tBack - tRestart) });
};
S['card-appears'] = async () => {
if (MAC) { verdict('card-appears', [{ ok: true, what: 'skipped on macOS (Apple silicon has no GPU hot-plug; the Metal path enumerates once)' }], {}); return; }
// the card leaves the enumeration (unplugged, or a problem code: the hot-plug pass marks it removed and stops its
// worker), then comes back: its worker must start with no tap
await until(mining, 300000, 'mining before the card leaves');
setMode('absent');
const tGone = Date.now();
const gone = await until((st, c) => c.state === 'removed' || c.removed === true || /removed/.test(c.state || ''), 150000, 'the hot-plug pass to mark the card removed');
const tMarked = Date.now();
await steady(30000);
setMode('ok');
const tAppear = Date.now();
const back = await until(mining, 300000, 'the worker to start with no tap once the card is back');
const tBack = Date.now();
verdict('card-appears', [
{ ok: !!gone, what: `the hot-plug pass marked the card removed when it left (${s(tMarked - tGone)})` },
{ ok: !!back, what: 'its worker started with no tap once it was back' },
], { gone_to_marked: s(tMarked - tGone), back_to_mining: s(tBack - tAppear) });
};
S['node-silent'] = async () => {
setMode('ok');
const st0 = await until(mining, 400000, 'mining');