igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

reliability injector: card-appears uses a second fake device (added, removed, revived); an empty device list is read by the engine as no answer

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs 2026-10-07 11:47:52 +00:00
parent 663deae80a
commit 28af10abc8
1 changed files with 112 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

114
tools/reliability/app-run.mjs
View file

@ -17,8 +17,8 @@
// (the ladder), the reason stays on the card in plain words, nothing says "restarted once already"
// and no card is ever "faulted"; the worker is healthy again: mining resumes with no tap
// no-status the miner process is stopped with SIGSTOP: no status line for 90 s, the app restarts it
// card-appears MF-3: the card is absent from the enumeration at start (unplugged, or a problem code) and appears
// two minutes later: the hot-plug pass starts its worker with no tap (Linux and Windows only)
// card-appears MF-3: a second card appears (plugged in, or driven after a driver install): its worker starts with
// no tap; it leaves: its row is marked removed; it comes back: mining again (Linux and Windows only)
// node-silent the node is stopped with SIGSTOP: no sign of life for 120 s, the app restarts the node in-process
// and the miner comes back once it is ready
// orphan-miner MF-7: a stray igneum-miner on this engine's node, not started by it, is killed by the minute sweep;
@ -249,6 +249,116 @@ S['no-status'] = async () => {
], { quiet_to_restart: s(tRestart - tInject), restart_to_mining: s(tBack - tRestart) });
};
S['card-appears'] = async () => {
if (MAC) { verdict('card-appears', [{ ok: true, what: 'skipped on macOS (Apple silicon has no GPU hot-plug; the Metal path enumerates once)' }], {}); return; }
// MF-3: a second card appears in the enumeration (plugged in, or driven after a driver install): the hot-plug pass
// starts its worker with no tap; it leaves (the tool still answers, with one device): its row is marked removed
// and its worker stops; it comes back: revived in its slot, mining again. An EMPTY list is not used: the engine
// reads a tool that lists nothing as "did not answer" and removes no card on it (the right call for a driver crash).
await until(mining, 300000, 'mining on the first card');
writeFileSync(CTL, 'ok\ndevices 2\n'); log('fake worker: a second device appears');
const tAppear = Date.now();
const listed = await until((st) => fakes(st).length >= 2, 200000, 'the second card to be listed by the hot-plug pass');
const tListed = Date.now();
const second = await until((st) => fakes(st).length >= 2 && fakes(st)[1].state === 'mining' && fakes(st)[1].hash_now > 0, 300000, 'the second card to mine with no tap');
const tSecond = Date.now();
writeFileSync(CTL, 'ok\ndevices 1\n'); log('fake worker: the second device leaves');
const tGone = Date.now();
const removed = await until((st) => fakes(st).length >= 2 && (fakes(st)[1].state === 'removed' || fakes(st)[1].removed === true), 200000, 'the hot-plug pass to mark the second card removed');
const tMarked = Date.now();
const stR = await poll();
const firstStill = stR && fakes(stR)[0] && fakes(stR)[0].state === 'mining' && fakes(stR)[0].hash_now > 0;
writeFileSync(CTL, 'ok\ndevices 2\n'); log('fake worker: the second device is back');
const tBack = Date.now();
const revived = await until((st) => fakes(st).length >= 2 && fakes(st)[1].state === 'mining' && fakes(st)[1].hash_now > 0, 300000, 'the second card to mine again with no tap');
const tRevived = Date.now();
verdict('card-appears', [
{ ok: !!listed, what: `the hot-plug pass listed the card that appeared (${s(tListed - tAppear)})` },
{ ok: !!second, what: `its worker started with no tap (${s(tSecond - tAppear)} after it appeared)` },
{ ok: !!removed, what: `the card that left was marked removed (${s(tMarked - tGone)})` },
{ ok: !!firstStill, what: 'the other card kept mining through it' },
{ ok: !!revived, what: `the card that came back mined again with no tap (${s(tRevived - tBack)})` },
], { appear_to_listed: s(tListed - tAppear), appear_to_mining: s(tSecond - tAppear), gone_to_marked: s(tMarked - tGone), back_to_mining: s(tRevived - tBack) });
// leave one device for the steps after this one
writeFileSync(CTL, 'ok\ndevices 1\n');
await until((st) => fakes(st).length >= 2 && fakes(st)[1].state === 'removed', 150000, 'the second card gone again');
};
S['own-restart'] = async () => {
setMode('ok');
const st0 = await until(mining, 300000, 'first mining');
const pid0 = card(st0).pid, r0 = card(st0).restarts;
const tInject = Date.now();
setMode('fast');
const faulted = await until((st, c) => c.faults >= 1, 90000, 'the worker fault to reach the card');
const tFault = Date.now();
setMode('ok');
const back = await until((st, c) => mining(st, c) && c.faults >= 1, 90000, 'mining again after the worker restart');
const tBack = Date.now();
await steady(15000);
const st1 = await poll();
verdict('own-restart', [
{ ok: !!st0, what: 'the card mined with a rate above 0 on the fake worker' },
{ ok: !!faulted && /worker fault|killed by a guard/.test(card(faulted).message || ''), what: `the card showed the worker fault (${JSON.stringify(card(faulted || {}).message)})` },
{ ok: !!back, what: 'mining resumed after the miner restarted its own worker' },
{ ok: st1 && card(st1).pid === pid0 && card(st1).restarts === r0, what: `the app did not restart the miner (pid ${pid0} -> ${card(st1 || {}).pid}, app restarts ${r0} -> ${card(st1 || {}).restarts})` },
], { inject_to_fault_on_card: s(tFault - tInject), fault_to_mining_again: s(tBack - tFault) });
};
S['zero-ladder'] = async () => {
setMode('ok');
const st0 = await until(mining, 120000, 'mining');
const r0 = card(st0).restarts;
const tInject = Date.now();
setMode('zero');
// three rungs: the gaps between consecutive watchdog restarts must grow 10, 30, 120 s (plus the 60 s rule each time)
const marks = [];
let lastR = r0;
const end = Date.now() + 600000;
let faultedSeen = false, onceAlready = false;
while (Date.now() < end && marks.length < 3) {
const st = await poll(); if (!st) break;
const c = card(st);
if (c.state === 'faulted') faultedSeen = true;
if (/restarted once already/.test(c.message || '')) onceAlready = true;
if ((c.restarts || 0) > lastR) { lastR = c.restarts; marks.push({ t: Date.now(), wait: c.restart_in_s, msg: c.message }); log(`rung ${marks.length}: restart_in_s ${c.restart_in_s} "${c.message}"`); }
await sleep(500);
}
setMode('ok');
const back = await until(mining, 400000, 'mining again on its own after the worker is healthy');
const tBack = Date.now();
const waits = marks.map(m => m.wait);
const ladderOk = waits.length === 3 && waits[0] >= 8 && waits[0] <= 10 && waits[1] >= 28 && waits[1] <= 30 && waits[2] >= 118 && waits[2] <= 120;
verdict('zero-ladder', [
{ ok: marks.length === 3, what: `three watchdog restarts observed (${marks.length})` },
{ ok: ladderOk, what: `the restart delays follow the ladder 10, 30, 120 s (saw ${waits.join(', ')})` },
{ ok: marks.every(m => /hash rate 0 for 60 s/.test(m.msg || '')), what: 'the reason stayed on the card in plain words' },
{ ok: !faultedSeen && !onceAlready, what: 'no "faulted" state and no "restarted once already" words' },
{ ok: !!back, what: 'mining resumed on its own once the worker was healthy' },
], { zero_to_rung1: marks[0] ? s(marks[0].t - tInject) : 'n/a', rung_gaps: marks.slice(1).map((m, i) => s(m.t - marks[i].t)).join(', '), healthy_to_mining: s(tBack - (marks[2] ? marks[2].t : tInject)) });
};
S['no-status'] = async () => {
setMode('ok');
const st0 = await until(mining, 400000, 'mining');
const pid0 = card(st0).pid, r0 = card(st0).restarts;
const tInject = Date.now();
process.kill(pid0, 'SIGSTOP');
log(`SIGSTOP miner pid ${pid0}`);
const rs = await until((st, c) => c.restarts > r0 || /no status line/.test(c.message || ''), 150000, 'the watchdog restart');
const tRestart = Date.now();
const back = await until((st, c) => mining(st, c) && c.pid !== pid0 && c.pid > 0, 400000, 'mining on the restarted miner');
const tBack = Date.now();
let gone = false; try { process.kill(pid0, 0); } catch { gone = true; }
if (!gone) { try { process.kill(pid0, 'SIGKILL'); } catch {} }
verdict('no-status', [
{ ok: !!rs && /no status line/.test(card(rs).message || ''), what: `the watchdog restarted the miner for missing status lines (${JSON.stringify(card(rs || {}).message)})` },
{ ok: rs && tRestart - tInject >= 85000 && tRestart - tInject <= 130000, what: `between 85 and 130 s after the miner went quiet (${s(tRestart - tInject)})` },
{ ok: !!back, what: 'mining resumed on a new miner process' },
{ ok: gone, what: 'the stopped miner process was killed' },
], { quiet_to_restart: s(tRestart - tInject), restart_to_mining: s(tBack - tRestart) });
};
S['card-appears'] = async () => {
if (MAC) { verdict('card-appears', [{ ok: true, what: 'skipped on macOS (Apple silicon has no GPU hot-plug; the Metal path enumerates once)' }], {}); return; }
// the card leaves the enumeration (unplugged, or a problem code: the hot-plug pass marks it removed and stops its