igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

reliability injector: reads the fake cards by name and switches a box's real GPU off through the app's own card path

Browse source 
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
(cherry picked from commit 663deae80a)
This commit is contained in:
igneum-labs 2026-10-07 11:40:53 +00:00
parent 3d8c3ac308
commit a87bdc2cf3
1 changed files with 21 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
tools/reliability/app-run.mjs
View file

@ -91,12 +91,27 @@ if (!url) { log('no app.url'); process.exit(1); }
async function state() {
try { const r = await fetch(url + 'api/state'); return await r.json(); } catch { return null; }
}
const card = (st) => (st && st.mining && st.mining.cards && st.mining.cards[0]) || {};
// the fake cards only: a pod or a PC also lists its real GPU, which is switched off at start (through the app's own
// card path) and never read by a step
const fakes = (st) => ((st && st.mining && st.mining.cards) || []).filter(c => /^Fake GPU/.test(c.name || ''));
const card = (st) => fakes(st)[0] || {};
let realOff = false;
async function switchRealCardsOff(st) {
if (realOff || !st || !st.mining) return;
const real = (st.mining.cards || []).filter(c => !/^Fake GPU/.test(c.name || '') && c.enabled);
if (!real.length) { realOff = true; return; }
try {
await fetch(url + 'api/cards', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ cards: real.map(c => ({ key: c.key, enabled: false, identities: c.identities || 1 })) }) });
log(`real card(s) switched off for the run: ${real.map(c => c.name).join(', ')}`);
realOff = true;
} catch (e) { log(`api/cards: ${e}`); }
}
const events = [];
let last = '';
async function poll() {
const st = await state();
if (!st) return null;
await switchRealCardsOff(st);
const c = card(st);
const key = [c.state, c.message, c.pid, c.restarts, c.faults, st.node.state, st.node.starts, c.hash_now > 0 ? '>0' : '0'].join('|');
if (key !== last) { last = key; events.push({ t: Date.now(), key }); log(`card ${c.state || '(none)'} pid ${c.pid} restarts ${c.restarts} faults ${c.faults} hash ${Number(c.hash_now || 0).toFixed(1)} node ${st.node.state} starts ${st.node.starts} ${c.message ? '"' + c.message + '"' : ''}`); }
@ -285,15 +300,15 @@ S['one-card-fails'] = async () => {
const exportsBefore = engineLogLines(/export-pack: /).length;
writeFileSync(CTL, 'ok\ndevices 3\ndev 2 selftest\n'); log('fake worker: 3 devices, device 2 fails its self-test');
const tInject = Date.now();
const listed = await until((st) => st.mining.cards.length >= 3, 200000, 'three cards listed');
const listed = await until((st) => fakes(st).length >= 3, 200000, 'three fake cards listed');
const tListed = Date.now();
const twoMine = await until((st) => st.mining.cards.filter(c => c.state === 'mining' && c.hash_now > 0).length >= 2, 300000, 'two healthy cards mining');
const twoMine = await until((st) => fakes(st).filter(c => c.state === 'mining' && c.hash_now > 0).length >= 2, 300000, 'two healthy cards mining');
const tTwo = Date.now();
const held = await until((st) => st.mining.cards.some(c => /not usable on this driver/.test(c.message || '')), 120000, 'the failing card held with the reason');
const held = await until((st) => fakes(st).some(c => /not usable on this driver/.test(c.message || '')), 120000, 'the failing card held with the reason');
await steady(90000);
const st1 = await poll();
const bad = st1 ? st1.mining.cards.find(c => /not usable on this driver/.test(c.message || '')) : null;
const good = st1 ? st1.mining.cards.filter(c => !/not usable/.test(c.message || '')) : [];
const bad = st1 ? fakes(st1).find(c => /not usable on this driver/.test(c.message || '')) : null;
const good = st1 ? fakes(st1).filter(c => !/not usable/.test(c.message || '')) : [];
const exportsAfter = engineLogLines(/export-pack: /).length - exportsBefore;
const reused = engineLogLines(/export-pack: reusing/).length;
verdict('one-card-fails', [