Console: a machine whose app logged a clean quit or an update, with no status line after it, shows 'stopped (quit|update) N ago' instead of 'silent' (parseAppTail moved to relay/lib/parse.mjs, test)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
parent
c207c674b9
commit
c98b6a546d
5 changed files with 76 additions and 41 deletions
|
|
@ -12,7 +12,7 @@
|
|||
// POST post {kind,title,body,who,key?,meta?} one item; with key it upserts
|
||||
// POST sync {items:[...]} bulk upsert by key
|
||||
import { neon, authed, readJson, str, iso } from '../lib/relay.mjs';
|
||||
import { kv, kvNum, lastMatch, FAULT, parseLabel, parseMinerTail, STALE_S, markStale } from '../lib/parse.mjs';
|
||||
import { kv, kvNum, lastMatch, FAULT, parseLabel, parseMinerTail, parseHeader, parseAppTail, STALE_S, markStale } from '../lib/parse.mjs';
|
||||
|
||||
const json = (res, status, obj) => { res.status(status).setHeader('Content-Type', 'application/json; charset=utf-8'); res.end(JSON.stringify(obj)); };
|
||||
const CACHE_MS = 10_000;
|
||||
|
|
@ -55,43 +55,6 @@ function parseNodeTail(tail) {
|
|||
n.last_warning = errs.length ? errs[errs.length - 1].replace(/^\S+ \S+ /, '').slice(0, 200) : null;
|
||||
return n;
|
||||
}
|
||||
const HEADER = /IGNEUM-APP version=(\S+) machine=(\S+) platform=(\S+) node=(\S+)/;
|
||||
function parseHeader(text) {
|
||||
const m = HEADER.exec(text || '');
|
||||
return m ? { version: m[1], id8: m[2], platform: m[3], node: m[4].replace(/^igneumd[_\/]/, '') } : null;
|
||||
}
|
||||
function parseAppTail(tail) {
|
||||
const lines = tail.split('\n');
|
||||
const a = { telemetry: [], events: [], jobs: [] };
|
||||
let m;
|
||||
if ((m = /Igneum Miner (\S+) on /.exec(tail))) a.version = m[1];
|
||||
const h = lastMatch(lines, HEADER); if (h) { a.version = h[1]; a.node_version = h[4].replace(/^igneumd[_\/]/, ''); }
|
||||
const stampOf = l => { const t = /^(\d+)(?:\.\d+)? /.exec(l); return t ? new Date(Number(t[1]) * 1000).toISOString() : null; };
|
||||
const strip = l => l.replace(/^\d+(\.\d+)? /, '');
|
||||
// OTA: the newest line decides the state shown on the card
|
||||
const ota = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(OTA: .+)$/);
|
||||
if (ota) { const t = ota[1]; a.ota = { text: t.slice(0, 220), at: stampOf(ota[0]), state: /waiting for administrator approval|approval not given/.test(t) ? 'waiting for approval' : /deferred/.test(t) ? 'deferred' : /updated to|installed|applied/.test(t) ? 'updated' : /download/.test(t) ? 'downloading' : 'note' }; }
|
||||
const pc = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(?:GPU )?(power cap.*)$/i);
|
||||
if (pc) a.power_cap = { text: pc[1].slice(0, 220), at: stampOf(pc[0]), ok: !/NOT applied|nothing to read back/.test(pc[1]) };
|
||||
const jf = lastMatch(lines, /^\d+(?:\.\d+)? jobs: (.+)$/);
|
||||
if (jf) a.jobs_file = { text: jf[1].slice(0, 220), at: stampOf(jf[0]) };
|
||||
a.jobs = lines.filter(l => /^\d+(?:\.\d+)? (?:\[\w+\] )?job [^:]+[:(]/.test(l) || /^\d+(?:\.\d+)? (?:\[\w+\] )?\d+ remote jobs? received/.test(l)).slice(-6).map(l => ({ at: stampOf(l), text: strip(l).replace(/^\[\w+\] /, '').slice(0, 220) }));
|
||||
for (const l of lines) {
|
||||
const s = /stability: (.+?): draw p95 (\d+) W, max (\d+) W \(cap (\d+) W\), max GPU (\d+) C, max memory (\d+) C, (\d+) samples/.exec(l);
|
||||
if (s) { const i = a.telemetry.findIndex(t => t.name === s[1]); const t = { name: s[1], p95_w: Number(s[2]), max_w: Number(s[3]), cap_w: Number(s[4]), max_gpu_c: Number(s[5]), max_mem_c: Number(s[6]), samples: Number(s[7]) }; if (i >= 0) a.telemetry[i] = t; else a.telemetry.push(t); }
|
||||
const e = /^(\d+(?:\.\d+)?) \[(error|ok|block|warn)\] (.+)$/.exec(l);
|
||||
if (e && e[2] !== 'block') a.events.push({ at: new Date(Number(e[1]) * 1000).toISOString(), kind: e[2], text: e[3].slice(0, 200) });
|
||||
}
|
||||
a.events = a.events.slice(-6);
|
||||
if ((m = lastMatch(lines, /^(\d+)(?:\.\d+)? status: (.+?) \| node (.+?) \| up (.+)$/))) {
|
||||
a.status = { miner: m[2], node: m[3], up: m[4], at: new Date(Number(m[1]) * 1000).toISOString() };
|
||||
const acc = /accepted (\d+) blocks \((\d+) this run\), ([\d.]+) MH\/s, (\w+)/.exec(m[2]);
|
||||
if (acc) Object.assign(a.status, { accepted_total: Number(acc[1]), accepted_run: Number(acc[2]), mhs: Number(acc[3]), mining: acc[4] });
|
||||
const nd = /(\d+) blocks, (\d+) peers, (\w+)/.exec(m[3]);
|
||||
if (nd) Object.assign(a.status, { node_blocks: Number(nd[1]), node_peers: Number(nd[2]), node_state: nd[3] });
|
||||
}
|
||||
return a;
|
||||
}
|
||||
async function machines(sql) {
|
||||
// node logs get a longer tail: the connection-manager (peers) line is rare on a busy node
|
||||
const rows = await sql(`SELECT DISTINCT ON (label) label, machine, run_id, received_at, left(lines, 300) AS head,
|
||||
|
|
@ -133,6 +96,7 @@ async function machines(sql) {
|
|||
name: id8 ? (NAMES[id8] || (g.platform === 'mac' ? 'Mac' : 'PC ' + id8)) : g.id.slice(7),
|
||||
host: [...g.hosts][0] || null, legacy: !id8,
|
||||
last_seen: g.last_seen, silent_s, silent: silent_s === null || silent_s > SILENT_S,
|
||||
stopped: (silent_s === null || silent_s > SILENT_S) && g.app && g.app.stopped ? g.app.stopped : null,
|
||||
app_version: (g.app && g.app.version) || (g.header && g.header.version) || null,
|
||||
app: g.app ? { at: g.app.at, status: g.app.status || null, ota: g.app.ota || null, power_cap: g.app.power_cap || null, jobs_file: g.app.jobs_file || null, jobs: g.app.jobs || [] } : null,
|
||||
node: { version: g.node.version || (g.header && g.header.node) || (g.app && g.app.node_version) || null, daa: g.node.daa ?? null, blue_score: g.node.blue_score ?? null, checkpoint: g.node.checkpoint ?? null,
|
||||
|
|
|
|||
|
|
@ -63,3 +63,51 @@ export function parseMinerTail(tail) {
|
|||
if (acc) c.last_accepted_at = new Date(Number(acc[1]) * 1000).toISOString();
|
||||
return c;
|
||||
}
|
||||
|
||||
// The app log: the IGNEUM-APP header, the status line, OTA and power-cap lines, telemetry, events, jobs, and a clean stop.
|
||||
export const HEADER = /IGNEUM-APP version=(\S+) machine=(\S+) platform=(\S+) node=(\S+)/;
|
||||
export function parseHeader(text) {
|
||||
const m = HEADER.exec(text || '');
|
||||
return m ? { version: m[1], id8: m[2], platform: m[3], node: m[4].replace(/^igneumd[_\/]/, '') } : null;
|
||||
}
|
||||
export function parseAppTail(tail) {
|
||||
const lines = tail.split('\n');
|
||||
const a = { telemetry: [], events: [], jobs: [] };
|
||||
let m;
|
||||
if ((m = /Igneum Miner (\S+) on /.exec(tail))) a.version = m[1];
|
||||
const h = lastMatch(lines, HEADER); if (h) { a.version = h[1]; a.node_version = h[4].replace(/^igneumd[_\/]/, ''); }
|
||||
const stampOf = l => { const t = /^(\d+)(?:\.\d+)? /.exec(l); return t ? new Date(Number(t[1]) * 1000).toISOString() : null; };
|
||||
const strip = l => l.replace(/^\d+(\.\d+)? /, '');
|
||||
// OTA: the newest line decides the state shown on the card
|
||||
const ota = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(OTA: .+)$/);
|
||||
if (ota) { const t = ota[1]; a.ota = { text: t.slice(0, 220), at: stampOf(ota[0]), state: /waiting for administrator approval|approval not given/.test(t) ? 'waiting for approval' : /deferred/.test(t) ? 'deferred' : /updated to|installed|applied/.test(t) ? 'updated' : /download/.test(t) ? 'downloading' : 'note' }; }
|
||||
const pc = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(?:GPU )?(power cap.*)$/i);
|
||||
if (pc) a.power_cap = { text: pc[1].slice(0, 220), at: stampOf(pc[0]), ok: !/NOT applied|nothing to read back/.test(pc[1]) };
|
||||
const jf = lastMatch(lines, /^\d+(?:\.\d+)? jobs: (.+)$/);
|
||||
if (jf) a.jobs_file = { text: jf[1].slice(0, 220), at: stampOf(jf[0]) };
|
||||
a.jobs = lines.filter(l => /^\d+(?:\.\d+)? (?:\[\w+\] )?job [^:]+[:(]/.test(l) || /^\d+(?:\.\d+)? (?:\[\w+\] )?\d+ remote jobs? received/.test(l)).slice(-6).map(l => ({ at: stampOf(l), text: strip(l).replace(/^\[\w+\] /, '').slice(0, 220) }));
|
||||
for (const l of lines) {
|
||||
const s = /stability: (.+?): draw p95 (\d+) W, max (\d+) W \(cap (\d+) W\), max GPU (\d+) C, max memory (\d+) C, (\d+) samples/.exec(l);
|
||||
if (s) { const i = a.telemetry.findIndex(t => t.name === s[1]); const t = { name: s[1], p95_w: Number(s[2]), max_w: Number(s[3]), cap_w: Number(s[4]), max_gpu_c: Number(s[5]), max_mem_c: Number(s[6]), samples: Number(s[7]) }; if (i >= 0) a.telemetry[i] = t; else a.telemetry.push(t); }
|
||||
const e = /^(\d+(?:\.\d+)?) \[(error|ok|block|warn)\] (.+)$/.exec(l);
|
||||
if (e && e[2] !== 'block') a.events.push({ at: new Date(Number(e[1]) * 1000).toISOString(), kind: e[2], text: e[3].slice(0, 200) });
|
||||
}
|
||||
a.events = a.events.slice(-6);
|
||||
// A clean stop: the app logs "quit: ..." (Quit, Stop, or an update: "[info] installing Igneum Miner ...") and uploads
|
||||
// once more before it exits; when no status line follows, the machine was stopped on purpose, not lost (4 October
|
||||
// 2026: a Mac stopped at 14:47 UTC read "silent 4h" for the whole afternoon, the same as a crash or a lost network).
|
||||
const stop = lastMatch(lines, /^(\d+)(?:\.\d+)? (quit: .+|stopped|\[info\] installing Igneum Miner .+)$/);
|
||||
const lastStatus = lastMatch(lines, /^(\d+)(?:\.\d+)? status: /);
|
||||
if (stop && (!lastStatus || Number(stop[1]) >= Number(lastStatus[1]))) {
|
||||
const update = lines.some(l => /\[info\] installing Igneum Miner/.test(l) && Number((/^(\d+)/.exec(l) || [])[1]) >= Number(stop[1]) - 30);
|
||||
a.stopped = { at: stampOf(stop[0]), reason: update ? 'update' : 'quit', text: strip(stop[0]).slice(0, 120) };
|
||||
}
|
||||
if ((m = lastMatch(lines, /^(\d+)(?:\.\d+)? status: (.+?) \| node (.+?) \| up (.+)$/))) {
|
||||
a.status = { miner: m[2], node: m[3], up: m[4], at: new Date(Number(m[1]) * 1000).toISOString() };
|
||||
const acc = /accepted (\d+) blocks \((\d+) this run\), ([\d.]+) MH\/s, (\w+)/.exec(m[2]);
|
||||
if (acc) Object.assign(a.status, { accepted_total: Number(acc[1]), accepted_run: Number(acc[2]), mhs: Number(acc[3]), mining: acc[4] });
|
||||
const nd = /(\d+) blocks, (\d+) peers, (\w+)/.exec(m[3]);
|
||||
if (nd) Object.assign(a.status, { node_blocks: Number(nd[1]), node_peers: Number(nd[2]), node_state: nd[3] });
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,3 +41,25 @@ test('stale: a card is "now" within STALE_S of its STATUS line and stale after i
|
|||
const oneMissed = { status_at: new Date(now - 150_000).toISOString(), hash_now: 18 };
|
||||
markStale([oneMissed], now); assert.equal(oneMissed.stale, false);
|
||||
});
|
||||
|
||||
test('app tail: a clean quit after the last status line is "stopped (quit)", an update is "stopped (update)", a running app is not stopped', async () => {
|
||||
const { parseAppTail } = await import('../lib/parse.mjs');
|
||||
const running = ['1791126700 status: accepted 10 blocks (5 this run), 18.00 MH/s, mining | node 100 blocks, 4 peers, synced | up 00:10:00'].join('\n');
|
||||
assert.equal(parseAppTail(running).stopped, undefined);
|
||||
const quit = [
|
||||
'1791126700 status: accepted 10 blocks (5 this run), 18.00 MH/s, mining | node 100 blocks, 4 peers, synced | up 00:10:00',
|
||||
'1791126735 quit: stopping the miners, then the node', '1791126735 stopping the miners (quit)', '1791126736 node stopped (exit Some(0))',
|
||||
'1791126736 SUMMARY after 00:26:53: node started 1 time(s), restarts 0, 1165 accepted blocks this run (1270 lifetime); data stays in x', '1791126737 stopped',
|
||||
].join('\n');
|
||||
const q = parseAppTail(quit);
|
||||
assert.deepEqual([q.stopped.reason, q.stopped.at], ['quit', new Date(1791126737000).toISOString()]);
|
||||
const update = [
|
||||
'1791125100 status: accepted 1 blocks (1 this run), 1.00 MH/s, mining | node 1 blocks, 1 peers, synced | up 00:01:00',
|
||||
'1791125117 [info] installing Igneum Miner 0.3.2: the miners stop, then the node, then the app opens again',
|
||||
'1791125118 quit: stopping the miners, then the node', '1791125121 stopped',
|
||||
].join('\n');
|
||||
assert.equal(parseAppTail(update).stopped.reason, 'update');
|
||||
// a quit line from an earlier run in the same tail, with a status line after it: still running
|
||||
const resumed = quit + '\n1791126800 status: accepted 11 blocks (1 this run), 18.00 MH/s, mining | node 120 blocks, 4 peers, synced | up 00:01:00';
|
||||
assert.equal(parseAppTail(resumed).stopped, undefined);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ main{padding-block:14px 48px}
|
|||
.card{background:var(--graphite);border:1px solid var(--line);border-radius:var(--r);padding:14px;min-width:0}
|
||||
.card.red{border-color:var(--red);box-shadow:inset 3px 0 0 var(--red)}
|
||||
.card.live{box-shadow:inset 3px 0 0 var(--green)}
|
||||
.card.stopped{box-shadow:inset 3px 0 0 var(--ash)}
|
||||
.card h3{margin:0;font-family:'Unbounded',sans-serif;font-weight:700;font-size:14px;letter-spacing:.02em;display:flex;align-items:center;gap:10px;flex-wrap:wrap}
|
||||
.card h3 .id{font-family:'IBM Plex Mono',monospace;font-weight:400;font-size:11px;color:var(--ash);letter-spacing:.04em}
|
||||
.card h3 .seen{margin-left:auto;font-family:'IBM Plex Mono',monospace;font-weight:400;font-size:11px;color:var(--ash)}
|
||||
|
|
@ -266,10 +267,10 @@ input[type=text]{width:100%}
|
|||
}
|
||||
function machineCard(mc) {
|
||||
const n = mc.node || {};
|
||||
const silentTxt = mc.never ? 'never seen' : (mc.silent ? `silent ${ago(mc.last_seen)}` : `seen ${ago(mc.last_seen)} ago`);
|
||||
const silentTxt = mc.never ? 'never seen' : mc.stopped ? `stopped (${esc(mc.stopped.reason)}) ${ago(mc.stopped.at)} ago` : (mc.silent ? `silent ${ago(mc.last_seen)}` : `seen ${ago(mc.last_seen)} ago`);
|
||||
const tele = mc.telemetry && mc.telemetry.length ? `<div class="tw"><table class="t"><thead><tr><th>telemetry</th><th class="num">p95</th><th class="num">max</th><th class="num">cap</th><th class="num">GPU</th><th class="num">mem</th></tr></thead><tbody>${mc.telemetry.map(t => `<tr><td>${esc(t.name)}</td><td class="num">${t.p95_w} W</td><td class="num">${t.max_w} W</td><td class="num">${t.cap_w} W</td><td class="num ${t.max_gpu_c >= 85 ? 'bad' : ''}">${t.max_gpu_c} C</td><td class="num ${t.max_mem_c >= 95 ? 'bad' : ''}">${t.max_mem_c} C</td></tr>`).join('')}</tbody></table></div>` : (mc.cards.length ? `<div class="note" style="margin-top:8px">${mc.app ? 'no stability lines yet (power and temperatures come from nvidia-smi every 5 min; none on a Mac)' : 'the app log is not in the intake (an app before 0.3.3)'}</div>` : '');
|
||||
const events = mc.events && mc.events.length ? `<div style="margin-top:8px">${mc.events.map(e => `<div class="ev"><span class="t">${when(e.at)}</span><span class="k ${e.kind}">${e.kind}</span><span>${esc(e.text)}</span></div>`).join('')}</div>` : '';
|
||||
return `<article class="card ${mc.silent ? 'red' : 'live'}">
|
||||
return `<article class="card ${mc.stopped ? 'stopped' : mc.silent ? 'red' : 'live'}">
|
||||
<h3>${esc(mc.name)}<span class="id">${esc(mc.id8 || '')}${mc.host ? ' ' + esc(mc.host) : ''}${mc.legacy ? ' (old launcher)' : ''}</span><span class="seen ${mc.silent ? 'bad' : ''}">${silentTxt}</span></h3>
|
||||
${mc.never ? '<div class="note" style="margin-top:8px">nothing from this machine in the log intake in the last 48 h</div>' : kvs([
|
||||
['app', mc.app_version ? esc(mc.app_version) : '<small>not in the intake</small>'],
|
||||
|
|
|
|||
|
|
@ -110,7 +110,7 @@ try {
|
|||
else if (cmd === 'machines') {
|
||||
const j = await api('machines');
|
||||
for (const m of j.machines) {
|
||||
console.log(`${m.name.padEnd(8)} ${m.id8 || '-'} ${m.silent ? 'SILENT' : 'live'} seen ${ago(m.last_seen)} ago | app ${m.app_version || '?'} node ${m.node.version || '?'} daa ${m.node.daa ?? '?'} peers ${m.node.peers ?? '?'} ${m.node.synced ? 'synced' : 'not synced'} | ${m.hash_total.toFixed(1)} MH/s, ${m.accepted_total} accepted, ${m.faults} faults`);
|
||||
console.log(`${m.name.padEnd(8)} ${m.id8 || '-'} ${m.stopped ? `STOPPED (${m.stopped.reason}) ${ago(m.stopped.at)} ago,` : m.silent ? 'SILENT' : 'live'} seen ${ago(m.last_seen)} ago | app ${m.app_version || '?'} node ${m.node.version || '?'} daa ${m.node.daa ?? '?'} peers ${m.node.peers ?? '?'} ${m.node.synced ? 'synced' : 'not synced'} | ${m.hash_total.toFixed(1)} MH/s, ${m.accepted_total} accepted, ${m.faults} faults`);
|
||||
for (const c of m.cards) console.log(` ${c.label.padEnd(22)} ${c.stale ? 'last ' + c.hash_now.toFixed(1) + ' MH/s (stale, not in the total)' : c.hash_now.toFixed(1) + ' MH/s now'}, ${c.accepted} accepted, ${c.rejected} rejected, ${c.mismatched} mismatched, ${c.restarts} restarts, status ${ago(c.status_at)} ago${c.fault ? ' | ' + c.fault : ''}`);
|
||||
for (const t of m.telemetry) console.log(` ${t.name}: p95 ${t.p95_w} W, max ${t.max_w} W (cap ${t.cap_w} W), GPU ${t.max_gpu_c} C, memory ${t.max_mem_c} C`);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue