Console Machines tab: app version, OTA state, power cap, jobs and status lines from the app log (0.3.3 header)
The IGNEUM-APP header is read from the newest upload's first line, from any tail, or from the first upload of the run; the app's status: line supplies peers, lifetime accepted and synced when the node log tail has none. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
parent
36faa7f73a
commit
89dd615fef
2 changed files with 57 additions and 17 deletions
|
|
@ -97,11 +97,27 @@ function parseMinerTail(tail) {
|
|||
if (acc) c.last_accepted_at = new Date(Number(acc[1]) * 1000).toISOString();
|
||||
return c;
|
||||
}
|
||||
const HEADER = /IGNEUM-APP version=(\S+) machine=(\S+) platform=(\S+) node=(\S+)/;
|
||||
function parseHeader(text) {
|
||||
const m = HEADER.exec(text || '');
|
||||
return m ? { version: m[1], id8: m[2], platform: m[3], node: m[4].replace(/^igneumd[_\/]/, '') } : null;
|
||||
}
|
||||
function parseAppTail(tail) {
|
||||
const lines = tail.split('\n');
|
||||
const a = { telemetry: [], events: [] };
|
||||
const a = { telemetry: [], events: [], jobs: [] };
|
||||
let m;
|
||||
if ((m = /Igneum Miner (\S+) on /.exec(tail))) a.version = m[1];
|
||||
const h = lastMatch(lines, HEADER); if (h) { a.version = h[1]; a.node_version = h[4].replace(/^igneumd[_\/]/, ''); }
|
||||
const stampOf = l => { const t = /^(\d+)(?:\.\d+)? /.exec(l); return t ? new Date(Number(t[1]) * 1000).toISOString() : null; };
|
||||
const strip = l => l.replace(/^\d+(\.\d+)? /, '');
|
||||
// OTA: the newest line decides the state shown on the card
|
||||
const ota = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(OTA: .+)$/);
|
||||
if (ota) { const t = ota[1]; a.ota = { text: t.slice(0, 220), at: stampOf(ota[0]), state: /waiting for administrator approval|approval not given/.test(t) ? 'waiting for approval' : /deferred/.test(t) ? 'deferred' : /updated to|installed|applied/.test(t) ? 'updated' : /download/.test(t) ? 'downloading' : 'note' }; }
|
||||
const pc = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(?:GPU )?(power cap.*)$/i);
|
||||
if (pc) a.power_cap = { text: pc[1].slice(0, 220), at: stampOf(pc[0]), ok: !/NOT applied|nothing to read back/.test(pc[1]) };
|
||||
const jf = lastMatch(lines, /^\d+(?:\.\d+)? jobs: (.+)$/);
|
||||
if (jf) a.jobs_file = { text: jf[1].slice(0, 220), at: stampOf(jf[0]) };
|
||||
a.jobs = lines.filter(l => /^\d+(?:\.\d+)? (?:\[\w+\] )?job [^:]+[:(]/.test(l) || /^\d+(?:\.\d+)? (?:\[\w+\] )?\d+ remote jobs? received/.test(l)).slice(-6).map(l => ({ at: stampOf(l), text: strip(l).replace(/^\[\w+\] /, '').slice(0, 220) }));
|
||||
for (const l of lines) {
|
||||
const s = /stability: (.+?): draw p95 (\d+) W, max (\d+) W \(cap (\d+) W\), max GPU (\d+) C, max memory (\d+) C, (\d+) samples/.exec(l);
|
||||
if (s) { const i = a.telemetry.findIndex(t => t.name === s[1]); const t = { name: s[1], p95_w: Number(s[2]), max_w: Number(s[3]), cap_w: Number(s[4]), max_gpu_c: Number(s[5]), max_mem_c: Number(s[6]), samples: Number(s[7]) }; if (i >= 0) a.telemetry[i] = t; else a.telemetry.push(t); }
|
||||
|
|
@ -109,19 +125,26 @@ function parseAppTail(tail) {
|
|||
if (e && e[2] !== 'block') a.events.push({ at: new Date(Number(e[1]) * 1000).toISOString(), kind: e[2], text: e[3].slice(0, 200) });
|
||||
}
|
||||
a.events = a.events.slice(-6);
|
||||
if ((m = lastMatch(lines, /status: (.+?) \| node (.+?) \| up (.+)$/))) a.status = { miner: m[1], node: m[2], up: m[3] };
|
||||
if ((m = lastMatch(lines, /^(\d+)(?:\.\d+)? status: (.+?) \| node (.+?) \| up (.+)$/))) {
|
||||
a.status = { miner: m[2], node: m[3], up: m[4], at: new Date(Number(m[1]) * 1000).toISOString() };
|
||||
const acc = /accepted (\d+) blocks \((\d+) this run\), ([\d.]+) MH\/s, (\w+)/.exec(m[2]);
|
||||
if (acc) Object.assign(a.status, { accepted_total: Number(acc[1]), accepted_run: Number(acc[2]), mhs: Number(acc[3]), mining: acc[4] });
|
||||
const nd = /(\d+) blocks, (\d+) peers, (\w+)/.exec(m[3]);
|
||||
if (nd) Object.assign(a.status, { node_blocks: Number(nd[1]), node_peers: Number(nd[2]), node_state: nd[3] });
|
||||
}
|
||||
return a;
|
||||
}
|
||||
async function machines(sql) {
|
||||
// node logs get a longer tail: the connection-manager (peers) line is rare on a busy node
|
||||
const rows = await sql(`SELECT DISTINCT ON (label) label, machine, run_id, received_at,
|
||||
right(lines, CASE WHEN label LIKE 'nodelog-%' THEN 80000 ELSE 20000 END) AS tail
|
||||
const rows = await sql(`SELECT DISTINCT ON (label) label, machine, run_id, received_at, left(lines, 300) AS head,
|
||||
right(lines, CASE WHEN label LIKE 'nodelog-%' THEN 80000 WHEN label ~ '^(win|mac)-[0-9a-f]{8}$' THEN 60000 ELSE 20000 END) AS tail
|
||||
FROM miner_logs WHERE received_at > now() - interval '36 hours' ORDER BY label, received_at DESC`);
|
||||
const groups = new Map();
|
||||
for (const r of rows) {
|
||||
const p = parseLabel(r.label, r.machine);
|
||||
if (!p) continue;
|
||||
const g = groups.get(p.id) || { id: p.id, platform: p.platform, hosts: new Set(), runs: new Set(), last_seen: null, node: {}, cards: [], app: null, labels: [] };
|
||||
const g = groups.get(p.id) || { id: p.id, platform: p.platform, hosts: new Set(), runs: new Set(), last_seen: null, node: {}, cards: [], app: null, labels: [], header: null };
|
||||
const hd = parseHeader(r.head) || parseHeader(r.tail); if (hd) g.header = hd;
|
||||
g.hosts.add(r.machine.replace(/-[0-9a-f]{8}$/, ''));
|
||||
g.runs.add(r.run_id);
|
||||
const at = iso(r.received_at);
|
||||
|
|
@ -133,10 +156,14 @@ async function machines(sql) {
|
|||
groups.set(p.id, g);
|
||||
}
|
||||
// node version lives in the first upload of a run (the tail may have lost the banner): one cheap query for the current runs
|
||||
const runIds = [...groups.values()].map(g => g.node.run_id).filter(Boolean);
|
||||
const runIds = [...new Set([...groups.values()].flatMap(g => [...g.runs]))];
|
||||
if (runIds.length) {
|
||||
const heads = await sql(`SELECT DISTINCT ON (run_id) run_id, left(lines, 3000) AS head FROM miner_logs WHERE run_id = ANY($1) AND label LIKE 'nodelog-%' ORDER BY run_id, received_at ASC`, [runIds]);
|
||||
for (const h of heads) { const m = /igneumd\/(\S+)/.exec(h.head); if (!m) continue; for (const g of groups.values()) if (g.node.run_id === h.run_id && !g.node.version) g.node.version = m[1]; }
|
||||
const heads = await sql(`SELECT DISTINCT ON (run_id, label) run_id, label, left(lines, 3000) AS head FROM miner_logs WHERE run_id = ANY($1) ORDER BY run_id, label, received_at ASC`, [runIds]);
|
||||
for (const h of heads) {
|
||||
const g = [...groups.values()].find(x => x.runs.has(h.run_id)); if (!g) continue;
|
||||
const m = /igneumd\/(\S+)/.exec(h.head); if (m && h.label.startsWith('nodelog-') && !g.node.version) g.node.version = m[1];
|
||||
const hd = parseHeader(h.head); if (hd && !g.header) g.header = hd;
|
||||
}
|
||||
}
|
||||
const now = Date.now();
|
||||
const out = [...groups.values()].map(g => {
|
||||
|
|
@ -148,12 +175,17 @@ async function machines(sql) {
|
|||
name: id8 ? (NAMES[id8] || (g.platform === 'mac' ? 'Mac' : 'PC ' + id8)) : g.id.slice(7),
|
||||
host: [...g.hosts][0] || null, legacy: !id8,
|
||||
last_seen: g.last_seen, silent_s, silent: silent_s === null || silent_s > SILENT_S,
|
||||
app_version: g.app ? g.app.version || null : null,
|
||||
node: { version: g.node.version || null, daa: g.node.daa ?? null, blue_score: g.node.blue_score ?? null, checkpoint: g.node.checkpoint ?? null, peers: g.node.peers ?? null, peers_wanted: g.node.peers_wanted ?? null, blocks_10s: g.node.blocks_10s ?? null, synced: cards.some(c => c.synced === true) || (g.node.blocks_10s != null && !g.node.ibd_seen), ibd: !!g.node.ibd_seen, last_line_at: g.node.last_line_at || null, warnings: g.node.warnings || 0, last_warning: g.node.last_warning || null, at: g.node.at || null },
|
||||
app_version: (g.app && g.app.version) || (g.header && g.header.version) || null,
|
||||
app: g.app ? { at: g.app.at, status: g.app.status || null, ota: g.app.ota || null, power_cap: g.app.power_cap || null, jobs_file: g.app.jobs_file || null, jobs: g.app.jobs || [] } : null,
|
||||
node: { version: g.node.version || (g.header && g.header.node) || (g.app && g.app.node_version) || null, daa: g.node.daa ?? null, blue_score: g.node.blue_score ?? null, checkpoint: g.node.checkpoint ?? null,
|
||||
peers: g.node.peers ?? (g.app && g.app.status && g.app.status.node_peers != null ? g.app.status.node_peers : null), peers_wanted: g.node.peers_wanted ?? null, peers_from: g.node.peers != null ? 'node log' : (g.app && g.app.status ? 'app status line' : null),
|
||||
blocks: g.app && g.app.status && g.app.status.node_blocks != null ? g.app.status.node_blocks : null,
|
||||
blocks_10s: g.node.blocks_10s ?? null, synced: cards.some(c => c.synced === true) || (g.node.blocks_10s != null && !g.node.ibd_seen) || !!(g.app && g.app.status && g.app.status.node_state === 'synced'), ibd: !!g.node.ibd_seen, last_line_at: g.node.last_line_at || null, warnings: g.node.warnings || 0, last_warning: g.node.last_warning || null, at: g.node.at || (g.app && g.app.status ? g.app.status.at : null) },
|
||||
cards, hash_total: Math.round(cards.reduce((a, c) => a + (c.hash_now || 0), 0) * 100) / 100,
|
||||
accepted_total: cards.reduce((a, c) => a + (c.accepted || 0), 0),
|
||||
accepted_total: cards.reduce((a, c) => a + (c.accepted || 0), 0) || (g.app && g.app.status && g.app.status.accepted_run) || 0,
|
||||
accepted_lifetime: g.app && g.app.status ? g.app.status.accepted_total ?? null : null,
|
||||
faults: cards.reduce((a, c) => a + (c.faults || 0), 0),
|
||||
telemetry: g.app ? g.app.telemetry : [], events: g.app ? g.app.events : [], status: g.app ? g.app.status || null : null,
|
||||
telemetry: g.app ? g.app.telemetry : [], events: g.app ? g.app.events : [],
|
||||
labels: g.labels.sort((a, b) => (b.at > a.at ? 1 : -1)), runs: [...g.runs],
|
||||
};
|
||||
});
|
||||
|
|
@ -163,7 +195,7 @@ async function machines(sql) {
|
|||
// several Mac installs (test runs): the newest is "Mac", the others keep their id
|
||||
let macSeen = false;
|
||||
for (const m of out) if (m.platform === 'mac' && m.name === 'Mac') { if (macSeen) m.name = `Mac ${m.id8}`; macSeen = true; }
|
||||
for (const n of ['Mac', 'PC 1', 'PC 2', "Sam's Mac"]) if (!out.some(m => m.name === n)) out.push({ id: n, id8: Object.keys(NAMES).find(k => NAMES[k] === n) || null, platform: n.endsWith('Mac') ? 'mac' : 'win', name: n, host: null, legacy: false, last_seen: null, silent_s: null, silent: true, app_version: null, node: {}, cards: [], hash_total: 0, accepted_total: 0, faults: 0, telemetry: [], events: [], status: null, labels: [], runs: [], never: true });
|
||||
for (const n of ['Mac', 'PC 1', 'PC 2', "Sam's Mac"]) if (!out.some(m => m.name === n)) out.push({ id: n, id8: Object.keys(NAMES).find(k => NAMES[k] === n) || null, platform: n.endsWith('Mac') ? 'mac' : 'win', name: n, host: null, legacy: false, last_seen: null, silent_s: null, silent: true, app_version: null, app: null, node: {}, cards: [], hash_total: 0, accepted_total: 0, faults: 0, telemetry: [], events: [], labels: [], runs: [], never: true });
|
||||
out.sort((a, b) => order(a) - order(b) || (b.last_seen || '').localeCompare(a.last_seen || ''));
|
||||
return { machines: out.filter(m => NAMED.has(m.name) || (m.silent_s !== null && m.silent_s < HIDE_AFTER_S)), silent_after_s: SILENT_S, hide_after_s: HIDE_AFTER_S };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -267,25 +267,33 @@ input[type=text]{width:100%}
|
|||
function machineCard(mc) {
|
||||
const n = mc.node || {};
|
||||
const silentTxt = mc.never ? 'never seen' : (mc.silent ? `silent ${ago(mc.last_seen)}` : `seen ${ago(mc.last_seen)} ago`);
|
||||
const tele = mc.telemetry && mc.telemetry.length ? `<div class="tw"><table class="t"><thead><tr><th>telemetry</th><th class="num">p95</th><th class="num">max</th><th class="num">cap</th><th class="num">GPU</th><th class="num">mem</th></tr></thead><tbody>${mc.telemetry.map(t => `<tr><td>${esc(t.name)}</td><td class="num">${t.p95_w} W</td><td class="num">${t.max_w} W</td><td class="num">${t.cap_w} W</td><td class="num ${t.max_gpu_c >= 85 ? 'bad' : ''}">${t.max_gpu_c} C</td><td class="num ${t.max_mem_c >= 95 ? 'bad' : ''}">${t.max_mem_c} C</td></tr>`).join('')}</tbody></table></div>` : (mc.cards.length ? `<div class="note" style="margin-top:8px">no power or temperature lines in the intake yet (the app log is not uploading)</div>` : '');
|
||||
const tele = mc.telemetry && mc.telemetry.length ? `<div class="tw"><table class="t"><thead><tr><th>telemetry</th><th class="num">p95</th><th class="num">max</th><th class="num">cap</th><th class="num">GPU</th><th class="num">mem</th></tr></thead><tbody>${mc.telemetry.map(t => `<tr><td>${esc(t.name)}</td><td class="num">${t.p95_w} W</td><td class="num">${t.max_w} W</td><td class="num">${t.cap_w} W</td><td class="num ${t.max_gpu_c >= 85 ? 'bad' : ''}">${t.max_gpu_c} C</td><td class="num ${t.max_mem_c >= 95 ? 'bad' : ''}">${t.max_mem_c} C</td></tr>`).join('')}</tbody></table></div>` : (mc.cards.length ? `<div class="note" style="margin-top:8px">${mc.app ? 'no stability lines yet (power and temperatures come from nvidia-smi every 5 min; none on a Mac)' : 'the app log is not in the intake (an app before 0.3.3)'}</div>` : '');
|
||||
const events = mc.events && mc.events.length ? `<div style="margin-top:8px">${mc.events.map(e => `<div class="ev"><span class="t">${when(e.at)}</span><span class="k ${e.kind}">${e.kind}</span><span>${esc(e.text)}</span></div>`).join('')}</div>` : '';
|
||||
return `<article class="card ${mc.silent ? 'red' : 'live'}">
|
||||
<h3>${esc(mc.name)}<span class="id">${esc(mc.id8 || '')}${mc.host ? ' ' + esc(mc.host) : ''}${mc.legacy ? ' (old launcher)' : ''}</span><span class="seen ${mc.silent ? 'bad' : ''}">${silentTxt}</span></h3>
|
||||
${mc.never ? '<div class="note" style="margin-top:8px">nothing from this machine in the log intake in the last 48 h</div>' : kvs([
|
||||
['app', mc.app_version ? esc(mc.app_version) : '<small>not in the intake</small>'],
|
||||
['ota', mc.app && mc.app.ota ? esc(mc.app.ota.state) : (mc.app_version ? 'no update pending' : '?'), mc.app && mc.app.ota ? (mc.app.ota.state === 'updated' ? 'good' : 'warm') : ''],
|
||||
['node', n.version ? esc(n.version) : '?'],
|
||||
['height (daa)', n.daa != null ? nf(n.daa) : '?', ''],
|
||||
['synced', n.at ? (n.synced ? 'yes' : (n.ibd ? 'IBD' : 'no')) : '?', n.at ? (n.synced ? 'good' : 'bad') : ''],
|
||||
['hash rate', mh(mc.hash_total), mc.hash_total ? '' : 'warm'],
|
||||
['accepted', nf(mc.accepted_total)],
|
||||
['peers', n.peers != null ? `${n.peers}<small>/${n.peers_wanted}</small>` : '?'],
|
||||
['accepted', `${nf(mc.accepted_total)}${mc.accepted_lifetime != null ? `<small> of ${nf(mc.accepted_lifetime)} lifetime</small>` : ''}`],
|
||||
['peers', n.peers != null ? `${n.peers}${n.peers_wanted ? `<small>/${n.peers_wanted}</small>` : ''}` : '?'],
|
||||
['blocks / 10 s', n.blocks_10s != null ? nf(n.blocks_10s) : '?'],
|
||||
['faults', nf(mc.faults), mc.faults ? 'bad' : 'good'],
|
||||
['node warnings', nf(n.warnings || 0), n.warnings ? 'warm' : ''],
|
||||
['checkpoint', n.checkpoint != null ? nf(n.checkpoint) : '?'],
|
||||
['run', `<small>${esc((n.run_id || (mc.runs && mc.runs[0]) || '').replace(/^.*-(\d{8}-\d{6})$/, '$1'))}</small>`],
|
||||
])}
|
||||
${cardTable(mc.cards)}${tele}${events}
|
||||
${cardTable(mc.cards)}${tele}
|
||||
${mc.app && (mc.app.ota || mc.app.power_cap || mc.app.jobs_file || (mc.app.jobs && mc.app.jobs.length) || mc.app.status) ? `<div style="margin-top:8px">
|
||||
${mc.app.status ? `<div class="ev"><span class="t">${when(mc.app.status.at)}</span><span class="k">status</span><span>${esc(mc.app.status.miner)} | node ${esc(mc.app.status.node)} | up ${esc(mc.app.status.up)}</span></div>` : ''}
|
||||
${mc.app.ota ? `<div class="ev"><span class="t">${when(mc.app.ota.at)}</span><span class="k ${mc.app.ota.state === 'updated' ? 'ok' : ''}">ota</span><span>${esc(mc.app.ota.text)}</span></div>` : ''}
|
||||
${mc.app.power_cap ? `<div class="ev"><span class="t">${when(mc.app.power_cap.at)}</span><span class="k ${mc.app.power_cap.ok ? 'ok' : 'error'}">power</span><span>${esc(mc.app.power_cap.text)}</span></div>` : ''}
|
||||
${mc.app.jobs_file ? `<div class="ev"><span class="t">${when(mc.app.jobs_file.at)}</span><span class="k">jobs</span><span>${esc(mc.app.jobs_file.text)}</span></div>` : ''}
|
||||
${(mc.app.jobs || []).map(j => `<div class="ev"><span class="t">${when(j.at)}</span><span class="k">job</span><span>${esc(j.text)}</span></div>`).join('')}
|
||||
</div>` : ''}${events}
|
||||
${n.last_warning ? `<div class="note" style="margin-top:8px;color:var(--molten)">${esc(n.last_warning)}</div>` : ''}
|
||||
</article>`;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue