diff --git a/relay/api/console.mjs b/relay/api/console.mjs
index dd11ef91b..919093f40 100644
--- a/relay/api/console.mjs
+++ b/relay/api/console.mjs
@@ -97,11 +97,27 @@ function parseMinerTail(tail) {
if (acc) c.last_accepted_at = new Date(Number(acc[1]) * 1000).toISOString();
return c;
}
+const HEADER = /IGNEUM-APP version=(\S+) machine=(\S+) platform=(\S+) node=(\S+)/;
+function parseHeader(text) {
+ const m = HEADER.exec(text || '');
+ return m ? { version: m[1], id8: m[2], platform: m[3], node: m[4].replace(/^igneumd[_\/]/, '') } : null;
+}
function parseAppTail(tail) {
const lines = tail.split('\n');
- const a = { telemetry: [], events: [] };
+ const a = { telemetry: [], events: [], jobs: [] };
let m;
if ((m = /Igneum Miner (\S+) on /.exec(tail))) a.version = m[1];
+ const h = lastMatch(lines, HEADER); if (h) { a.version = h[1]; a.node_version = h[4].replace(/^igneumd[_\/]/, ''); }
+ const stampOf = l => { const t = /^(\d+)(?:\.\d+)? /.exec(l); return t ? new Date(Number(t[1]) * 1000).toISOString() : null; };
+ const strip = l => l.replace(/^\d+(\.\d+)? /, '');
+ // OTA: the newest line decides the state shown on the card
+ const ota = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(OTA: .+)$/);
+ if (ota) { const t = ota[1]; a.ota = { text: t.slice(0, 220), at: stampOf(ota[0]), state: /waiting for administrator approval|approval not given/.test(t) ? 'waiting for approval' : /deferred/.test(t) ? 'deferred' : /updated to|installed|applied/.test(t) ? 'updated' : /download/.test(t) ? 'downloading' : 'note' }; }
+ const pc = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(?:GPU )?(power cap.*)$/i);
+ if (pc) a.power_cap = { text: pc[1].slice(0, 220), at: stampOf(pc[0]), ok: !/NOT applied|nothing to read back/.test(pc[1]) };
+ const jf = lastMatch(lines, /^\d+(?:\.\d+)? jobs: (.+)$/);
+ if (jf) a.jobs_file = { text: jf[1].slice(0, 220), at: stampOf(jf[0]) };
+ a.jobs = lines.filter(l => /^\d+(?:\.\d+)? (?:\[\w+\] )?job [^:]+[:(]/.test(l) || /^\d+(?:\.\d+)? (?:\[\w+\] )?\d+ remote jobs? received/.test(l)).slice(-6).map(l => ({ at: stampOf(l), text: strip(l).replace(/^\[\w+\] /, '').slice(0, 220) }));
for (const l of lines) {
const s = /stability: (.+?): draw p95 (\d+) W, max (\d+) W \(cap (\d+) W\), max GPU (\d+) C, max memory (\d+) C, (\d+) samples/.exec(l);
if (s) { const i = a.telemetry.findIndex(t => t.name === s[1]); const t = { name: s[1], p95_w: Number(s[2]), max_w: Number(s[3]), cap_w: Number(s[4]), max_gpu_c: Number(s[5]), max_mem_c: Number(s[6]), samples: Number(s[7]) }; if (i >= 0) a.telemetry[i] = t; else a.telemetry.push(t); }
@@ -109,19 +125,26 @@ function parseAppTail(tail) {
if (e && e[2] !== 'block') a.events.push({ at: new Date(Number(e[1]) * 1000).toISOString(), kind: e[2], text: e[3].slice(0, 200) });
}
a.events = a.events.slice(-6);
- if ((m = lastMatch(lines, /status: (.+?) \| node (.+?) \| up (.+)$/))) a.status = { miner: m[1], node: m[2], up: m[3] };
+ if ((m = lastMatch(lines, /^(\d+)(?:\.\d+)? status: (.+?) \| node (.+?) \| up (.+)$/))) {
+ a.status = { miner: m[2], node: m[3], up: m[4], at: new Date(Number(m[1]) * 1000).toISOString() };
+ const acc = /accepted (\d+) blocks \((\d+) this run\), ([\d.]+) MH\/s, (\w+)/.exec(m[2]);
+ if (acc) Object.assign(a.status, { accepted_total: Number(acc[1]), accepted_run: Number(acc[2]), mhs: Number(acc[3]), mining: acc[4] });
+ const nd = /(\d+) blocks, (\d+) peers, (\w+)/.exec(m[3]);
+ if (nd) Object.assign(a.status, { node_blocks: Number(nd[1]), node_peers: Number(nd[2]), node_state: nd[3] });
+ }
return a;
}
async function machines(sql) {
// node logs get a longer tail: the connection-manager (peers) line is rare on a busy node
- const rows = await sql(`SELECT DISTINCT ON (label) label, machine, run_id, received_at,
- right(lines, CASE WHEN label LIKE 'nodelog-%' THEN 80000 ELSE 20000 END) AS tail
+ const rows = await sql(`SELECT DISTINCT ON (label) label, machine, run_id, received_at, left(lines, 300) AS head,
+ right(lines, CASE WHEN label LIKE 'nodelog-%' THEN 80000 WHEN label ~ '^(win|mac)-[0-9a-f]{8}$' THEN 60000 ELSE 20000 END) AS tail
FROM miner_logs WHERE received_at > now() - interval '36 hours' ORDER BY label, received_at DESC`);
const groups = new Map();
for (const r of rows) {
const p = parseLabel(r.label, r.machine);
if (!p) continue;
- const g = groups.get(p.id) || { id: p.id, platform: p.platform, hosts: new Set(), runs: new Set(), last_seen: null, node: {}, cards: [], app: null, labels: [] };
+ const g = groups.get(p.id) || { id: p.id, platform: p.platform, hosts: new Set(), runs: new Set(), last_seen: null, node: {}, cards: [], app: null, labels: [], header: null };
+ const hd = parseHeader(r.head) || parseHeader(r.tail); if (hd) g.header = hd;
g.hosts.add(r.machine.replace(/-[0-9a-f]{8}$/, ''));
g.runs.add(r.run_id);
const at = iso(r.received_at);
@@ -133,10 +156,14 @@ async function machines(sql) {
groups.set(p.id, g);
}
// node version lives in the first upload of a run (the tail may have lost the banner): one cheap query for the current runs
- const runIds = [...groups.values()].map(g => g.node.run_id).filter(Boolean);
+ const runIds = [...new Set([...groups.values()].flatMap(g => [...g.runs]))];
if (runIds.length) {
- const heads = await sql(`SELECT DISTINCT ON (run_id) run_id, left(lines, 3000) AS head FROM miner_logs WHERE run_id = ANY($1) AND label LIKE 'nodelog-%' ORDER BY run_id, received_at ASC`, [runIds]);
- for (const h of heads) { const m = /igneumd\/(\S+)/.exec(h.head); if (!m) continue; for (const g of groups.values()) if (g.node.run_id === h.run_id && !g.node.version) g.node.version = m[1]; }
+ const heads = await sql(`SELECT DISTINCT ON (run_id, label) run_id, label, left(lines, 3000) AS head FROM miner_logs WHERE run_id = ANY($1) ORDER BY run_id, label, received_at ASC`, [runIds]);
+ for (const h of heads) {
+ const g = [...groups.values()].find(x => x.runs.has(h.run_id)); if (!g) continue;
+ const m = /igneumd\/(\S+)/.exec(h.head); if (m && h.label.startsWith('nodelog-') && !g.node.version) g.node.version = m[1];
+ const hd = parseHeader(h.head); if (hd && !g.header) g.header = hd;
+ }
}
const now = Date.now();
const out = [...groups.values()].map(g => {
@@ -148,12 +175,17 @@ async function machines(sql) {
name: id8 ? (NAMES[id8] || (g.platform === 'mac' ? 'Mac' : 'PC ' + id8)) : g.id.slice(7),
host: [...g.hosts][0] || null, legacy: !id8,
last_seen: g.last_seen, silent_s, silent: silent_s === null || silent_s > SILENT_S,
- app_version: g.app ? g.app.version || null : null,
- node: { version: g.node.version || null, daa: g.node.daa ?? null, blue_score: g.node.blue_score ?? null, checkpoint: g.node.checkpoint ?? null, peers: g.node.peers ?? null, peers_wanted: g.node.peers_wanted ?? null, blocks_10s: g.node.blocks_10s ?? null, synced: cards.some(c => c.synced === true) || (g.node.blocks_10s != null && !g.node.ibd_seen), ibd: !!g.node.ibd_seen, last_line_at: g.node.last_line_at || null, warnings: g.node.warnings || 0, last_warning: g.node.last_warning || null, at: g.node.at || null },
+ app_version: (g.app && g.app.version) || (g.header && g.header.version) || null,
+ app: g.app ? { at: g.app.at, status: g.app.status || null, ota: g.app.ota || null, power_cap: g.app.power_cap || null, jobs_file: g.app.jobs_file || null, jobs: g.app.jobs || [] } : null,
+ node: { version: g.node.version || (g.header && g.header.node) || (g.app && g.app.node_version) || null, daa: g.node.daa ?? null, blue_score: g.node.blue_score ?? null, checkpoint: g.node.checkpoint ?? null,
+ peers: g.node.peers ?? (g.app && g.app.status && g.app.status.node_peers != null ? g.app.status.node_peers : null), peers_wanted: g.node.peers_wanted ?? null, peers_from: g.node.peers != null ? 'node log' : (g.app && g.app.status ? 'app status line' : null),
+ blocks: g.app && g.app.status && g.app.status.node_blocks != null ? g.app.status.node_blocks : null,
+ blocks_10s: g.node.blocks_10s ?? null, synced: cards.some(c => c.synced === true) || (g.node.blocks_10s != null && !g.node.ibd_seen) || !!(g.app && g.app.status && g.app.status.node_state === 'synced'), ibd: !!g.node.ibd_seen, last_line_at: g.node.last_line_at || null, warnings: g.node.warnings || 0, last_warning: g.node.last_warning || null, at: g.node.at || (g.app && g.app.status ? g.app.status.at : null) },
cards, hash_total: Math.round(cards.reduce((a, c) => a + (c.hash_now || 0), 0) * 100) / 100,
- accepted_total: cards.reduce((a, c) => a + (c.accepted || 0), 0),
+ accepted_total: cards.reduce((a, c) => a + (c.accepted || 0), 0) || (g.app && g.app.status && g.app.status.accepted_run) || 0,
+ accepted_lifetime: g.app && g.app.status ? g.app.status.accepted_total ?? null : null,
faults: cards.reduce((a, c) => a + (c.faults || 0), 0),
- telemetry: g.app ? g.app.telemetry : [], events: g.app ? g.app.events : [], status: g.app ? g.app.status || null : null,
+ telemetry: g.app ? g.app.telemetry : [], events: g.app ? g.app.events : [],
labels: g.labels.sort((a, b) => (b.at > a.at ? 1 : -1)), runs: [...g.runs],
};
});
@@ -163,7 +195,7 @@ async function machines(sql) {
// several Mac installs (test runs): the newest is "Mac", the others keep their id
let macSeen = false;
for (const m of out) if (m.platform === 'mac' && m.name === 'Mac') { if (macSeen) m.name = `Mac ${m.id8}`; macSeen = true; }
- for (const n of ['Mac', 'PC 1', 'PC 2', "Sam's Mac"]) if (!out.some(m => m.name === n)) out.push({ id: n, id8: Object.keys(NAMES).find(k => NAMES[k] === n) || null, platform: n.endsWith('Mac') ? 'mac' : 'win', name: n, host: null, legacy: false, last_seen: null, silent_s: null, silent: true, app_version: null, node: {}, cards: [], hash_total: 0, accepted_total: 0, faults: 0, telemetry: [], events: [], status: null, labels: [], runs: [], never: true });
+ for (const n of ['Mac', 'PC 1', 'PC 2', "Sam's Mac"]) if (!out.some(m => m.name === n)) out.push({ id: n, id8: Object.keys(NAMES).find(k => NAMES[k] === n) || null, platform: n.endsWith('Mac') ? 'mac' : 'win', name: n, host: null, legacy: false, last_seen: null, silent_s: null, silent: true, app_version: null, app: null, node: {}, cards: [], hash_total: 0, accepted_total: 0, faults: 0, telemetry: [], events: [], labels: [], runs: [], never: true });
out.sort((a, b) => order(a) - order(b) || (b.last_seen || '').localeCompare(a.last_seen || ''));
return { machines: out.filter(m => NAMED.has(m.name) || (m.silent_s !== null && m.silent_s < HIDE_AFTER_S)), silent_after_s: SILENT_S, hide_after_s: HIDE_AFTER_S };
}
diff --git a/relay/ui.html b/relay/ui.html
index 1154148bf..af7ec4d02 100644
--- a/relay/ui.html
+++ b/relay/ui.html
@@ -267,25 +267,33 @@ input[type=text]{width:100%}
function machineCard(mc) {
const n = mc.node || {};
const silentTxt = mc.never ? 'never seen' : (mc.silent ? `silent ${ago(mc.last_seen)}` : `seen ${ago(mc.last_seen)} ago`);
- const tele = mc.telemetry && mc.telemetry.length ? `
| telemetry | p95 | max | cap | GPU | mem |
${mc.telemetry.map(t => `| ${esc(t.name)} | ${t.p95_w} W | ${t.max_w} W | ${t.cap_w} W | ${t.max_gpu_c} C | ${t.max_mem_c} C |
`).join('')}
` : (mc.cards.length ? `no power or temperature lines in the intake yet (the app log is not uploading)
` : '');
+ const tele = mc.telemetry && mc.telemetry.length ? `| telemetry | p95 | max | cap | GPU | mem |
${mc.telemetry.map(t => `| ${esc(t.name)} | ${t.p95_w} W | ${t.max_w} W | ${t.cap_w} W | ${t.max_gpu_c} C | ${t.max_mem_c} C |
`).join('')}
` : (mc.cards.length ? `${mc.app ? 'no stability lines yet (power and temperatures come from nvidia-smi every 5 min; none on a Mac)' : 'the app log is not in the intake (an app before 0.3.3)'}
` : '');
const events = mc.events && mc.events.length ? `${mc.events.map(e => `
${when(e.at)}${e.kind}${esc(e.text)}
`).join('')}
` : '';
return `
${esc(mc.name)}${esc(mc.id8 || '')}${mc.host ? ' ' + esc(mc.host) : ''}${mc.legacy ? ' (old launcher)' : ''}${silentTxt}
${mc.never ? 'nothing from this machine in the log intake in the last 48 h
' : kvs([
['app', mc.app_version ? esc(mc.app_version) : 'not in the intake'],
+ ['ota', mc.app && mc.app.ota ? esc(mc.app.ota.state) : (mc.app_version ? 'no update pending' : '?'), mc.app && mc.app.ota ? (mc.app.ota.state === 'updated' ? 'good' : 'warm') : ''],
['node', n.version ? esc(n.version) : '?'],
['height (daa)', n.daa != null ? nf(n.daa) : '?', ''],
['synced', n.at ? (n.synced ? 'yes' : (n.ibd ? 'IBD' : 'no')) : '?', n.at ? (n.synced ? 'good' : 'bad') : ''],
['hash rate', mh(mc.hash_total), mc.hash_total ? '' : 'warm'],
- ['accepted', nf(mc.accepted_total)],
- ['peers', n.peers != null ? `${n.peers}/${n.peers_wanted}` : '?'],
+ ['accepted', `${nf(mc.accepted_total)}${mc.accepted_lifetime != null ? ` of ${nf(mc.accepted_lifetime)} lifetime` : ''}`],
+ ['peers', n.peers != null ? `${n.peers}${n.peers_wanted ? `/${n.peers_wanted}` : ''}` : '?'],
['blocks / 10 s', n.blocks_10s != null ? nf(n.blocks_10s) : '?'],
['faults', nf(mc.faults), mc.faults ? 'bad' : 'good'],
['node warnings', nf(n.warnings || 0), n.warnings ? 'warm' : ''],
['checkpoint', n.checkpoint != null ? nf(n.checkpoint) : '?'],
['run', `${esc((n.run_id || (mc.runs && mc.runs[0]) || '').replace(/^.*-(\d{8}-\d{6})$/, '$1'))}`],
])}
- ${cardTable(mc.cards)}${tele}${events}
+ ${cardTable(mc.cards)}${tele}
+ ${mc.app && (mc.app.ota || mc.app.power_cap || mc.app.jobs_file || (mc.app.jobs && mc.app.jobs.length) || mc.app.status) ? `
+ ${mc.app.status ? `
${when(mc.app.status.at)}status${esc(mc.app.status.miner)} | node ${esc(mc.app.status.node)} | up ${esc(mc.app.status.up)}
` : ''}
+ ${mc.app.ota ? `
${when(mc.app.ota.at)}ota${esc(mc.app.ota.text)}
` : ''}
+ ${mc.app.power_cap ? `
${when(mc.app.power_cap.at)}power${esc(mc.app.power_cap.text)}
` : ''}
+ ${mc.app.jobs_file ? `
${when(mc.app.jobs_file.at)}jobs${esc(mc.app.jobs_file.text)}
` : ''}
+ ${(mc.app.jobs || []).map(j => `
${when(j.at)}job${esc(j.text)}
`).join('')}
+
` : ''}${events}
${n.last_warning ? `${esc(n.last_warning)}
` : ''}
`;
}