From 89dd615fefbdb6075182dae95ec8d89bf093e3da Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Sun, 4 Oct 2026 16:50:21 +0000 Subject: [PATCH] Console Machines tab: app version, OTA state, power cap, jobs and status lines from the app log (0.3.3 header) The IGNEUM-APP header is read from the newest upload's first line, from any tail, or from the first upload of the run; the app's status: line supplies peers, lifetime accepted and synced when the node log tail has none. Co-Authored-By: Claude Fable 5.1 --- relay/api/console.mjs | 58 +++++++++++++++++++++++++++++++++---------- relay/ui.html | 16 +++++++++--- 2 files changed, 57 insertions(+), 17 deletions(-) diff --git a/relay/api/console.mjs b/relay/api/console.mjs index dd11ef91b..919093f40 100644 --- a/relay/api/console.mjs +++ b/relay/api/console.mjs @@ -97,11 +97,27 @@ function parseMinerTail(tail) { if (acc) c.last_accepted_at = new Date(Number(acc[1]) * 1000).toISOString(); return c; } +const HEADER = /IGNEUM-APP version=(\S+) machine=(\S+) platform=(\S+) node=(\S+)/; +function parseHeader(text) { + const m = HEADER.exec(text || ''); + return m ? { version: m[1], id8: m[2], platform: m[3], node: m[4].replace(/^igneumd[_\/]/, '') } : null; +} function parseAppTail(tail) { const lines = tail.split('\n'); - const a = { telemetry: [], events: [] }; + const a = { telemetry: [], events: [], jobs: [] }; let m; if ((m = /Igneum Miner (\S+) on /.exec(tail))) a.version = m[1]; + const h = lastMatch(lines, HEADER); if (h) { a.version = h[1]; a.node_version = h[4].replace(/^igneumd[_\/]/, ''); } + const stampOf = l => { const t = /^(\d+)(?:\.\d+)? /.exec(l); return t ? new Date(Number(t[1]) * 1000).toISOString() : null; }; + const strip = l => l.replace(/^\d+(\.\d+)? /, ''); + // OTA: the newest line decides the state shown on the card + const ota = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(OTA: .+)$/); + if (ota) { const t = ota[1]; a.ota = { text: t.slice(0, 220), at: stampOf(ota[0]), state: /waiting for administrator approval|approval not given/.test(t) ? 'waiting for approval' : /deferred/.test(t) ? 'deferred' : /updated to|installed|applied/.test(t) ? 'updated' : /download/.test(t) ? 'downloading' : 'note' }; } + const pc = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(?:GPU )?(power cap.*)$/i); + if (pc) a.power_cap = { text: pc[1].slice(0, 220), at: stampOf(pc[0]), ok: !/NOT applied|nothing to read back/.test(pc[1]) }; + const jf = lastMatch(lines, /^\d+(?:\.\d+)? jobs: (.+)$/); + if (jf) a.jobs_file = { text: jf[1].slice(0, 220), at: stampOf(jf[0]) }; + a.jobs = lines.filter(l => /^\d+(?:\.\d+)? (?:\[\w+\] )?job [^:]+[:(]/.test(l) || /^\d+(?:\.\d+)? (?:\[\w+\] )?\d+ remote jobs? received/.test(l)).slice(-6).map(l => ({ at: stampOf(l), text: strip(l).replace(/^\[\w+\] /, '').slice(0, 220) })); for (const l of lines) { const s = /stability: (.+?): draw p95 (\d+) W, max (\d+) W \(cap (\d+) W\), max GPU (\d+) C, max memory (\d+) C, (\d+) samples/.exec(l); if (s) { const i = a.telemetry.findIndex(t => t.name === s[1]); const t = { name: s[1], p95_w: Number(s[2]), max_w: Number(s[3]), cap_w: Number(s[4]), max_gpu_c: Number(s[5]), max_mem_c: Number(s[6]), samples: Number(s[7]) }; if (i >= 0) a.telemetry[i] = t; else a.telemetry.push(t); } @@ -109,19 +125,26 @@ function parseAppTail(tail) { if (e && e[2] !== 'block') a.events.push({ at: new Date(Number(e[1]) * 1000).toISOString(), kind: e[2], text: e[3].slice(0, 200) }); } a.events = a.events.slice(-6); - if ((m = lastMatch(lines, /status: (.+?) \| node (.+?) \| up (.+)$/))) a.status = { miner: m[1], node: m[2], up: m[3] }; + if ((m = lastMatch(lines, /^(\d+)(?:\.\d+)? status: (.+?) \| node (.+?) \| up (.+)$/))) { + a.status = { miner: m[2], node: m[3], up: m[4], at: new Date(Number(m[1]) * 1000).toISOString() }; + const acc = /accepted (\d+) blocks \((\d+) this run\), ([\d.]+) MH\/s, (\w+)/.exec(m[2]); + if (acc) Object.assign(a.status, { accepted_total: Number(acc[1]), accepted_run: Number(acc[2]), mhs: Number(acc[3]), mining: acc[4] }); + const nd = /(\d+) blocks, (\d+) peers, (\w+)/.exec(m[3]); + if (nd) Object.assign(a.status, { node_blocks: Number(nd[1]), node_peers: Number(nd[2]), node_state: nd[3] }); + } return a; } async function machines(sql) { // node logs get a longer tail: the connection-manager (peers) line is rare on a busy node - const rows = await sql(`SELECT DISTINCT ON (label) label, machine, run_id, received_at, - right(lines, CASE WHEN label LIKE 'nodelog-%' THEN 80000 ELSE 20000 END) AS tail + const rows = await sql(`SELECT DISTINCT ON (label) label, machine, run_id, received_at, left(lines, 300) AS head, + right(lines, CASE WHEN label LIKE 'nodelog-%' THEN 80000 WHEN label ~ '^(win|mac)-[0-9a-f]{8}$' THEN 60000 ELSE 20000 END) AS tail FROM miner_logs WHERE received_at > now() - interval '36 hours' ORDER BY label, received_at DESC`); const groups = new Map(); for (const r of rows) { const p = parseLabel(r.label, r.machine); if (!p) continue; - const g = groups.get(p.id) || { id: p.id, platform: p.platform, hosts: new Set(), runs: new Set(), last_seen: null, node: {}, cards: [], app: null, labels: [] }; + const g = groups.get(p.id) || { id: p.id, platform: p.platform, hosts: new Set(), runs: new Set(), last_seen: null, node: {}, cards: [], app: null, labels: [], header: null }; + const hd = parseHeader(r.head) || parseHeader(r.tail); if (hd) g.header = hd; g.hosts.add(r.machine.replace(/-[0-9a-f]{8}$/, '')); g.runs.add(r.run_id); const at = iso(r.received_at); @@ -133,10 +156,14 @@ async function machines(sql) { groups.set(p.id, g); } // node version lives in the first upload of a run (the tail may have lost the banner): one cheap query for the current runs - const runIds = [...groups.values()].map(g => g.node.run_id).filter(Boolean); + const runIds = [...new Set([...groups.values()].flatMap(g => [...g.runs]))]; if (runIds.length) { - const heads = await sql(`SELECT DISTINCT ON (run_id) run_id, left(lines, 3000) AS head FROM miner_logs WHERE run_id = ANY($1) AND label LIKE 'nodelog-%' ORDER BY run_id, received_at ASC`, [runIds]); - for (const h of heads) { const m = /igneumd\/(\S+)/.exec(h.head); if (!m) continue; for (const g of groups.values()) if (g.node.run_id === h.run_id && !g.node.version) g.node.version = m[1]; } + const heads = await sql(`SELECT DISTINCT ON (run_id, label) run_id, label, left(lines, 3000) AS head FROM miner_logs WHERE run_id = ANY($1) ORDER BY run_id, label, received_at ASC`, [runIds]); + for (const h of heads) { + const g = [...groups.values()].find(x => x.runs.has(h.run_id)); if (!g) continue; + const m = /igneumd\/(\S+)/.exec(h.head); if (m && h.label.startsWith('nodelog-') && !g.node.version) g.node.version = m[1]; + const hd = parseHeader(h.head); if (hd && !g.header) g.header = hd; + } } const now = Date.now(); const out = [...groups.values()].map(g => { @@ -148,12 +175,17 @@ async function machines(sql) { name: id8 ? (NAMES[id8] || (g.platform === 'mac' ? 'Mac' : 'PC ' + id8)) : g.id.slice(7), host: [...g.hosts][0] || null, legacy: !id8, last_seen: g.last_seen, silent_s, silent: silent_s === null || silent_s > SILENT_S, - app_version: g.app ? g.app.version || null : null, - node: { version: g.node.version || null, daa: g.node.daa ?? null, blue_score: g.node.blue_score ?? null, checkpoint: g.node.checkpoint ?? null, peers: g.node.peers ?? null, peers_wanted: g.node.peers_wanted ?? null, blocks_10s: g.node.blocks_10s ?? null, synced: cards.some(c => c.synced === true) || (g.node.blocks_10s != null && !g.node.ibd_seen), ibd: !!g.node.ibd_seen, last_line_at: g.node.last_line_at || null, warnings: g.node.warnings || 0, last_warning: g.node.last_warning || null, at: g.node.at || null }, + app_version: (g.app && g.app.version) || (g.header && g.header.version) || null, + app: g.app ? { at: g.app.at, status: g.app.status || null, ota: g.app.ota || null, power_cap: g.app.power_cap || null, jobs_file: g.app.jobs_file || null, jobs: g.app.jobs || [] } : null, + node: { version: g.node.version || (g.header && g.header.node) || (g.app && g.app.node_version) || null, daa: g.node.daa ?? null, blue_score: g.node.blue_score ?? null, checkpoint: g.node.checkpoint ?? null, + peers: g.node.peers ?? (g.app && g.app.status && g.app.status.node_peers != null ? g.app.status.node_peers : null), peers_wanted: g.node.peers_wanted ?? null, peers_from: g.node.peers != null ? 'node log' : (g.app && g.app.status ? 'app status line' : null), + blocks: g.app && g.app.status && g.app.status.node_blocks != null ? g.app.status.node_blocks : null, + blocks_10s: g.node.blocks_10s ?? null, synced: cards.some(c => c.synced === true) || (g.node.blocks_10s != null && !g.node.ibd_seen) || !!(g.app && g.app.status && g.app.status.node_state === 'synced'), ibd: !!g.node.ibd_seen, last_line_at: g.node.last_line_at || null, warnings: g.node.warnings || 0, last_warning: g.node.last_warning || null, at: g.node.at || (g.app && g.app.status ? g.app.status.at : null) }, cards, hash_total: Math.round(cards.reduce((a, c) => a + (c.hash_now || 0), 0) * 100) / 100, - accepted_total: cards.reduce((a, c) => a + (c.accepted || 0), 0), + accepted_total: cards.reduce((a, c) => a + (c.accepted || 0), 0) || (g.app && g.app.status && g.app.status.accepted_run) || 0, + accepted_lifetime: g.app && g.app.status ? g.app.status.accepted_total ?? null : null, faults: cards.reduce((a, c) => a + (c.faults || 0), 0), - telemetry: g.app ? g.app.telemetry : [], events: g.app ? g.app.events : [], status: g.app ? g.app.status || null : null, + telemetry: g.app ? g.app.telemetry : [], events: g.app ? g.app.events : [], labels: g.labels.sort((a, b) => (b.at > a.at ? 1 : -1)), runs: [...g.runs], }; }); @@ -163,7 +195,7 @@ async function machines(sql) { // several Mac installs (test runs): the newest is "Mac", the others keep their id let macSeen = false; for (const m of out) if (m.platform === 'mac' && m.name === 'Mac') { if (macSeen) m.name = `Mac ${m.id8}`; macSeen = true; } - for (const n of ['Mac', 'PC 1', 'PC 2', "Sam's Mac"]) if (!out.some(m => m.name === n)) out.push({ id: n, id8: Object.keys(NAMES).find(k => NAMES[k] === n) || null, platform: n.endsWith('Mac') ? 'mac' : 'win', name: n, host: null, legacy: false, last_seen: null, silent_s: null, silent: true, app_version: null, node: {}, cards: [], hash_total: 0, accepted_total: 0, faults: 0, telemetry: [], events: [], status: null, labels: [], runs: [], never: true }); + for (const n of ['Mac', 'PC 1', 'PC 2', "Sam's Mac"]) if (!out.some(m => m.name === n)) out.push({ id: n, id8: Object.keys(NAMES).find(k => NAMES[k] === n) || null, platform: n.endsWith('Mac') ? 'mac' : 'win', name: n, host: null, legacy: false, last_seen: null, silent_s: null, silent: true, app_version: null, app: null, node: {}, cards: [], hash_total: 0, accepted_total: 0, faults: 0, telemetry: [], events: [], labels: [], runs: [], never: true }); out.sort((a, b) => order(a) - order(b) || (b.last_seen || '').localeCompare(a.last_seen || '')); return { machines: out.filter(m => NAMED.has(m.name) || (m.silent_s !== null && m.silent_s < HIDE_AFTER_S)), silent_after_s: SILENT_S, hide_after_s: HIDE_AFTER_S }; } diff --git a/relay/ui.html b/relay/ui.html index 1154148bf..af7ec4d02 100644 --- a/relay/ui.html +++ b/relay/ui.html @@ -267,25 +267,33 @@ input[type=text]{width:100%} function machineCard(mc) { const n = mc.node || {}; const silentTxt = mc.never ? 'never seen' : (mc.silent ? `silent ${ago(mc.last_seen)}` : `seen ${ago(mc.last_seen)} ago`); - const tele = mc.telemetry && mc.telemetry.length ? `
${mc.telemetry.map(t => ``).join('')}
telemetryp95maxcapGPUmem
${esc(t.name)}${t.p95_w} W${t.max_w} W${t.cap_w} W${t.max_gpu_c} C${t.max_mem_c} C
` : (mc.cards.length ? `
no power or temperature lines in the intake yet (the app log is not uploading)
` : ''); + const tele = mc.telemetry && mc.telemetry.length ? `
${mc.telemetry.map(t => ``).join('')}
telemetryp95maxcapGPUmem
${esc(t.name)}${t.p95_w} W${t.max_w} W${t.cap_w} W${t.max_gpu_c} C${t.max_mem_c} C
` : (mc.cards.length ? `
${mc.app ? 'no stability lines yet (power and temperatures come from nvidia-smi every 5 min; none on a Mac)' : 'the app log is not in the intake (an app before 0.3.3)'}
` : ''); const events = mc.events && mc.events.length ? `
${mc.events.map(e => `
${when(e.at)}${e.kind}${esc(e.text)}
`).join('')}
` : ''; return `

${esc(mc.name)}${esc(mc.id8 || '')}${mc.host ? ' ' + esc(mc.host) : ''}${mc.legacy ? ' (old launcher)' : ''}${silentTxt}

${mc.never ? '
nothing from this machine in the log intake in the last 48 h
' : kvs([ ['app', mc.app_version ? esc(mc.app_version) : 'not in the intake'], + ['ota', mc.app && mc.app.ota ? esc(mc.app.ota.state) : (mc.app_version ? 'no update pending' : '?'), mc.app && mc.app.ota ? (mc.app.ota.state === 'updated' ? 'good' : 'warm') : ''], ['node', n.version ? esc(n.version) : '?'], ['height (daa)', n.daa != null ? nf(n.daa) : '?', ''], ['synced', n.at ? (n.synced ? 'yes' : (n.ibd ? 'IBD' : 'no')) : '?', n.at ? (n.synced ? 'good' : 'bad') : ''], ['hash rate', mh(mc.hash_total), mc.hash_total ? '' : 'warm'], - ['accepted', nf(mc.accepted_total)], - ['peers', n.peers != null ? `${n.peers}/${n.peers_wanted}` : '?'], + ['accepted', `${nf(mc.accepted_total)}${mc.accepted_lifetime != null ? ` of ${nf(mc.accepted_lifetime)} lifetime` : ''}`], + ['peers', n.peers != null ? `${n.peers}${n.peers_wanted ? `/${n.peers_wanted}` : ''}` : '?'], ['blocks / 10 s', n.blocks_10s != null ? nf(n.blocks_10s) : '?'], ['faults', nf(mc.faults), mc.faults ? 'bad' : 'good'], ['node warnings', nf(n.warnings || 0), n.warnings ? 'warm' : ''], ['checkpoint', n.checkpoint != null ? nf(n.checkpoint) : '?'], ['run', `${esc((n.run_id || (mc.runs && mc.runs[0]) || '').replace(/^.*-(\d{8}-\d{6})$/, '$1'))}`], ])} - ${cardTable(mc.cards)}${tele}${events} + ${cardTable(mc.cards)}${tele} + ${mc.app && (mc.app.ota || mc.app.power_cap || mc.app.jobs_file || (mc.app.jobs && mc.app.jobs.length) || mc.app.status) ? `
+ ${mc.app.status ? `
${when(mc.app.status.at)}status${esc(mc.app.status.miner)} | node ${esc(mc.app.status.node)} | up ${esc(mc.app.status.up)}
` : ''} + ${mc.app.ota ? `
${when(mc.app.ota.at)}ota${esc(mc.app.ota.text)}
` : ''} + ${mc.app.power_cap ? `
${when(mc.app.power_cap.at)}power${esc(mc.app.power_cap.text)}
` : ''} + ${mc.app.jobs_file ? `
${when(mc.app.jobs_file.at)}jobs${esc(mc.app.jobs_file.text)}
` : ''} + ${(mc.app.jobs || []).map(j => `
${when(j.at)}job${esc(j.text)}
`).join('')} +
` : ''}${events} ${n.last_warning ? `
${esc(n.last_warning)}
` : ''}
`; }