From c98b6a546db7ab1bdef2a00a6d6cda069b6df789 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Sun, 4 Oct 2026 20:04:18 +0000 Subject: [PATCH] Console: a machine whose app logged a clean quit or an update, with no status line after it, shows 'stopped (quit|update) N ago' instead of 'silent' (parseAppTail moved to relay/lib/parse.mjs, test) Co-Authored-By: Claude Fable 5.1 --- relay/api/console.mjs | 40 ++------------------------------ relay/lib/parse.mjs | 48 +++++++++++++++++++++++++++++++++++++++ relay/test/parse.test.mjs | 22 ++++++++++++++++++ relay/ui.html | 5 ++-- tools/console.mjs | 2 +- 5 files changed, 76 insertions(+), 41 deletions(-) diff --git a/relay/api/console.mjs b/relay/api/console.mjs index 111024789..06275da40 100644 --- a/relay/api/console.mjs +++ b/relay/api/console.mjs @@ -12,7 +12,7 @@ // POST post {kind,title,body,who,key?,meta?} one item; with key it upserts // POST sync {items:[...]} bulk upsert by key import { neon, authed, readJson, str, iso } from '../lib/relay.mjs'; -import { kv, kvNum, lastMatch, FAULT, parseLabel, parseMinerTail, STALE_S, markStale } from '../lib/parse.mjs'; +import { kv, kvNum, lastMatch, FAULT, parseLabel, parseMinerTail, parseHeader, parseAppTail, STALE_S, markStale } from '../lib/parse.mjs'; const json = (res, status, obj) => { res.status(status).setHeader('Content-Type', 'application/json; charset=utf-8'); res.end(JSON.stringify(obj)); }; const CACHE_MS = 10_000; @@ -55,43 +55,6 @@ function parseNodeTail(tail) { n.last_warning = errs.length ? errs[errs.length - 1].replace(/^\S+ \S+ /, '').slice(0, 200) : null; return n; } -const HEADER = /IGNEUM-APP version=(\S+) machine=(\S+) platform=(\S+) node=(\S+)/; -function parseHeader(text) { - const m = HEADER.exec(text || ''); - return m ? { version: m[1], id8: m[2], platform: m[3], node: m[4].replace(/^igneumd[_\/]/, '') } : null; -} -function parseAppTail(tail) { - const lines = tail.split('\n'); - const a = { telemetry: [], events: [], jobs: [] }; - let m; - if ((m = /Igneum Miner (\S+) on /.exec(tail))) a.version = m[1]; - const h = lastMatch(lines, HEADER); if (h) { a.version = h[1]; a.node_version = h[4].replace(/^igneumd[_\/]/, ''); } - const stampOf = l => { const t = /^(\d+)(?:\.\d+)? /.exec(l); return t ? new Date(Number(t[1]) * 1000).toISOString() : null; }; - const strip = l => l.replace(/^\d+(\.\d+)? /, ''); - // OTA: the newest line decides the state shown on the card - const ota = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(OTA: .+)$/); - if (ota) { const t = ota[1]; a.ota = { text: t.slice(0, 220), at: stampOf(ota[0]), state: /waiting for administrator approval|approval not given/.test(t) ? 'waiting for approval' : /deferred/.test(t) ? 'deferred' : /updated to|installed|applied/.test(t) ? 'updated' : /download/.test(t) ? 'downloading' : 'note' }; } - const pc = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(?:GPU )?(power cap.*)$/i); - if (pc) a.power_cap = { text: pc[1].slice(0, 220), at: stampOf(pc[0]), ok: !/NOT applied|nothing to read back/.test(pc[1]) }; - const jf = lastMatch(lines, /^\d+(?:\.\d+)? jobs: (.+)$/); - if (jf) a.jobs_file = { text: jf[1].slice(0, 220), at: stampOf(jf[0]) }; - a.jobs = lines.filter(l => /^\d+(?:\.\d+)? (?:\[\w+\] )?job [^:]+[:(]/.test(l) || /^\d+(?:\.\d+)? (?:\[\w+\] )?\d+ remote jobs? received/.test(l)).slice(-6).map(l => ({ at: stampOf(l), text: strip(l).replace(/^\[\w+\] /, '').slice(0, 220) })); - for (const l of lines) { - const s = /stability: (.+?): draw p95 (\d+) W, max (\d+) W \(cap (\d+) W\), max GPU (\d+) C, max memory (\d+) C, (\d+) samples/.exec(l); - if (s) { const i = a.telemetry.findIndex(t => t.name === s[1]); const t = { name: s[1], p95_w: Number(s[2]), max_w: Number(s[3]), cap_w: Number(s[4]), max_gpu_c: Number(s[5]), max_mem_c: Number(s[6]), samples: Number(s[7]) }; if (i >= 0) a.telemetry[i] = t; else a.telemetry.push(t); } - const e = /^(\d+(?:\.\d+)?) \[(error|ok|block|warn)\] (.+)$/.exec(l); - if (e && e[2] !== 'block') a.events.push({ at: new Date(Number(e[1]) * 1000).toISOString(), kind: e[2], text: e[3].slice(0, 200) }); - } - a.events = a.events.slice(-6); - if ((m = lastMatch(lines, /^(\d+)(?:\.\d+)? status: (.+?) \| node (.+?) \| up (.+)$/))) { - a.status = { miner: m[2], node: m[3], up: m[4], at: new Date(Number(m[1]) * 1000).toISOString() }; - const acc = /accepted (\d+) blocks \((\d+) this run\), ([\d.]+) MH\/s, (\w+)/.exec(m[2]); - if (acc) Object.assign(a.status, { accepted_total: Number(acc[1]), accepted_run: Number(acc[2]), mhs: Number(acc[3]), mining: acc[4] }); - const nd = /(\d+) blocks, (\d+) peers, (\w+)/.exec(m[3]); - if (nd) Object.assign(a.status, { node_blocks: Number(nd[1]), node_peers: Number(nd[2]), node_state: nd[3] }); - } - return a; -} async function machines(sql) { // node logs get a longer tail: the connection-manager (peers) line is rare on a busy node const rows = await sql(`SELECT DISTINCT ON (label) label, machine, run_id, received_at, left(lines, 300) AS head, @@ -133,6 +96,7 @@ async function machines(sql) { name: id8 ? (NAMES[id8] || (g.platform === 'mac' ? 'Mac' : 'PC ' + id8)) : g.id.slice(7), host: [...g.hosts][0] || null, legacy: !id8, last_seen: g.last_seen, silent_s, silent: silent_s === null || silent_s > SILENT_S, + stopped: (silent_s === null || silent_s > SILENT_S) && g.app && g.app.stopped ? g.app.stopped : null, app_version: (g.app && g.app.version) || (g.header && g.header.version) || null, app: g.app ? { at: g.app.at, status: g.app.status || null, ota: g.app.ota || null, power_cap: g.app.power_cap || null, jobs_file: g.app.jobs_file || null, jobs: g.app.jobs || [] } : null, node: { version: g.node.version || (g.header && g.header.node) || (g.app && g.app.node_version) || null, daa: g.node.daa ?? null, blue_score: g.node.blue_score ?? null, checkpoint: g.node.checkpoint ?? null, diff --git a/relay/lib/parse.mjs b/relay/lib/parse.mjs index e185bd489..9565ca731 100644 --- a/relay/lib/parse.mjs +++ b/relay/lib/parse.mjs @@ -63,3 +63,51 @@ export function parseMinerTail(tail) { if (acc) c.last_accepted_at = new Date(Number(acc[1]) * 1000).toISOString(); return c; } + +// The app log: the IGNEUM-APP header, the status line, OTA and power-cap lines, telemetry, events, jobs, and a clean stop. +export const HEADER = /IGNEUM-APP version=(\S+) machine=(\S+) platform=(\S+) node=(\S+)/; +export function parseHeader(text) { + const m = HEADER.exec(text || ''); + return m ? { version: m[1], id8: m[2], platform: m[3], node: m[4].replace(/^igneumd[_\/]/, '') } : null; +} +export function parseAppTail(tail) { + const lines = tail.split('\n'); + const a = { telemetry: [], events: [], jobs: [] }; + let m; + if ((m = /Igneum Miner (\S+) on /.exec(tail))) a.version = m[1]; + const h = lastMatch(lines, HEADER); if (h) { a.version = h[1]; a.node_version = h[4].replace(/^igneumd[_\/]/, ''); } + const stampOf = l => { const t = /^(\d+)(?:\.\d+)? /.exec(l); return t ? new Date(Number(t[1]) * 1000).toISOString() : null; }; + const strip = l => l.replace(/^\d+(\.\d+)? /, ''); + // OTA: the newest line decides the state shown on the card + const ota = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(OTA: .+)$/); + if (ota) { const t = ota[1]; a.ota = { text: t.slice(0, 220), at: stampOf(ota[0]), state: /waiting for administrator approval|approval not given/.test(t) ? 'waiting for approval' : /deferred/.test(t) ? 'deferred' : /updated to|installed|applied/.test(t) ? 'updated' : /download/.test(t) ? 'downloading' : 'note' }; } + const pc = lastMatch(lines, /^\d+(?:\.\d+)? (?:\[\w+\] )?(?:GPU )?(power cap.*)$/i); + if (pc) a.power_cap = { text: pc[1].slice(0, 220), at: stampOf(pc[0]), ok: !/NOT applied|nothing to read back/.test(pc[1]) }; + const jf = lastMatch(lines, /^\d+(?:\.\d+)? jobs: (.+)$/); + if (jf) a.jobs_file = { text: jf[1].slice(0, 220), at: stampOf(jf[0]) }; + a.jobs = lines.filter(l => /^\d+(?:\.\d+)? (?:\[\w+\] )?job [^:]+[:(]/.test(l) || /^\d+(?:\.\d+)? (?:\[\w+\] )?\d+ remote jobs? received/.test(l)).slice(-6).map(l => ({ at: stampOf(l), text: strip(l).replace(/^\[\w+\] /, '').slice(0, 220) })); + for (const l of lines) { + const s = /stability: (.+?): draw p95 (\d+) W, max (\d+) W \(cap (\d+) W\), max GPU (\d+) C, max memory (\d+) C, (\d+) samples/.exec(l); + if (s) { const i = a.telemetry.findIndex(t => t.name === s[1]); const t = { name: s[1], p95_w: Number(s[2]), max_w: Number(s[3]), cap_w: Number(s[4]), max_gpu_c: Number(s[5]), max_mem_c: Number(s[6]), samples: Number(s[7]) }; if (i >= 0) a.telemetry[i] = t; else a.telemetry.push(t); } + const e = /^(\d+(?:\.\d+)?) \[(error|ok|block|warn)\] (.+)$/.exec(l); + if (e && e[2] !== 'block') a.events.push({ at: new Date(Number(e[1]) * 1000).toISOString(), kind: e[2], text: e[3].slice(0, 200) }); + } + a.events = a.events.slice(-6); + // A clean stop: the app logs "quit: ..." (Quit, Stop, or an update: "[info] installing Igneum Miner ...") and uploads + // once more before it exits; when no status line follows, the machine was stopped on purpose, not lost (4 October + // 2026: a Mac stopped at 14:47 UTC read "silent 4h" for the whole afternoon, the same as a crash or a lost network). + const stop = lastMatch(lines, /^(\d+)(?:\.\d+)? (quit: .+|stopped|\[info\] installing Igneum Miner .+)$/); + const lastStatus = lastMatch(lines, /^(\d+)(?:\.\d+)? status: /); + if (stop && (!lastStatus || Number(stop[1]) >= Number(lastStatus[1]))) { + const update = lines.some(l => /\[info\] installing Igneum Miner/.test(l) && Number((/^(\d+)/.exec(l) || [])[1]) >= Number(stop[1]) - 30); + a.stopped = { at: stampOf(stop[0]), reason: update ? 'update' : 'quit', text: strip(stop[0]).slice(0, 120) }; + } + if ((m = lastMatch(lines, /^(\d+)(?:\.\d+)? status: (.+?) \| node (.+?) \| up (.+)$/))) { + a.status = { miner: m[2], node: m[3], up: m[4], at: new Date(Number(m[1]) * 1000).toISOString() }; + const acc = /accepted (\d+) blocks \((\d+) this run\), ([\d.]+) MH\/s, (\w+)/.exec(m[2]); + if (acc) Object.assign(a.status, { accepted_total: Number(acc[1]), accepted_run: Number(acc[2]), mhs: Number(acc[3]), mining: acc[4] }); + const nd = /(\d+) blocks, (\d+) peers, (\w+)/.exec(m[3]); + if (nd) Object.assign(a.status, { node_blocks: Number(nd[1]), node_peers: Number(nd[2]), node_state: nd[3] }); + } + return a; +} diff --git a/relay/test/parse.test.mjs b/relay/test/parse.test.mjs index ade4eadf5..991e0a774 100644 --- a/relay/test/parse.test.mjs +++ b/relay/test/parse.test.mjs @@ -41,3 +41,25 @@ test('stale: a card is "now" within STALE_S of its STATUS line and stale after i const oneMissed = { status_at: new Date(now - 150_000).toISOString(), hash_now: 18 }; markStale([oneMissed], now); assert.equal(oneMissed.stale, false); }); + +test('app tail: a clean quit after the last status line is "stopped (quit)", an update is "stopped (update)", a running app is not stopped', async () => { + const { parseAppTail } = await import('../lib/parse.mjs'); + const running = ['1791126700 status: accepted 10 blocks (5 this run), 18.00 MH/s, mining | node 100 blocks, 4 peers, synced | up 00:10:00'].join('\n'); + assert.equal(parseAppTail(running).stopped, undefined); + const quit = [ + '1791126700 status: accepted 10 blocks (5 this run), 18.00 MH/s, mining | node 100 blocks, 4 peers, synced | up 00:10:00', + '1791126735 quit: stopping the miners, then the node', '1791126735 stopping the miners (quit)', '1791126736 node stopped (exit Some(0))', + '1791126736 SUMMARY after 00:26:53: node started 1 time(s), restarts 0, 1165 accepted blocks this run (1270 lifetime); data stays in x', '1791126737 stopped', + ].join('\n'); + const q = parseAppTail(quit); + assert.deepEqual([q.stopped.reason, q.stopped.at], ['quit', new Date(1791126737000).toISOString()]); + const update = [ + '1791125100 status: accepted 1 blocks (1 this run), 1.00 MH/s, mining | node 1 blocks, 1 peers, synced | up 00:01:00', + '1791125117 [info] installing Igneum Miner 0.3.2: the miners stop, then the node, then the app opens again', + '1791125118 quit: stopping the miners, then the node', '1791125121 stopped', + ].join('\n'); + assert.equal(parseAppTail(update).stopped.reason, 'update'); + // a quit line from an earlier run in the same tail, with a status line after it: still running + const resumed = quit + '\n1791126800 status: accepted 11 blocks (1 this run), 18.00 MH/s, mining | node 120 blocks, 4 peers, synced | up 00:01:00'; + assert.equal(parseAppTail(resumed).stopped, undefined); +}); diff --git a/relay/ui.html b/relay/ui.html index 716c0f770..46e971f4e 100644 --- a/relay/ui.html +++ b/relay/ui.html @@ -42,6 +42,7 @@ main{padding-block:14px 48px} .card{background:var(--graphite);border:1px solid var(--line);border-radius:var(--r);padding:14px;min-width:0} .card.red{border-color:var(--red);box-shadow:inset 3px 0 0 var(--red)} .card.live{box-shadow:inset 3px 0 0 var(--green)} +.card.stopped{box-shadow:inset 3px 0 0 var(--ash)} .card h3{margin:0;font-family:'Unbounded',sans-serif;font-weight:700;font-size:14px;letter-spacing:.02em;display:flex;align-items:center;gap:10px;flex-wrap:wrap} .card h3 .id{font-family:'IBM Plex Mono',monospace;font-weight:400;font-size:11px;color:var(--ash);letter-spacing:.04em} .card h3 .seen{margin-left:auto;font-family:'IBM Plex Mono',monospace;font-weight:400;font-size:11px;color:var(--ash)} @@ -266,10 +267,10 @@ input[type=text]{width:100%} } function machineCard(mc) { const n = mc.node || {}; - const silentTxt = mc.never ? 'never seen' : (mc.silent ? `silent ${ago(mc.last_seen)}` : `seen ${ago(mc.last_seen)} ago`); + const silentTxt = mc.never ? 'never seen' : mc.stopped ? `stopped (${esc(mc.stopped.reason)}) ${ago(mc.stopped.at)} ago` : (mc.silent ? `silent ${ago(mc.last_seen)}` : `seen ${ago(mc.last_seen)} ago`); const tele = mc.telemetry && mc.telemetry.length ? `
${mc.telemetry.map(t => ``).join('')}
telemetryp95maxcapGPUmem
${esc(t.name)}${t.p95_w} W${t.max_w} W${t.cap_w} W${t.max_gpu_c} C${t.max_mem_c} C
` : (mc.cards.length ? `
${mc.app ? 'no stability lines yet (power and temperatures come from nvidia-smi every 5 min; none on a Mac)' : 'the app log is not in the intake (an app before 0.3.3)'}
` : ''); const events = mc.events && mc.events.length ? `
${mc.events.map(e => `
${when(e.at)}${e.kind}${esc(e.text)}
`).join('')}
` : ''; - return `
+ return `

${esc(mc.name)}${esc(mc.id8 || '')}${mc.host ? ' ' + esc(mc.host) : ''}${mc.legacy ? ' (old launcher)' : ''}${silentTxt}

${mc.never ? '
nothing from this machine in the log intake in the last 48 h
' : kvs([ ['app', mc.app_version ? esc(mc.app_version) : 'not in the intake'], diff --git a/tools/console.mjs b/tools/console.mjs index c8929007e..40feee60d 100644 --- a/tools/console.mjs +++ b/tools/console.mjs @@ -110,7 +110,7 @@ try { else if (cmd === 'machines') { const j = await api('machines'); for (const m of j.machines) { - console.log(`${m.name.padEnd(8)} ${m.id8 || '-'} ${m.silent ? 'SILENT' : 'live'} seen ${ago(m.last_seen)} ago | app ${m.app_version || '?'} node ${m.node.version || '?'} daa ${m.node.daa ?? '?'} peers ${m.node.peers ?? '?'} ${m.node.synced ? 'synced' : 'not synced'} | ${m.hash_total.toFixed(1)} MH/s, ${m.accepted_total} accepted, ${m.faults} faults`); + console.log(`${m.name.padEnd(8)} ${m.id8 || '-'} ${m.stopped ? `STOPPED (${m.stopped.reason}) ${ago(m.stopped.at)} ago,` : m.silent ? 'SILENT' : 'live'} seen ${ago(m.last_seen)} ago | app ${m.app_version || '?'} node ${m.node.version || '?'} daa ${m.node.daa ?? '?'} peers ${m.node.peers ?? '?'} ${m.node.synced ? 'synced' : 'not synced'} | ${m.hash_total.toFixed(1)} MH/s, ${m.accepted_total} accepted, ${m.faults} faults`); for (const c of m.cards) console.log(` ${c.label.padEnd(22)} ${c.stale ? 'last ' + c.hash_now.toFixed(1) + ' MH/s (stale, not in the total)' : c.hash_now.toFixed(1) + ' MH/s now'}, ${c.accepted} accepted, ${c.rejected} rejected, ${c.mismatched} mismatched, ${c.restarts} restarts, status ${ago(c.status_at)} ago${c.fault ? ' | ' + c.fault : ''}`); for (const t of m.telemetry) console.log(` ${t.name}: p95 ${t.p95_w} W, max ${t.max_w} W (cap ${t.cap_w} W), GPU ${t.max_gpu_c} C, memory ${t.max_mem_c} C`); }