From 9915c883b189fcd3bf7a2dfa4178e9abbfc9906b Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:15:03 +0000 Subject: [PATCH] Bug hunt: console cards for other/intel workers and a stale mark on old STATUS lines (relay/lib/parse.mjs + test in CI); publish-jobs verifies the live file with retries and named reasons, a verify command, a failed deploy stops, a collect command without $_ is refused; the dl token masked in printed URLs; docs/bugs.md Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 2 ++ docs/bugs.md | 20 +++++++++++ packaging/ota/publish-jobs.sh | 51 ++++++++++++++++++++++------ relay/api/console.mjs | 48 ++------------------------ relay/lib/parse.mjs | 63 +++++++++++++++++++++++++++++++++++ relay/test/parse.test.mjs | 41 +++++++++++++++++++++++ relay/ui.html | 2 +- tools/console.mjs | 2 +- tools/jobs.mjs | 2 +- 9 files changed, 173 insertions(+), 58 deletions(-) create mode 100644 docs/bugs.md create mode 100644 relay/lib/parse.mjs create mode 100644 relay/test/parse.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab3c3e94..3d9613b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,3 +61,5 @@ jobs: run: node tools/ci/link-check.mjs - name: identity grep of the public export list run: bash tools/ci/identity-check.sh + - name: console parsers (relay/lib/parse.mjs) + run: node --test relay/test/parse.test.mjs diff --git a/docs/bugs.md b/docs/bugs.md new file mode 100644 index 00000000..75bce1fe --- /dev/null +++ b/docs/bugs.md @@ -0,0 +1,20 @@ +# Bugs found and fixed (bug hunt log) + +One line per bug: date, symptom, cause, fix commit, how it was proven. Open items at the end. Started 4 October 2026, +20:20 BST, after two were found by hand half an hour late (three job watchers that never fired, 096b99e; a failed +shard run reported as exit 0, 7a7e873). + +| Date | Symptom | Cause | Fix | Proven by | +|---|---|---|---|---| +| 4 Oct 2026 | Every `ci` run on master red since 67bf226 (eleven pushes), unnoticed | `sim/difficulty/records/testnet-v2-2026-10-04.schedule.log` carried a home path; `.log` was outside the identity scrub's extension list in `tools/ci/identity-check.sh` (and in the mirror's `tools/sync.sh`) | 2996cca: `.log` scrubbed like the other text files; the record rewritten with `~`; the same list in igneum-public `tools/sync.sh` (local commit e18256d, not pushed) | `bash tools/ci/identity-check.sh` 0 hits locally; run 37226816xxx on master green | +| 4 Oct 2026 | `collect-pc1-board3` printed PowerShell parse errors (`.Name`, `.AdapterRAM`) | the publishing shell expanded `$_` inside double quotes to nothing before the command reached the jobs file; nothing to do with Format-List or Out-String (board2 and board4 printed their values) | publish-jobs.sh refuses a collect command that pipes into a script block without `$_` or `$PSItem` | the eaten form refused with the reason, the single-quoted form published to a test folder | +| 4 Oct 2026 | the same job reported `done (exit 0)` over `command exit Some(1)` | `run_collect` in `app/igneum-app/src/jobrun.rs` builds `Done` from the upload count only; the command's exit code is logged and dropped | branch `bugfix-collect-exit` (app engine; merge by the main session) | unit test on the outcome function, see the branch | +| 4 Oct 2026 | `publish-jobs.sh --deploy` said "not reachable, differs from the local one, or does not verify yet" after a deploy that had succeeded | one check the instant the CLI returned, while the edge still served the previous file; the deploy's own exit status was hidden by `\|\| true` | `verify_live`: up to `--tries` (12) checks 5 s apart, each failure names its condition; `publish-jobs.sh verify` re-checks on its own; a failed deploy stops before the check | finished: `verify --tries 2` against the live file (try 1 of 2); failed: a local server with an older file ("differs", both publish stamps named) and a closed port ("is not reachable") | +| 4 Oct 2026 | console Machines: PC 37ba0461 showed 0.0 MH/s and 0 accepted while its log held an accepted block at 1 MH/s | `parseLabel` in the console API knew nvidia, amd, mac, metal and opencl; the OpenCL fallback on an iGPU is labelled `other--n` and the card was dropped | parsers moved to `relay/lib/parse.mjs`, vendors `other` and `intel` added, `node --test relay/test/parse.test.mjs` in CI | the test; the live console after the deploy shows the card | +| 4 Oct 2026 | console Machines: a card said "117.2 MH/s now" while its "status" column said 4 m ago (PC 2 during shard run 3: the prover held the GPU and the worker's STATUS line stopped) | the card's hash came from the last STATUS line in the tail with no age check; the machine total summed it | `markStale`: a card whose STATUS line is older than 120 s is `stale`, shown as "last N MH/s" with a red "stale" mark, and left out of the machine total (API, page and `tools/console.mjs`) | the test (58 s fresh, 240 s stale, none stale); the live console after the deploy | +| 4 Oct 2026 | `tools/jobs.mjs` and `publish-jobs.sh` print the downloads-folder token inside URLs on every run (X24 pattern) | the tokened base URL is echoed as is | the token masked as `` in every printed URL | by eye, this log's own transcript | + +## Open + +- Sam's Mac (3a9bf309): the last node upload (14:47:02 UTC) ends with `SIGTERM - shutting down` and `igneumd has stopped`, the miner upload stops at 14:46:56, nothing after: a clean app-driven stop (quit or Stop), not a crash and not a network loss (the stop lines reached the intake). Its app predates the app-log upload (`app ?` on the console), so there is no app stream to say which. The app posts nothing on a clean quit, so the console shows "silent 4h" for a machine that was stopped on purpose. Proposed: one `[ok] app quit by the user` line uploaded before the engine stops, and the console card saying "stopped (quit) at 14:47" instead of "silent". +- `collect` jobs: the Format-List / Out-String loss did not reproduce (board2 printed `Sum`, board4 printed `Capacity` and `Speed`); only the `$_` case failed. Closed unless it shows again. diff --git a/packaging/ota/publish-jobs.sh b/packaging/ota/publish-jobs.sh index 9e1a4cee..d9fd4ea9 100755 --- a/packaging/ota/publish-jobs.sh +++ b/packaging/ota/publish-jobs.sh @@ -21,11 +21,13 @@ # packaging/ota/publish-jobs.sh list what is published (expired jobs marked) # packaging/ota/publish-jobs.sh remove [--deploy] # packaging/ota/publish-jobs.sh sign [--deploy] re-sign the file as it is (expired jobs dropped) +# packaging/ota/publish-jobs.sh verify [--tries N] check the live file against the local one (reachable, identical, verifies) # # Jobs already in the file stay (expired ones are dropped on every write). A machine runs an id once: to run the # same thing again, add it again (a new id is generated from the kind and the time unless --id is given). # Without --deploy the script prints the deploy command; with --deploy it runs the Vercel CLI from the downloads -# folder and verifies the live file. Testing: --dest writes elsewhere; --base-url overrides the file URLs. +# folder and verifies the live file (up to --tries times, 5 s apart: the edge serves the previous file for a few +# seconds after a deploy). Testing: --dest writes elsewhere; --base-url overrides the file URLs. set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" ROOT="$(cd "$HERE/../.." && pwd)" @@ -36,7 +38,7 @@ TOKEN_FILE="$HOME/.config/igneum/dl-token" SIGNER="$ROOT/app/igneum-app/target/release/igneum-ota-sign" CMD="${1:-}"; [ $# -gt 0 ] && shift -KIND="" TARGET="" PLATFORM="" REQUIRES="" REQUIRES_SET=0 ID="" TITLE="" EXPIRES_H="48" DEPLOY=0 BASE="" DEST="" +KIND="" TARGET="" PLATFORM="" REQUIRES="" REQUIRES_SET=0 ID="" TITLE="" EXPIRES_H="48" DEPLOY=0 BASE="" DEST="" TRIES=12 SCRIPT="" SHELL_KIND="" ELEVATED=0 STOP_MINERS=0 TIMEOUT_MIN="" FILE="" URL="" SHA="" SIZE="" DIR="" TO="" EXTRACT=0 EXTRACT_DIR="" FRESH=0 GLOBS=() COMMAND="" WHAT="" @@ -79,10 +81,11 @@ while [ $# -gt 0 ]; do --no-deploy) DEPLOY=0; shift ;; --base-url) BASE="$2"; shift 2 ;; --dest) DEST="$2"; shift 2 ;; + --tries) TRIES="$2"; shift 2 ;; *) echo "unknown argument: $1" >&2; exit 2 ;; esac done -case "$CMD" in add|list|remove|sign) ;; *) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; esac +case "$CMD" in add|list|remove|sign|verify) ;; *) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; esac [ -f "$KEY" ] || { echo "no $KEY (see packaging/ota/README.md, Keys)" >&2; exit 1; } [ -f "$PUB" ] || { echo "no $PUB" >&2; exit 1; } @@ -113,6 +116,34 @@ if [ "$EMBEDDED" != "$OURS" ]; then exit 1 fi +# Checks the live jobs file against the local one: reachable, byte-identical, signature verifies. Retries, because the +# edge serves the previous deployment for some seconds after a deploy (4 October 2026: a deploy that had succeeded was +# reported as "not reachable, differs from the local one, or does not verify" by the one check made the moment the CLI +# returned). Each failure names the condition that failed. +verify_live() { # (5 s apart); 0 = verified, 1 = not, with the reason on stderr + local tries="${1:-12}" t=0 verdict="" tmp + tmp="$(mktemp -d)" + while [ "$t" -lt "$tries" ]; do + t=$((t + 1)); verdict="" + if ! curl -fsSL -H 'Cache-Control: no-cache' -o "$tmp/j.json" "$BASE/igneum-jobs.json"; then verdict="is not reachable" + elif ! curl -fsSL -H 'Cache-Control: no-cache' -o "$tmp/j.sig" "$BASE/igneum-jobs.json.sig"; then verdict="has no reachable signature" + elif ! cmp -s "$tmp/j.json" "$JOBS"; then verdict="differs from the local one (live $(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("published_at", "?"))' "$tmp/j.json" 2>/dev/null || echo unreadable), local $(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("published_at", "?"))' "$JOBS" 2>/dev/null || echo unreadable))" + elif ! "$SIGNER" verify-jobs "$PUB" "$tmp/j.json" "$tmp/j.sig" >/dev/null 2>&1; then verdict="does not verify against $PUB" + fi + [ -z "$verdict" ] && break + [ "$t" -lt "$tries" ] && sleep 5 + done + rm -rf "$tmp" + if [ -z "$verdict" ]; then echo "live jobs file verified at ${BASE//$TOKEN/}/igneum-jobs.json (try $t of $tries)"; return 0; fi + echo "the live jobs file $verdict after $t tr$([ "$t" = 1 ] && echo y || echo ies) ($(( (t - 1) * 5 )) s); check the deploy output, then: $0 verify" >&2 + return 1 +} + +if [ "$CMD" = verify ]; then + [ -f "$JOBS" ] || { echo "no local jobs file at $JOBS" >&2; exit 1; } + verify_live "$TRIES"; exit $? +fi + if [ "$CMD" = list ]; then [ -f "$JOBS" ] || { echo "no jobs file in $DEST"; exit 0; } "$SIGNER" verify-jobs "$PUB" "$JOBS" "$JOBS.sig" || { echo "the file in $DEST does not verify; run: $0 sign" >&2; exit 1; } @@ -167,6 +198,9 @@ print(json.dumps(d))' "$URL" "$SHA" "$SIZE" "$DIR" "$TO" "$EXTRACT" "$EXTRACT_DI ;; collect) [ ${#GLOBS[@]} -gt 0 ] || [ -n "$COMMAND" ] || { echo "collect: --glob and/or --command" >&2; exit 2; } + if printf '%s' "$COMMAND" | grep -qE 'ForEach-Object|Where-Object|\| *% |\| *\? ' && ! printf '%s' "$COMMAND" | grep -qE '\$_|\$PSItem'; then + echo "collect: the command pipes into a script block but holds no \$_ or \$PSItem; the shell that published it has expanded \$_ to nothing (4 October 2026, collect-pc1-board3: PowerShell saw '.Name' and failed to parse). Pass the command in single quotes." >&2; exit 2 + fi PARAMS="$(python3 -c 'import json,sys; d={"globs": [g for g in sys.argv[2:] if g]} if sys.argv[1]: d["command"]=sys.argv[1] print(json.dumps(d))' "$COMMAND" "${GLOBS[@]:-}")" @@ -238,17 +272,14 @@ echo "jobs file: $JOBS ($(wc -c < "$JOBS" | tr -d ' ') bytes)" if [ "$DEPLOY" = 1 ]; then [ -n "$DLSITE" ] || { echo "--deploy needs the real downloads folder (no --dest)" >&2; exit 1; } echo "deploying $DLSITE" - (cd "$DLSITE" && npx --yes vercel@latest --global-config "$HOME/.config/igneum/vercel" deploy --prod --yes 2>&1 | grep -v "$TOKEN" || true) - TMP="$(mktemp -d)" - curl -fsSL -o "$TMP/j.json" "$BASE/igneum-jobs.json" && curl -fsSL -o "$TMP/j.sig" "$BASE/igneum-jobs.json.sig" \ - && cmp -s "$TMP/j.json" "$JOBS" && "$SIGNER" verify-jobs "$PUB" "$TMP/j.json" "$TMP/j.sig" >/dev/null && echo "live jobs file verified at $BASE/igneum-jobs.json" \ - || { echo "the live jobs file is not reachable, differs from the local one, or does not verify yet; check the deploy output" >&2; rm -rf "$TMP"; exit 1; } - rm -rf "$TMP" + (cd "$DLSITE" && npx --yes vercel@latest --global-config "$HOME/.config/igneum/vercel" deploy --prod --yes 2>&1 | sed "s#$TOKEN##g"; exit "${PIPESTATUS[0]}") \ + || { echo "the deploy failed (the Vercel CLI's exit status above); nothing verified" >&2; exit 1; } + verify_live "$TRIES" || exit 1 echo "the apps pick it up within 10 minutes (Settings > remote jobs > Check now at once); results: node tools/jobs.mjs ${ID:-}" else if [ -n "$DLSITE" ]; then echo "not deployed: cd $DLSITE && npx --yes vercel@latest --global-config ~/.config/igneum/vercel deploy --prod --yes (or re-run with --deploy)" else - echo "written to $DEST for $BASE (test file; not the downloads folder)" + echo "written to $DEST for ${BASE//$TOKEN/} (test file; not the downloads folder)" fi fi diff --git a/relay/api/console.mjs b/relay/api/console.mjs index d5e4eef5..11102478 100644 --- a/relay/api/console.mjs +++ b/relay/api/console.mjs @@ -12,6 +12,7 @@ // POST post {kind,title,body,who,key?,meta?} one item; with key it upserts // POST sync {items:[...]} bulk upsert by key import { neon, authed, readJson, str, iso } from '../lib/relay.mjs'; +import { kv, kvNum, lastMatch, FAULT, parseLabel, parseMinerTail, STALE_S, markStale } from '../lib/parse.mjs'; const json = (res, status, obj) => { res.status(status).setHeader('Content-Type', 'application/json; charset=utf-8'); res.end(JSON.stringify(obj)); }; const CACHE_MS = 10_000; @@ -30,10 +31,6 @@ const NAMED = new Set(Object.values(NAMES).concat('Mac')); const HIDE_AFTER_S = 1800; // ids other than the named machines drop off 30 min after their last upload (test runs) const SILENT_S = 180; const num = v => (v === null || v === undefined || v === '' ? null : Number(v)); -const kv = (text, k) => { const m = new RegExp(`(?:^|[\\s(])${k}=([^\\s,)]+)`).exec(text); return m ? m[1] : null; }; -const kvNum = (text, k) => { const v = kv(text, k); return v === null ? null : Number(v); }; -const lastMatch = (lines, re) => { for (let i = lines.length - 1; i >= 0; i--) { const m = re.exec(lines[i]); if (m) return m; } return null; }; -const FAULT = /WORKER MISMATCH|worker error|worker exited|panicked|CUDA error|submit error/; async function fetchJson(url, ms = 6000) { const r = await fetch(url, { signal: AbortSignal.timeout(ms), headers: { accept: 'application/json' } }); @@ -42,17 +39,6 @@ async function fetchJson(url, ms = 6000) { } // ---- machines --------------------------------------------------------------------------------------------------- -function parseLabel(label, machine) { - let m; - if ((m = /^nodelog-(win|mac)-([0-9a-f]{8})$/.exec(label))) return { id: m[2], platform: m[1], stream: 'node' }; - if ((m = /^miner-(nvidia|amd|mac|metal|opencl)-([0-9a-f]{8})-(\d+)$/.exec(label))) return { id: m[2], platform: m[1] === 'mac' || m[1] === 'metal' ? 'mac' : 'win', stream: 'miner', vendor: m[1], card: Number(m[3]) }; - if ((m = /^(win|mac)-([0-9a-f]{8})$/.exec(label))) return { id: m[2], platform: m[1], stream: 'app' }; - // the launchers before the app: nodelog-, nvidia-[-n], amd-[-n], igneum-, launcher- - if ((m = /^nodelog-(.+)$/.exec(label))) return { id: 'legacy:' + m[1], platform: 'legacy', stream: 'node' }; - if ((m = /^(nvidia|amd)-(.+?)(?:-(\d+))?$/.exec(label))) return { id: 'legacy:' + m[2], platform: 'legacy', stream: 'miner', vendor: m[1], card: Number(m[3] || 1) }; - if ((m = /^(igneum|launcher)-(.+)$/.exec(label))) return { id: 'legacy:' + m[2], platform: 'legacy', stream: 'app' }; - return null; -} function parseNodeTail(tail) { const lines = tail.split('\n'); const n = {}; @@ -69,34 +55,6 @@ function parseNodeTail(tail) { n.last_warning = errs.length ? errs[errs.length - 1].replace(/^\S+ \S+ /, '').slice(0, 200) : null; return n; } -function parseMinerTail(tail) { - const lines = tail.split('\n'); - const c = { hash_now: 0, hash_avg: 0, accepted: 0, rejected: 0, mismatched: 0, restarts: 0, identities: null, synced: null, uptime_s: null, status_at: null, fault: null, faults: 0, template_age_s: null }; - const st = lastMatch(lines, /^(\d+(?:\.\d+)?) STATUS '([^']+)'.*$/); - if (st) { - const t = st[0]; - c.status_at = new Date(Number(st[1]) * 1000).toISOString(); - c.hash_avg = kvNum(t, 'hash') ?? 0; - c.hash_now = kvNum(t, 'now') ?? c.hash_avg; - c.accepted = kvNum(t, 'accepted') ?? 0; - c.rejected = kvNum(t, 'rejected') ?? 0; - c.mismatched = kvNum(t, 'mismatched') ?? 0; - c.restarts = kvNum(t, 'restarts') ?? 0; - c.identities = kvNum(t, 'identities'); - c.synced = kv(t, 'synced') === 'true'; - c.template_age_s = (() => { const v = kv(t, 'template_age'); return v ? Number(v.replace('s', '')) : null; })(); - const up = /STATUS '[^']+' \[[^\]]*\]: (\d+)s /.exec(t); if (up) c.uptime_s = Number(up[1]); - } else { - // no STATUS line in the tail: count the ACCEPTED lines at least - c.accepted = lines.filter(l => l.includes(' ACCEPTED block')).length; - } - const faults = lines.filter(l => FAULT.test(l)); - c.faults = faults.length; - if (faults.length) c.fault = faults[faults.length - 1].replace(/^\d+(\.\d+)? /, '').slice(0, 200); - const acc = lastMatch(lines, /^(\d+(?:\.\d+)?) ACCEPTED block/); - if (acc) c.last_accepted_at = new Date(Number(acc[1]) * 1000).toISOString(); - return c; -} const HEADER = /IGNEUM-APP version=(\S+) machine=(\S+) platform=(\S+) node=(\S+)/; function parseHeader(text) { const m = HEADER.exec(text || ''); @@ -168,7 +126,7 @@ async function machines(sql) { const now = Date.now(); const out = [...groups.values()].map(g => { const id8 = g.id.startsWith('legacy:') ? null : g.id; - const cards = g.cards.sort((a, b) => a.card - b.card); + const cards = markStale(g.cards.sort((a, b) => a.card - b.card), now); const silent_s = g.last_seen ? Math.round((now - new Date(g.last_seen).getTime()) / 1000) : null; return { id: g.id, id8, platform: g.platform, @@ -181,7 +139,7 @@ async function machines(sql) { peers: g.node.peers ?? (g.app && g.app.status && g.app.status.node_peers != null ? g.app.status.node_peers : null), peers_wanted: g.node.peers_wanted ?? null, peers_from: g.node.peers != null ? 'node log' : (g.app && g.app.status ? 'app status line' : null), blocks: g.app && g.app.status && g.app.status.node_blocks != null ? g.app.status.node_blocks : null, blocks_10s: g.node.blocks_10s ?? null, synced: cards.some(c => c.synced === true) || (g.node.blocks_10s != null && !g.node.ibd_seen) || !!(g.app && g.app.status && g.app.status.node_state === 'synced'), ibd: !!g.node.ibd_seen, last_line_at: g.node.last_line_at || null, warnings: g.node.warnings || 0, last_warning: g.node.last_warning || null, at: g.node.at || (g.app && g.app.status ? g.app.status.at : null) }, - cards, hash_total: Math.round(cards.reduce((a, c) => a + (c.hash_now || 0), 0) * 100) / 100, + cards, hash_total: Math.round(cards.reduce((a, c) => a + (c.stale ? 0 : c.hash_now || 0), 0) * 100) / 100, stale_after_s: STALE_S, accepted_total: cards.reduce((a, c) => a + (c.accepted || 0), 0) || (g.app && g.app.status && g.app.status.accepted_run) || 0, accepted_lifetime: g.app && g.app.status ? g.app.status.accepted_total ?? null : null, faults: cards.reduce((a, c) => a + (c.faults || 0), 0), diff --git a/relay/lib/parse.mjs b/relay/lib/parse.mjs new file mode 100644 index 00000000..1b8dc49a --- /dev/null +++ b/relay/lib/parse.mjs @@ -0,0 +1,63 @@ +// Pure parsers of the console (no dependencies, so `node --test relay/test` covers them): the log-intake labels, +// a miner log's tail, and the stale mark on a card whose last STATUS line is old. +export const kv = (text, k) => { const m = new RegExp(`(?:^|[\\s(])${k}=([^\\s,)]+)`).exec(text); return m ? m[1] : null; }; +export const kvNum = (text, k) => { const v = kv(text, k); return v === null ? null : Number(v); }; +export const lastMatch = (lines, re) => { for (let i = lines.length - 1; i >= 0; i--) { const m = re.exec(lines[i]); if (m) return m; } return null; }; +export const FAULT = /WORKER MISMATCH|worker error|worker exited|panicked|CUDA error|submit error/; + +// A card's hash is "now" only while its worker's STATUS line (every 30 s) is this fresh; older than this the card is +// marked stale and its hash leaves the machine total (4 October 2026: PC 2 showed 117 MH/s "now" from a STATUS line +// four minutes old while the shard prover held the GPU). +export const STALE_S = 120; +export function markStale(cards, now = Date.now()) { + for (const c of cards) { + const age = c.status_at ? Math.round((now - Date.parse(c.status_at)) / 1000) : null; + c.status_age_s = age; + c.stale = age === null || age > STALE_S; + } + return cards; +} + +// Labels the app uploads: nodelog--, miner--- (vendor as the app names the worker: nvidia, amd, +// mac/metal, opencl, and other/intel for the OpenCL fallback on an iGPU; 4 October 2026: PC 37ba0461's other-* worker +// was dropped and the card showed 0 MH/s), - for the app log; the launchers before the app are "legacy". +export function parseLabel(label, machine) { + let m; + if ((m = /^nodelog-(win|mac)-([0-9a-f]{8})$/.exec(label))) return { id: m[2], platform: m[1], stream: 'node' }; + if ((m = /^miner-(nvidia|amd|mac|metal|opencl|other|intel)-([0-9a-f]{8})-(\d+)$/.exec(label))) return { id: m[2], platform: m[1] === 'mac' || m[1] === 'metal' ? 'mac' : 'win', stream: 'miner', vendor: m[1], card: Number(m[3]) }; + if ((m = /^(win|mac)-([0-9a-f]{8})$/.exec(label))) return { id: m[2], platform: m[1], stream: 'app' }; + // the launchers before the app: nodelog-, nvidia-[-n], amd-[-n], igneum-, launcher- + if ((m = /^nodelog-(.+)$/.exec(label))) return { id: 'legacy:' + m[1], platform: 'legacy', stream: 'node' }; + if ((m = /^(nvidia|amd)-(.+?)(?:-(\d+))?$/.exec(label))) return { id: 'legacy:' + m[2], platform: 'legacy', stream: 'miner', vendor: m[1], card: Number(m[3] || 1) }; + if ((m = /^(igneum|launcher)-(.+)$/.exec(label))) return { id: 'legacy:' + m[2], platform: 'legacy', stream: 'app' }; + return null; +} + +export function parseMinerTail(tail) { + const lines = tail.split('\n'); + const c = { hash_now: 0, hash_avg: 0, accepted: 0, rejected: 0, mismatched: 0, restarts: 0, identities: null, synced: null, uptime_s: null, status_at: null, fault: null, faults: 0, template_age_s: null }; + const st = lastMatch(lines, /^(\d+(?:\.\d+)?) STATUS '([^']+)'.*$/); + if (st) { + const t = st[0]; + c.status_at = new Date(Number(st[1]) * 1000).toISOString(); + c.hash_avg = kvNum(t, 'hash') ?? 0; + c.hash_now = kvNum(t, 'now') ?? c.hash_avg; + c.accepted = kvNum(t, 'accepted') ?? 0; + c.rejected = kvNum(t, 'rejected') ?? 0; + c.mismatched = kvNum(t, 'mismatched') ?? 0; + c.restarts = kvNum(t, 'restarts') ?? 0; + c.identities = kvNum(t, 'identities'); + c.synced = kv(t, 'synced') === 'true'; + c.template_age_s = (() => { const v = kv(t, 'template_age'); return v ? Number(v.replace('s', '')) : null; })(); + const up = /STATUS '[^']+' \[[^\]]*\]: (\d+)s /.exec(t); if (up) c.uptime_s = Number(up[1]); + } else { + // no STATUS line in the tail: count the ACCEPTED lines at least + c.accepted = lines.filter(l => l.includes(' ACCEPTED block')).length; + } + const faults = lines.filter(l => FAULT.test(l)); + c.faults = faults.length; + if (faults.length) c.fault = faults[faults.length - 1].replace(/^\d+(\.\d+)? /, '').slice(0, 200); + const acc = lastMatch(lines, /^(\d+(?:\.\d+)?) ACCEPTED block/); + if (acc) c.last_accepted_at = new Date(Number(acc[1]) * 1000).toISOString(); + return c; +} diff --git a/relay/test/parse.test.mjs b/relay/test/parse.test.mjs new file mode 100644 index 00000000..926ebdf4 --- /dev/null +++ b/relay/test/parse.test.mjs @@ -0,0 +1,41 @@ +// node --test relay/test/parse.test.mjs (no dependencies; CI runs it in the site job) +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { parseLabel, parseMinerTail, markStale, STALE_S } from '../lib/parse.mjs'; + +test('labels: every vendor the app names, the app log, the node log, the legacy launchers', () => { + assert.deepEqual(parseLabel('miner-nvidia-ae432dc7-1'), { id: 'ae432dc7', platform: 'win', stream: 'miner', vendor: 'nvidia', card: 1 }); + assert.deepEqual(parseLabel('miner-other-37ba0461-1'), { id: '37ba0461', platform: 'win', stream: 'miner', vendor: 'other', card: 1 }); + assert.deepEqual(parseLabel('miner-intel-37ba0461-2'), { id: '37ba0461', platform: 'win', stream: 'miner', vendor: 'intel', card: 2 }); + assert.deepEqual(parseLabel('miner-mac-d937c69d-1'), { id: 'd937c69d', platform: 'mac', stream: 'miner', vendor: 'mac', card: 1 }); + assert.deepEqual(parseLabel('nodelog-win-1ccfe586'), { id: '1ccfe586', platform: 'win', stream: 'node' }); + assert.deepEqual(parseLabel('win-1ccfe586'), { id: '1ccfe586', platform: 'win', stream: 'app' }); + assert.equal(parseLabel('nodelog-DESKTOP-KMCV30N').platform, 'legacy'); + assert.equal(parseLabel('job-run'), null); +}); + +test('miner tail: the last STATUS line gives the numbers, ACCEPTED lines are counted without one', () => { + const tail = [ + "1791140549.352 STATUS 'other-37ba0461-1' [worker]: 30s jobs=15 accepted=1 rejected=0 mismatched=0 extra=0 rate=0.03 blocks/s hash=1.04 MH/s wall (1.57 MH/s inside jobs) now=1.04 MH/s wall", + '1791140553.360 ACCEPTED block nonce=0xa27b8e5b000a82f6 (gpu worker, cpu re-check ok, identity other-37ba0461-1)', + "1791140579.479 STATUS 'other-37ba0461-1' [worker]: 61s jobs=37 accepted=2 rejected=0 mismatched=0 extra=0 rate=0.02 blocks/s hash=1.28 MH/s wall (1.55 MH/s inside jobs) now=1.53 MH/s wall", + ].join('\n'); + const c = parseMinerTail(tail); + assert.equal(c.accepted, 2); assert.equal(c.hash_now, 1.53); assert.equal(c.hash_avg, 1.28); assert.equal(c.uptime_s, 61); + assert.equal(c.status_at, new Date(1791140579479).toISOString()); + assert.equal(c.faults, 0); + const noStatus = parseMinerTail('1 ACCEPTED block x\n2 ACCEPTED block y\n3 WORKER MISMATCH z'); + assert.equal(noStatus.accepted, 2); assert.equal(noStatus.status_at, null); assert.equal(noStatus.faults, 1); +}); + +test('stale: a card is "now" within STALE_S of its STATUS line and stale after it or without one', () => { + const now = Date.parse('2026-10-04T19:03:00Z'); + const fresh = { status_at: new Date(now - 58_000).toISOString(), hash_now: 18.9 }; + const old = { status_at: new Date(now - 4 * 60_000).toISOString(), hash_now: 117.2 }; + const none = { status_at: null, hash_now: 0 }; + markStale([fresh, old, none], now); + assert.equal(fresh.stale, false); assert.equal(fresh.status_age_s, 58); + assert.equal(old.stale, true); assert.equal(old.status_age_s, 240); + assert.equal(none.stale, true); assert.equal(none.status_age_s, null); + assert.ok(STALE_S >= 90, 'two missed 30 s STATUS lines are not stale yet'); +}); diff --git a/relay/ui.html b/relay/ui.html index af7ec4d0..716c0f77 100644 --- a/relay/ui.html +++ b/relay/ui.html @@ -262,7 +262,7 @@ input[type=text]{width:100%} // ---- machines ---- function cardTable(cards) { if (!cards.length) return ''; - return `
${cards.map(c => ``).join('')}
cardnowacceptedrej/misrestartsstatus
${esc(c.label.replace(/-[0-9a-f]{8}-/, ' '))}${c.identities ? `
${c.identities} ids${c.synced === false ? ', NOT SYNCED' : ''}` : ''}		${mh(c.hash_now)}${nf(c.accepted)}${nf(c.rejected)}/${nf(c.mismatched)}${nf(c.restarts)}${c.status_at ? ago(c.status_at) + ' ago' : 'no STATUS'}${c.fault ? `
${esc(c.fault)}` : ''}
`; + return `
${cards.map(c => ``).join('')}
cardnowacceptedrej/misrestartsstatus
${esc(c.label.replace(/-[0-9a-f]{8}-/, ' '))}${c.identities ? `
${c.identities} ids${c.synced === false ? ', NOT SYNCED' : ''}` : ''}		${c.stale ? `last ${mh(c.hash_now)}` : mh(c.hash_now)}${nf(c.accepted)}${nf(c.rejected)}/${nf(c.mismatched)}${nf(c.restarts)}${c.status_at ? ago(c.status_at) + ' ago' : 'no STATUS'}${c.stale ? ' stale' : ''}${c.fault ? `
${esc(c.fault)}` : ''}
`; } function machineCard(mc) { const n = mc.node || {}; diff --git a/tools/console.mjs b/tools/console.mjs index 8155a22a..c8929007 100644 --- a/tools/console.mjs +++ b/tools/console.mjs @@ -111,7 +111,7 @@ try { const j = await api('machines'); for (const m of j.machines) { console.log(`${m.name.padEnd(8)} ${m.id8 || '-'} ${m.silent ? 'SILENT' : 'live'} seen ${ago(m.last_seen)} ago | app ${m.app_version || '?'} node ${m.node.version || '?'} daa ${m.node.daa ?? '?'} peers ${m.node.peers ?? '?'} ${m.node.synced ? 'synced' : 'not synced'} | ${m.hash_total.toFixed(1)} MH/s, ${m.accepted_total} accepted, ${m.faults} faults`); - for (const c of m.cards) console.log(` ${c.label.padEnd(22)} ${c.hash_now.toFixed(1)} MH/s now, ${c.accepted} accepted, ${c.rejected} rejected, ${c.mismatched} mismatched, ${c.restarts} restarts, status ${ago(c.status_at)} ago${c.fault ? ' | ' + c.fault : ''}`); + for (const c of m.cards) console.log(` ${c.label.padEnd(22)} ${c.stale ? 'last ' + c.hash_now.toFixed(1) + ' MH/s (stale, not in the total)' : c.hash_now.toFixed(1) + ' MH/s now'}, ${c.accepted} accepted, ${c.rejected} rejected, ${c.mismatched} mismatched, ${c.restarts} restarts, status ${ago(c.status_at)} ago${c.fault ? ' | ' + c.fault : ''}`); for (const t of m.telemetry) console.log(` ${t.name}: p95 ${t.p95_w} W, max ${t.max_w} W (cap ${t.cap_w} W), GPU ${t.max_gpu_c} C, memory ${t.max_mem_c} C`); } } diff --git a/tools/jobs.mjs b/tools/jobs.mjs index 0f32418e..d124169a 100755 --- a/tools/jobs.mjs +++ b/tools/jobs.mjs @@ -60,7 +60,7 @@ if (!a) { verified = verify(null, bytes, key, Buffer.from(sig, 'hex')) ? 'signature OK' : 'SIGNATURE DOES NOT VERIFY (the apps refuse this file)'; } const f = JSON.parse(bytes.toString('utf8')); - console.log(`${base}/igneum-jobs.json: published ${f.published_at}, ${f.jobs.length} job(s), ${verified}`); + console.log(`${base.replace(tok, '')}/igneum-jobs.json: published ${f.published_at}, ${f.jobs.length} job(s), ${verified}`); const now = Date.now(); for (const j of f.jobs) { const exp = Date.parse(j.expires_at);