From 942a58a0efe92dac3f07d6beadf9507932497dbb Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:59:07 +0000 Subject: [PATCH] Worker dashboard: collector on igneum-build-1, Mac pusher, workers.html next to the fleet page (the project lead, 6 October 2026) tools/workers/collect.mjs runs every 30 s on the box (infra/build-server/workers/igneum-workers.{service,timer}, install.sh) and writes /srv/workers/workers.json from the machine itself: /proc/stat deltas per core, meminfo, df on /srv, net bytes, hwmon temperatures, the flock state of /srv/builds/_locks, cargo processes with worktree, target and start time, flock waiters, /srv/builds/_log/builds.jsonl (the format agreed with the build-server agent), sccache --show-stats, headline.json. tools/workers/push.mjs (launchd every 60 s) adds the Mac's with-lock slots and waiters and the two PCs' relay job states from the intake, drops them on the box so its file is whole, merges the box's file and writes the fleet folder's workers.json; it deploys only when the live copy is over 6 min old, otherwise the fleet orchestrator's 5-minute deploy carries it. The page (tools/workers/page/workers.html, shape.js) tries https://build.igneum.network/workers.json first and falls back to the folder copy; core strip, arcs, now building, queue, recently done with closing lines, headline timings, analytics; UK time with UTC tooltips; phone width. node --test tools/workers/test: 13 tests over /proc/stat, lock dir, JSONL and sccache fixtures and the page shaping. Co-Authored-By: Claude Fable 5.1 --- .../workers/igneum-workers.service | 21 + .../build-server/workers/igneum-workers.timer | 13 + infra/build-server/workers/install.sh | 14 + tools/workers/collect.mjs | 160 ++++++++ tools/workers/fixtures/builds.jsonl | 6 + tools/workers/fixtures/locks/build-0 | 1 + tools/workers/fixtures/locks/build-1 | 1 + tools/workers/fixtures/locks/build-2 | 0 tools/workers/fixtures/locks/slots | 1 + tools/workers/fixtures/proc-stat-0.txt | 8 + tools/workers/fixtures/proc-stat-1.txt | 8 + tools/workers/fixtures/sccache-stats.txt | 27 ++ tools/workers/fixtures/workers.demo.json | 1 + .../launchd/com.igneum.workers-push.plist | 16 + tools/workers/launchd/install.sh | 19 + tools/workers/lib.mjs | 221 +++++++++++ tools/workers/page/workers.html | 366 ++++++++++++++++++ tools/workers/publish-workers.sh | 35 ++ tools/workers/push.mjs | 205 ++++++++++ tools/workers/shape.js | 99 +++++ tools/workers/test/parsers.test.mjs | 154 ++++++++ 21 files changed, 1376 insertions(+) create mode 100644 infra/build-server/workers/igneum-workers.service create mode 100644 infra/build-server/workers/igneum-workers.timer create mode 100755 infra/build-server/workers/install.sh create mode 100644 tools/workers/collect.mjs create mode 100644 tools/workers/fixtures/builds.jsonl create mode 100644 tools/workers/fixtures/locks/build-0 create mode 100644 tools/workers/fixtures/locks/build-1 create mode 100644 tools/workers/fixtures/locks/build-2 create mode 100644 tools/workers/fixtures/locks/slots create mode 100644 tools/workers/fixtures/proc-stat-0.txt create mode 100644 tools/workers/fixtures/proc-stat-1.txt create mode 100644 tools/workers/fixtures/sccache-stats.txt create mode 100644 tools/workers/fixtures/workers.demo.json create mode 100644 tools/workers/launchd/com.igneum.workers-push.plist create mode 100755 tools/workers/launchd/install.sh create mode 100644 tools/workers/lib.mjs create mode 100644 tools/workers/page/workers.html create mode 100755 tools/workers/publish-workers.sh create mode 100644 tools/workers/push.mjs create mode 100644 tools/workers/shape.js create mode 100644 tools/workers/test/parsers.test.mjs diff --git a/infra/build-server/workers/igneum-workers.service b/infra/build-server/workers/igneum-workers.service new file mode 100644 index 00000000..ce963fec --- /dev/null +++ b/infra/build-server/workers/igneum-workers.service @@ -0,0 +1,21 @@ +# The worker dashboard's collector on igneum-build-1: one pass, written by igneum-workers.timer every 30 s. +# Reads /proc, /sys, /srv/builds/_locks, the process table and /srv/builds/_log/builds.jsonl; writes /srv/workers/workers.json. +# Installed by infra/build-server/workers/install.sh (copies tools/workers/{lib,collect}.mjs to /srv/workers/bin). +[Unit] +Description=Igneum worker dashboard collector (one pass) +After=local-fs.target + +[Service] +Type=oneshot +User=build +Group=build +Environment=HOME=/home/build +Environment=PATH=/home/build/.cargo/bin:/usr/local/bin:/usr/bin:/bin +Environment=SCCACHE_DIR=/srv/sccache +Environment=IGNEUM_WORKERS_DIR=/srv/workers +Environment=IGNEUM_BUILD_SLOTS_DIR=/srv/builds/_locks +Environment=IGNEUM_BUILD_ROOT=/srv/builds +WorkingDirectory=/srv/workers/bin +ExecStart=/usr/local/bin/node /srv/workers/bin/collect.mjs +TimeoutStartSec=25 +Nice=10 diff --git a/infra/build-server/workers/igneum-workers.timer b/infra/build-server/workers/igneum-workers.timer new file mode 100644 index 00000000..a6a6e5cb --- /dev/null +++ b/infra/build-server/workers/igneum-workers.timer @@ -0,0 +1,13 @@ +# Fires the collector every 30 s from boot. `systemctl list-timers igneum-workers.timer` shows the next pass; +# `journalctl -u igneum-workers.service -n 20` the last errors. +[Unit] +Description=Igneum worker dashboard collector, every 30 s + +[Timer] +OnBootSec=20s +OnUnitActiveSec=30s +AccuracySec=1s +Unit=igneum-workers.service + +[Install] +WantedBy=timers.target diff --git a/infra/build-server/workers/install.sh b/infra/build-server/workers/install.sh new file mode 100755 index 00000000..6842091e --- /dev/null +++ b/infra/build-server/workers/install.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Install or refresh the worker dashboard collector on igneum-build-1 from this Mac. +# infra/build-server/workers/install.sh copies tools/workers/{lib,collect}.mjs and the two units, enables the timer +# Needs root over ssh (root@ with ~/.ssh/igneum_ed25519); the host ip comes from ~/.config/igneum/build-server (build@). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../../.." && pwd)" +KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}" +HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')" +IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server" >&2; exit 1; } +SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10) +"${SSH[@]}" "root@$IP" 'install -d -o build -g build /srv/workers /srv/workers/bin /srv/workers/sources /srv/builds/_log; chown build:build /srv/builds/_log' +scp -q -i "$KEY" "$ROOT/tools/workers/lib.mjs" "$ROOT/tools/workers/collect.mjs" "build@$IP:/srv/workers/bin/" +scp -q -i "$KEY" "$HERE/igneum-workers.service" "$HERE/igneum-workers.timer" "root@$IP:/etc/systemd/system/" +"${SSH[@]}" "root@$IP" 'systemctl daemon-reload && systemctl enable --now igneum-workers.timer >/dev/null 2>&1; systemctl start igneum-workers.service; systemctl is-active igneum-workers.timer; systemctl list-timers igneum-workers.timer --no-pager | sed -n 2p; ls -la /srv/workers/workers.json' diff --git a/tools/workers/collect.mjs b/tools/workers/collect.mjs new file mode 100644 index 00000000..03cede18 --- /dev/null +++ b/tools/workers/collect.mjs @@ -0,0 +1,160 @@ +#!/usr/bin/env node +// The worker dashboard's collector. Runs on igneum-build-1 every 30 s (infra/build-server/workers/igneum-workers.timer) +// as the build user and writes /srv/workers/workers.json: what the MACHINE says (uptime, load, /proc/stat deltas per core, +// memory, the /srv disk, network bytes, hwmon temperatures), the slot and lock state under /srv/builds/_locks, the cargo +// processes with their worktree and target, the slot waiters, the recent builds from /srv/builds/_log/builds.jsonl and +// sccache's own counters. Nothing here is a reported number: every field is read from /proc, /sys, the lock files, the +// process table or the log the build itself appended. +// +// node tools/workers/collect.mjs write /srv/workers/workers.json (IGNEUM_WORKERS_DIR overrides) +// node tools/workers/collect.mjs --stdout print instead of writing +// node tools/workers/collect.mjs --sample-ms 1000 the /proc/stat and network delta window (default 1000) +// +// The Mac side (tools/workers/push.mjs) drops its own facts at /srv/workers/sources/mac.json (the Mac's with-lock slots) and +// /srv/workers/sources/pcs.json (the two PCs' relay job states); when present and under 10 minutes old they are merged +// into the same workers.json, so one file answers for the whole shop whichever side serves it. Needs Node 22, no deps. +import { readFileSync, readdirSync, writeFileSync, renameSync, mkdirSync, existsSync, statSync, readlinkSync, openSync, closeSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { hostname } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseProcStat, cpuBusy, parseMeminfo, parseNetDev, parseLoadavg, parseLockDir, parseBuildsJsonl, parseSccacheStats, pickTemps, cargoProcess, deriveKind, BASELINES } from './lib.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const argv = process.argv.slice(2); +const flag = (n, d) => { const i = argv.indexOf(`--${n}`); return i >= 0 ? (argv[i + 1] ?? true) : d; }; +const OUT_DIR = process.env.IGNEUM_WORKERS_DIR || '/srv/workers'; +const LOCKS = process.env.IGNEUM_BUILD_SLOTS_DIR || '/srv/builds/_locks'; +const BUILDS = process.env.IGNEUM_BUILD_ROOT || '/srv/builds'; +const LOG = process.env.IGNEUM_BUILDS_LOG || join(BUILDS, '_log', 'builds.jsonl'); +const SAMPLE_MS = Number(flag('sample-ms', 1000)); +const iso = ms => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z'); +const read = p => { try { return readFileSync(p, 'utf8'); } catch { return ''; } }; +const sh = (cmd, args, opts = {}) => { const r = spawnSync(cmd, args, { encoding: 'utf8', timeout: 8000, ...opts }); return r.status === 0 ? r.stdout : ''; }; + +// ---- machine facts ---------------------------------------------------------------------------------------------------- +function machine() { + const stat0 = parseProcStat(read('/proc/stat')); const net0 = parseNetDev(read('/proc/net/dev')); const t0 = Date.now(); + const end = t0 + SAMPLE_MS; while (Date.now() < end) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, end - Date.now()); + const stat1 = parseProcStat(read('/proc/stat')); const net1 = parseNetDev(read('/proc/net/dev')); const dt = (Date.now() - t0) / 1000; + const cpu = cpuBusy(stat0, stat1); + const up = Number(read('/proc/uptime').split(' ')[0]) || 0; + const la = parseLoadavg(read('/proc/loadavg')); + const mem = parseMeminfo(read('/proc/meminfo')); + const df = sh('df', ['-B1', '--output=target,size,used,avail', BUILDS]).trim().split('\n').pop() || ''; + const dfp = df.trim().split(/\s+/); + const disk = dfp.length >= 4 ? { mount: dfp[0], total_bytes: Number(dfp[1]), used_bytes: Number(dfp[2]), avail_bytes: Number(dfp[3]), used_pct: Number(dfp[1]) ? Math.round((Number(dfp[2]) / Number(dfp[1])) * 1000) / 10 : 0 } : null; + // the busiest interface carries the build traffic (rsync in, artefacts out) + const nets = net1.map(n => { const b = net0.find(x => x.iface === n.iface) || n; return { ...n, rx_bps: Math.max(0, Math.round((n.rx_bytes - b.rx_bytes) * 8 / dt)), tx_bps: Math.max(0, Math.round((n.tx_bytes - b.tx_bytes) * 8 / dt)) }; }).filter(n => n.rx_bytes + n.tx_bytes > 0).sort((a, b) => (b.rx_bytes + b.tx_bytes) - (a.rx_bytes + a.tx_bytes)); + const sensors = []; + for (const h of safeList('/sys/class/hwmon')) { + const dir = join('/sys/class/hwmon', h); const name = read(join(dir, 'name')).trim(); + for (const f of safeList(dir)) { const m = /^(temp\d+)_input$/.exec(f); if (!m) continue; sensors.push({ name, label: read(join(dir, `${m[1]}_label`)).trim() || null, millic: Number(read(join(dir, f))) }); } + } + const osr = read('/etc/os-release'); const pretty = (/^PRETTY_NAME="?([^"\n]+)"?/m.exec(osr) || [])[1] || null; + const clk = Number(sh('getconf', ['CLK_TCK']).trim()) || 100; + return { name: hostname(), os: pretty, cores: stat1.cores.length, uptime_s: Math.round(up), boot_at: iso(Date.now() - up * 1000), ...la, cpu, mem, disk, net: nets[0] || null, temps: pickTemps(sensors), clk, kernel: read('/proc/sys/kernel/osrelease').trim() || null }; +} +const safeList = d => { try { return readdirSync(d); } catch { return []; } }; +const alive = pid => { try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; } }; + +// flock probe without python: `flock -n true` fails while the slot is held (the slot is held by another process's fd) +const flockHeld = p => { const r = spawnSync('flock', ['-n', p, 'true'], { timeout: 3000 }); return r.status !== 0; }; + +// ---- slots, waiters, running builds --------------------------------------------------------------------------------------- +function slots(now) { + const files = safeList(LOCKS).map(n => ({ name: n, text: read(join(LOCKS, n)), held: /^(build-\d+|run-\d+|build|measure)$/.test(n) ? flockHeld(join(LOCKS, n)) : undefined })); + return parseLockDir(files, { now, aliveFn: alive }); +} +function procs(bootSec, clk, now) { + const running = [], waiters = []; let compilers = 0; + for (const d of safeList('/proc')) { + if (!/^\d+$/.test(d)) continue; + const pid = Number(d); const comm = read(`/proc/${d}/comm`).trim(); + if (['rustc', 'cc1plus', 'cc1', 'ld', 'ld.lld', 'x86_64-w64-mingw32-ld'].includes(comm)) compilers++; + if (comm === 'cargo') { + let cwd = null; try { cwd = readlinkSync(`/proc/${d}/cwd`); } catch { cwd = null; } + const p = cargoProcess({ pid, cwd, cmdline: read(`/proc/${d}/cmdline`), stat: read(`/proc/${d}/stat`), bootSec, clk, now, root: BUILDS }); + if (p) running.push(p); + } + if (comm === 'flock') { + const args = read(`/proc/${d}/cmdline`).split('\0').filter(Boolean); + if (args[1] === '-w' && /^\d+$/.test(args[2] || '')) { + const stat = read(`/proc/${d}/stat`); const rest = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + const ppid = Number(rest[1]); const startTicks = Number(rest[19]); + const since = Number.isFinite(startTicks) ? iso((bootSec + startTicks / clk) * 1000) : null; + const wait = read(join(LOCKS, `wait-${ppid}`)).trim(); // a waiter's own line, when the appender writes one + waiters.push({ pid: ppid, since, waiting_s: since ? Math.max(0, Math.round((now - Date.parse(since)) / 1000)) : null, label: wait || null, ...(wait ? { kind: deriveKind(parseLabelSafe(wait)) } : {}) }); + } + } + } + // the innermost cargo per worktree: a `cargo build` spawns no nested cargo, but a `cargo test` runs test binaries, not cargo + running.sort((a, b) => String(a.started_at).localeCompare(String(b.started_at))); + return { running, waiters, compilers }; +} +function parseLabelSafe(line) { const r = /: (.*)$/.exec(line); return { command: r ? r[1] : line }; } + +function sccache() { + const bin = existsSync('/home/build/.cargo/bin/sccache') ? '/home/build/.cargo/bin/sccache' : 'sccache'; + const out = sh(bin, ['--show-stats'], { env: { ...process.env, SCCACHE_DIR: process.env.SCCACHE_DIR || '/srv/sccache' } }); + return out ? parseSccacheStats(out) : null; +} + +// ---- sources dropped by the Mac ------------------------------------------------------------------------------------------ +function source(name, now) { + const p = join(OUT_DIR, 'sources', `${name}.json`); + try { + const st = statSync(p); const age = Math.round((now - st.mtimeMs) / 1000); + const j = JSON.parse(readFileSync(p, 'utf8')); + return { data: j, meta: { at: iso(st.mtimeMs), age_s: age, stale: age > 600, ok: true } }; + } catch (e) { return { data: null, meta: { ok: false, error: e.code === 'ENOENT' ? 'the Mac has not pushed yet' : String(e.message || e) } }; } +} + +// ---- main ------------------------------------------------------------------------------------------------------------------- +export function collect() { + const now = Date.now(); + const m = machine(); + const bootSec = Math.round((now - m.uptime_s * 1000) / 1000); + const s = slots(now); + const p = procs(bootSec, m.clk, now); + // attach the slot holder to its build (the label names the worktree and crate the cargo runs in) + for (const r of p.running) { + const h = s.held.find(h => h.worktree === r.worktree && (!h.crate || h.crate === r.crate || !r.crate)); + r.slot = h ? h.slot : null; r.agent = h ? h.agent : null; r.queued_at = h ? h.since : null; r.waited_s = h ? h.waited_s : null; + } + const logText = read(LOG); + const recent = parseBuildsJsonl(logText, { limit: 200 }); + const mac = source('mac', now), pcs = source('pcs', now); + let headline = null; try { headline = JSON.parse(readFileSync(join(OUT_DIR, 'headline.json'), 'utf8')); } catch { headline = null; } + const doc = { + v: 1, + generated_at: iso(now), + generated_by: 'collect.mjs on ' + m.name, + box: { + ...m, clk: undefined, + collected_at: iso(now), + sccache: sccache(), + slots: { count: s.count, held: s.held, free: s.free }, + running: p.running, + queue: p.waiters, + compilers: p.compilers, + recent: recent.rows, + log: { path: LOG, present: !!logText, bad_lines: recent.bad, total: recent.total, note: logText ? null : 'builds.jsonl is not written yet: the build-server agent adds the append to bs_remote_run; until then this lane fills from nothing' }, + }, + mac: mac.data, pcs: pcs.data, + headline, baselines: BASELINES, + sources: { box: { ok: true, at: iso(now) }, mac: mac.meta, pcs: pcs.meta }, + }; + return doc; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + const doc = collect(); + const text = JSON.stringify(doc); + if (flag('stdout', false)) { process.stdout.write(text + '\n'); } + else { + mkdirSync(OUT_DIR, { recursive: true }); mkdirSync(join(OUT_DIR, 'sources'), { recursive: true }); + const tmp = join(OUT_DIR, '.workers.json.tmp'); writeFileSync(tmp, text); renameSync(tmp, join(OUT_DIR, 'workers.json')); + const fd = openSync(join(OUT_DIR, 'last-run'), 'w'); closeSync(fd); + } +} diff --git a/tools/workers/fixtures/builds.jsonl b/tools/workers/fixtures/builds.jsonl new file mode 100644 index 00000000..b00e98ae --- /dev/null +++ b/tools/workers/fixtures/builds.jsonl @@ -0,0 +1,6 @@ +{"v":1,"id":"igneum-build-1-1759770840-36992","host":"igneum-build-1","tool":"cross-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu -j 90","target":"x86_64-pc-windows-gnu","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:33:58Z","start":"2026-10-06T17:34:00Z","end":"2026-10-06T17:51:12Z","secs":1032,"exit":0,"compiles":1954,"sccache":{"hits":0,"misses":1886,"hits_total":0,"misses_total":1886},"load_end":[13.86,9.61,4.32],"artefacts":[{"path":"target/x86_64-pc-windows-gnu/release/igneumd.exe","bytes":52911104,"sha256":"0f3a9c2e1d8b7a6f5e4d3c2b1a0f9e8d7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f"},{"path":"target/x86_64-pc-windows-gnu/release/igneum-miner.exe","bytes":9811456,"sha256":"1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b"}]} +{"v":1,"id":"igneum-build-1-1759772000-37001","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow -j 90","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","slot":0,"wait_s":0,"start":"2026-10-06T17:53:20Z","end":"2026-10-06T18:06:41Z","secs":801,"exit":0,"compiles":1102,"sccache":{"hits":790,"misses":312,"hits_total":790,"misses_total":2198},"load_end":[61.2,40.1,22.0],"artefacts":[{"path":"target/release/igneumd","bytes":61120512,"sha256":"2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c"},{"path":"target/release/igneum-miner","bytes":11010048,"sha256":"3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d"}]} +{"v":1,"id":"igneum-build-1-1759773000-37100","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ship0314","crate":"app/igneum-app","command":"cargo build --release -j 90","branch":"release-0.3.14","sha":"952ce8f","label":"igneum-wt-ship0314/app/igneum-app cargo build --release","slot":0,"wait_s":412,"queued_at":"2026-10-06T18:03:10Z","start":"2026-10-06T18:10:02Z","end":"2026-10-06T18:13:48Z","secs":226,"exit":0,"compiles":388,"sccache":{"hits":301,"misses":87,"hits_total":1091,"misses_total":2285},"load_end":[70.3,55.8,30.2],"artefacts":[{"path":"target/release/igneum-app","bytes":30801920,"sha256":"4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e"}]} +{"v":1,"id":"igneum-build-1-1759773600-37200","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ca3-v4-node","crate":"vendor/igneum-node-v4","command":"cargo test --release -p kaspa-consensus-core --lib -j 90","branch":"ca3-v4-node","sha":"f39a8eb","label":"igneum-wt-ca3-v4-node/vendor/igneum-node-v4 cargo test --release -p kaspa-consensus-core --lib","slot":0,"wait_s":0,"start":"2026-10-06T18:14:00Z","end":"2026-10-06T18:19:33Z","secs":333,"exit":101,"compiles":210,"sccache":{"hits":180,"misses":30,"hits_total":1271,"misses_total":2315},"load_end":[55.0,50.2,33.1],"artefacts":[]} +not json at all +{"v":1,"id":"igneum-build-1-1759774200-37300","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-proving-v2","crate":"proving/igneum-prove","command":"cargo build --release -j 90","branch":"proving-v2","sha":"621f258","label":"igneum-wt-proving-v2/proving/igneum-prove cargo build --release","slot":0,"start":"2026-10-06T18:21:00Z","end":"2026-10-06T18:29:10Z","secs":490,"exit":0,"compiles":640,"sccache":{"hits":600,"misses":40,"hits_total":1871,"misses_total":2355},"load_end":[60.1,52.0,38.4],"artefacts":[{"path":"target/release/igneum-prove-host","bytes":80000000,"sha256":"5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f"}]} diff --git a/tools/workers/fixtures/locks/build-0 b/tools/workers/fixtures/locks/build-0 new file mode 100644 index 00000000..65a76e6a --- /dev/null +++ b/tools/workers/fixtures/locks/build-0 @@ -0,0 +1 @@ +pid 36992 since 17:34:00Z waited 0 s: igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server diff --git a/tools/workers/fixtures/locks/build-1 b/tools/workers/fixtures/locks/build-1 new file mode 100644 index 00000000..144af4e9 --- /dev/null +++ b/tools/workers/fixtures/locks/build-1 @@ -0,0 +1 @@ +pid 1 since 09:00:00Z waited 3 s: igneum-wt-old/app/igneum-app cargo build --release diff --git a/tools/workers/fixtures/locks/build-2 b/tools/workers/fixtures/locks/build-2 new file mode 100644 index 00000000..e69de29b diff --git a/tools/workers/fixtures/locks/slots b/tools/workers/fixtures/locks/slots new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/tools/workers/fixtures/locks/slots @@ -0,0 +1 @@ +1 diff --git a/tools/workers/fixtures/proc-stat-0.txt b/tools/workers/fixtures/proc-stat-0.txt new file mode 100644 index 00000000..a2e7c285 --- /dev/null +++ b/tools/workers/fixtures/proc-stat-0.txt @@ -0,0 +1,8 @@ +cpu 513633 175 77715 5794769 1804 0 128 0 0 0 +cpu0 10000 0 1000 50000 10 0 1 0 0 0 +cpu1 20000 0 2000 40000 10 0 1 0 0 0 +cpu2 100 0 10 60000 10 0 1 0 0 0 +cpu3 30000 0 500 30000 10 0 1 0 0 0 +intr 1 2 3 +ctxt 99 +btime 1759772000 diff --git a/tools/workers/fixtures/proc-stat-1.txt b/tools/workers/fixtures/proc-stat-1.txt new file mode 100644 index 00000000..cf0b599a --- /dev/null +++ b/tools/workers/fixtures/proc-stat-1.txt @@ -0,0 +1,8 @@ +cpu 514633 175 77915 5795569 1804 0 128 0 0 0 +cpu0 10100 0 1000 50000 10 0 1 0 0 0 +cpu1 20000 0 2000 40100 10 0 1 0 0 0 +cpu2 150 0 60 60000 10 0 1 0 0 0 +cpu3 30025 0 500 30075 10 0 1 0 0 0 +intr 1 2 3 +ctxt 99 +btime 1759772000 diff --git a/tools/workers/fixtures/sccache-stats.txt b/tools/workers/fixtures/sccache-stats.txt new file mode 100644 index 00000000..b31ae8b5 --- /dev/null +++ b/tools/workers/fixtures/sccache-stats.txt @@ -0,0 +1,27 @@ +Compile requests 2294 +Compile requests executed 1954 +Cache hits 790 +Cache misses 1886 +Cache misses (Assembler) 27 +Cache misses (C/C++) 777 +Cache misses (Rust) 1082 +Cache hits rate 29.52 % +Cache timeouts 0 +Cache read errors 0 +Forced recaches 0 +Cache write errors 0 +Compilation failures 0 +Cache errors 0 +Non-cacheable compilations 0 +Non-cacheable calls 340 +Non-compilation calls 0 +Unsupported compiler calls 0 +Average cache write 0.002 s +Average compiler 1.624 s +Average cache read hit 0.001 s +Failed distributed compilations 0 +Cache location Local disk: "/srv/sccache" +Use direct/preprocessor mode? yes +Version (client) 0.10.0 +Cache size 12 GiB +Max cache size 100 GiB diff --git a/tools/workers/fixtures/workers.demo.json b/tools/workers/fixtures/workers.demo.json new file mode 100644 index 00000000..f959d092 --- /dev/null +++ b/tools/workers/fixtures/workers.demo.json @@ -0,0 +1 @@ +{"v":1,"generated_at":"2026-10-06T17:49:21Z","generated_by":"demo document (fixtures over a live snapshot)","box":{"name":"igneum-build-1","os":"Ubuntu 24.04.5 LTS","cores":96,"uptime_s":1518,"boot_at":"2026-10-06T17:23:37Z","load":[68.2,41.3,19.7],"running":[{"pid":37210,"cwd":"/srv/builds/igneum-wt-ship0314/vendor/igneum-node-0314","worktree":"igneum-wt-ship0314","crate":"vendor/igneum-node-0314","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu -j 90","target":"x86_64-pc-windows-gnu","kind":"node-windows","started_at":"2026-10-06T17:42:31Z","elapsed_s":410,"slot":"build-0","agent":"release-0.3.14"}],"threads":1229,"cpu":{"busy_pct":71.4,"per_core":[55,63,71,12,84,89,92,94,95,94,12,87,82,76,68,61,57,12,73,79,85,90,93,95,12,93,90,86,80,74,66,12,60,67,75,81,87,91,12,95,95,93,89,84,78,12,64,56,62,70,77,83,12,92,94,95,94,92,88,12,76,69,62,56,64,72,12,85,89,93,95,95,94,12,87,81,74,67,59,59,12,74,81,86,91,93,95,12,93,90,85,79,72,65,12,61],"cores_busy":82},"mem":{"total_kb":131556268,"available_kb":122888768,"used_kb":8667500,"used_pct":6.6,"swap_total_kb":0,"swap_free_kb":0},"disk":{"mount":"/","total_bytes":3777582440448,"used_bytes":17311846400,"avail_bytes":3568303419392,"used_pct":0.5},"net":{"iface":"enp193s0f0np0","rx_bytes":1000335434,"tx_bytes":321279277,"rx_bps":528,"tx_bps":0},"temps":[{"name":"cpu","label":"Tctl","c":63},{"name":"nvme","label":"Sensor 2","c":46.9},{"name":"nic","label":"bnxt_en","c":54}],"kernel":"6.8.0-139-generic","collected_at":"2026-10-06T17:48:54Z","sccache":{"requests":4416,"executed":3862,"hits":864,"misses":2977,"hit_rate_pct":22.5,"cache_size":"2 GiB","max_size":"100 GiB"},"slots":{"count":1,"held":[{"slot":"build-0","pid":36992,"since":"2026-10-06T17:42:29Z","waited_s":0,"label":"igneum-wt-ship0314/vendor/igneum-node-0314 cross x86_64-pc-windows-gnu; agent=release-0.3.14","agent":"release-0.3.14","worktree":"igneum-wt-ship0314","crate":"vendor/igneum-node-0314","command":"cargo build --release --target x86_64-pc-windows-gnu","target":"x86_64-pc-windows-gnu","kind":"node-windows"}],"free":[]},"queue":[{"pid":38001,"since":"2026-10-06T17:47:46Z","waiting_s":95,"label":"pid 38001 since 00:00:00Z waited 0 s: igneum-wt-ca3-v4-node/vendor/igneum-node-v4 cargo test --release -p kaspa-consensus-core --lib","kind":"suite"}],"compilers":86,"recent":[{"v":1,"id":"igneum-build-1-1759774200-37300","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-proving-v2","crate":"proving/igneum-prove","command":"cargo build --release -j 90","branch":"proving-v2","sha":"621f258","label":"igneum-wt-proving-v2/proving/igneum-prove cargo build --release","slot":0,"start":"2026-10-06T17:11:11Z","end":"2026-10-06T17:19:21Z","secs":490,"exit":0,"compiles":640,"sccache":{"hits":600,"misses":40,"hit_rate_pct":93.8,"hits_total":1871,"misses_total":2355},"load_end":[60.1,52,38.4],"artefacts":[{"path":"target/release/igneum-prove-host","bytes":80000000,"sha256":"5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f"}],"kind":"prove","target":"x86_64-unknown-linux-gnu","ok":true},{"v":1,"id":"igneum-build-1-1759773600-37200","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ca3-v4-node","crate":"vendor/igneum-node-v4","command":"cargo test --release -p kaspa-consensus-core --lib -j 90","branch":"ca3-v4-node","sha":"f39a8eb","label":"igneum-wt-ca3-v4-node/vendor/igneum-node-v4 cargo test --release -p kaspa-consensus-core --lib","slot":0,"wait_s":0,"start":"2026-10-06T17:04:11Z","end":"2026-10-06T17:09:44Z","secs":333,"exit":101,"compiles":210,"sccache":{"hits":180,"misses":30,"hit_rate_pct":85.7,"hits_total":1271,"misses_total":2315},"load_end":[55,50.2,33.1],"artefacts":[],"kind":"suite","target":"x86_64-unknown-linux-gnu","ok":false},{"v":1,"id":"igneum-build-1-1759773000-37100","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ship0314","crate":"app/igneum-app","command":"cargo build --release -j 90","branch":"release-0.3.14","sha":"952ce8f","label":"igneum-wt-ship0314/app/igneum-app cargo build --release","slot":0,"wait_s":412,"queued_at":"2026-10-06T18:03:10Z","start":"2026-10-06T17:00:13Z","end":"2026-10-06T17:03:59Z","secs":226,"exit":0,"compiles":388,"sccache":{"hits":301,"misses":87,"hit_rate_pct":77.6,"hits_total":1091,"misses_total":2285},"load_end":[70.3,55.8,30.2],"artefacts":[{"path":"target/release/igneum-app","bytes":30801920,"sha256":"4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e"}],"kind":"app","target":"x86_64-unknown-linux-gnu","ok":true},{"v":1,"id":"igneum-build-1-1759772000-37001","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow -j 90","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","slot":0,"wait_s":0,"start":"2026-10-06T16:43:31Z","end":"2026-10-06T16:56:52Z","secs":801,"exit":0,"compiles":1102,"sccache":{"hits":790,"misses":312,"hit_rate_pct":71.7,"hits_total":790,"misses_total":2198},"load_end":[61.2,40.1,22],"artefacts":[{"path":"target/release/igneumd","bytes":61120512,"sha256":"2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c"},{"path":"target/release/igneum-miner","bytes":11010048,"sha256":"3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d"}],"kind":"node-linux","target":"x86_64-unknown-linux-gnu","ok":true},{"v":1,"id":"igneum-build-1-1759770840-36992","host":"igneum-build-1","tool":"cross-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu -j 90","target":"x86_64-pc-windows-gnu","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:33:58Z","start":"2026-10-06T16:24:11Z","end":"2026-10-06T16:41:23Z","secs":1032,"exit":0,"compiles":1954,"sccache":{"hits":0,"misses":1886,"hit_rate_pct":0,"hits_total":0,"misses_total":1886},"load_end":[13.86,9.61,4.32],"artefacts":[{"path":"target/x86_64-pc-windows-gnu/release/igneumd.exe","bytes":52911104,"sha256":"0f3a9c2e1d8b7a6f5e4d3c2b1a0f9e8d7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f"},{"path":"target/x86_64-pc-windows-gnu/release/igneum-miner.exe","bytes":9811456,"sha256":"1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b"}],"kind":"node-windows","ok":true},{"v":1,"id":"igneum-build-1-1791308825-86828","host":"igneum-build-1","tool":"cross-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","kind":"node-windows","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu","target":"x86_64-pc-windows-gnu","branch":"detached-3bfe346f","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:47:05Z","start":"2026-10-06T17:47:05Z","end":"2026-10-06T17:48:46Z","secs":101,"exit":0,"compiles":867,"sccache":{"hits":259,"misses":605,"hit_rate_pct":30,"hits_total":863,"misses_total":2977},"load_end":[28.91,15.83,10.29],"artefacts":[{"path":"target/x86_64-pc-windows-gnu/release/igneumd.exe","bytes":49436672,"sha256":"c38b7570fbe49fad6c3437efdca2571734f7bc1fe79731492611f0b5448274f2"},{"path":"target/x86_64-pc-windows-gnu/release/igneum-miner.exe","bytes":10201600,"sha256":"c3a0fc2be734cb61f1afe71ad01f44103ef6651ed2092e9a1a63a839d0749e77"}],"ok":true},{"v":1,"id":"igneum-build-1-1791308605-67585","host":"igneum-build-1","tool":"build-remote","worktree":"igneum","crate":"vendor/igneum-node-v4","kind":"node-linux","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","target":"x86_64-unknown-linux-gnu","branch":"devnet-v4","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum/vendor/igneum-node-v4 cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:43:25Z","start":"2026-10-06T17:43:25Z","end":"2026-10-06T17:44:44Z","secs":79,"exit":0,"compiles":999,"sccache":{"hits":604,"misses":389,"hit_rate_pct":60.8,"hits_total":604,"misses_total":2372},"load_end":[22.32,13.04,8.65],"artefacts":[{"path":"target/release/igneumd","bytes":48222304,"sha256":"f236e6cf9d5b92a9253c30504a83293c0d05b095d6a963168fc95e86d2a3285a"},{"path":"target/release/igneum-miner","bytes":9106976,"sha256":"6b580e6401523b6a77815dd2c0e5b63bcf36392c8a59c5af5133d7373ac7fd5c"}],"ok":true},{"v":1,"id":"igneum-build-1-1791308425-64707","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","kind":"node-linux","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","target":"x86_64-unknown-linux-gnu","branch":"detached-3bfe346f","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:40:25Z","start":"2026-10-06T17:40:25Z","end":"2026-10-06T17:40:35Z","secs":10,"exit":0,"compiles":1,"sccache":{"hits":0,"misses":1,"hit_rate_pct":0,"hits_total":0,"misses_total":1983},"load_end":[16.06,12.32,7.7],"artefacts":[{"path":"target/release/igneumd","bytes":48221920,"sha256":"c35899ee6b1e95abf72c522be693a62c8f576b83aa94e59f3a0a9e147a029f2e"},{"path":"target/release/igneum-miner","bytes":9107232,"sha256":"c752893f78e1d284daa2f9e389fed266aaa7d5497edab299a93d1537dc0bb075"}],"ok":true},{"v":1,"id":"igneum-build-1-1791308374-62696","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","kind":"node-linux","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","target":"x86_64-unknown-linux-gnu","branch":"detached-3bfe346f","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:39:34Z","start":"2026-10-06T17:39:34Z","end":"2026-10-06T17:39:41Z","secs":7,"exit":0,"compiles":0,"sccache":{"hits":0,"misses":0,"hit_rate_pct":null,"hits_total":0,"misses_total":1982},"load_end":[15.75,12.32,7.45],"artefacts":[{"path":"target/release/igneumd","bytes":48220896,"sha256":"f9e0a4d198658b34b3d530e2f8395b3de1a96ebc928553a8d709d2b14cbcdfb0"},{"path":"target/release/igneum-miner","bytes":9107232,"sha256":"c752893f78e1d284daa2f9e389fed266aaa7d5497edab299a93d1537dc0bb075"}],"ok":true},{"v":1,"id":"igneum-build-1-1791308317-60832","host":"igneum-build-1","tool":"cross-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","kind":"node-windows","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu","target":"x86_64-pc-windows-gnu","branch":"detached-3bfe346f","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:38:37Z","start":"2026-10-06T17:38:37Z","end":"2026-10-06T17:38:46Z","secs":9,"exit":0,"compiles":0,"sccache":{"hits":0,"misses":0,"hit_rate_pct":null,"hits_total":0,"misses_total":1982},"load_end":[18.57,12.67,7.3],"artefacts":[{"path":"target/x86_64-pc-windows-gnu/release/igneumd.exe","bytes":49434624,"sha256":"bfbff01565f21408e4d5df80a02dc979684839497517b4c84b50d0e8bf5da715"},{"path":"target/x86_64-pc-windows-gnu/release/igneum-miner.exe","bytes":10201600,"sha256":"5812b74a3eb0672e9872e00f58b9c517e0a0861cef6ac0854598971fe556e8bb"}],"ok":true},{"v":1,"id":"igneum-build-1-1791308306-58965","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","kind":"node-linux","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","target":"x86_64-unknown-linux-gnu","branch":"detached-3bfe346f","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:38:26Z","start":"2026-10-06T17:38:26Z","end":"2026-10-06T17:38:33Z","secs":7,"exit":0,"compiles":0,"sccache":{"hits":0,"misses":0,"hit_rate_pct":null,"hits_total":0,"misses_total":1982},"load_end":[9.75,10.89,6.68],"artefacts":[{"path":"target/release/igneumd","bytes":48220896,"sha256":"f9e0a4d198658b34b3d530e2f8395b3de1a96ebc928553a8d709d2b14cbcdfb0"},{"path":"target/release/igneum-miner","bytes":9107232,"sha256":"c752893f78e1d284daa2f9e389fed266aaa7d5497edab299a93d1537dc0bb075"}],"ok":true}],"log":{"path":"/srv/builds/_log/builds.jsonl","present":true,"bad_lines":0,"total":6,"note":null}},"mac":{"name":"MacBook-Pro","collected_at":"2026-10-06T17:49:20Z","build_slots":3,"slots":{"count":3,"held":[{"slot":"build-0","pid":4242,"since":"2026-10-06T17:27:41Z","waited_s":0,"label":"cargo build --release -p igneum-app","agent":null,"worktree":"igneum-wt-ship0314","crate":null,"command":"cargo build --release -p igneum-app","target":"x86_64-unknown-linux-gnu","kind":"app"}],"free":["build","build-0","build-1","build-2","measure","run-0","run-1","run-2"]},"queue":[{"pid":4300,"mode":"measure","waiting_s":240,"command":"node infra/gpu-bench/hashrate.mjs --secs 120","kind":"other"}]},"pcs":{"collected_at":"2026-10-06T17:49:21Z","machines":[{"id":"ae432dc7","name":"PC 1","role":"builds","note":"Linux and Windows binaries, the project lead's desk","machine":"DESKTOP-KMCV30N-ae432dc7","running":null,"queue":[],"recent":[{"job":"run-ca3-pc1-4070-shadow-20261006","kind":"run","title":"CA3 PC 1: the shadow ladder on the RTX 4070","status":"done","ok":true,"exit":0,"secs":725,"started_at":"2026-10-06T17:36:05Z","finished_at":"2026-10-06T17:48:10Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-family-20261006-e","kind":"run","title":"CA3 PC 1: family step costs on the 9070 XT (run e, the card alone)","status":"done","ok":true,"exit":0,"secs":170,"started_at":"2026-10-06T17:31:57Z","finished_at":"2026-10-06T17:34:47Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-derive-20261006","kind":"run","title":"CA3 PC 1: dr736 build, compile and fingerprints on the 9070 XT","status":"done","ok":true,"exit":0,"secs":317,"started_at":"2026-10-06T17:23:56Z","finished_at":"2026-10-06T17:29:13Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-g2-20261006","kind":"run","title":"CA3 PC 1: gate G2 on the 9070 XT","status":"done","ok":true,"exit":0,"secs":4,"started_at":"2026-10-06T17:22:33Z","finished_at":"2026-10-06T17:22:37Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"fetch-ca3-pc1-amd-g2-20261006","kind":"fetch","title":"CA3 PC 1 G2 kit","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T17:21:33Z","finished_at":"2026-10-06T17:21:33Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-pc1-amd-g2-20261006\\igneum-ca3-pc1-amd-g2-kit.zip (198597 bytes, sha256 ok), extracted into C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-pc1-amd-g2-20261006","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-family-20261006-d","kind":"run","title":"CA3 PC 1: family step costs on the 9070 XT (run d, by name, fixed splat)","status":"done","ok":true,"exit":0,"secs":161,"started_at":"2026-10-06T17:17:34Z","finished_at":"2026-10-06T17:20:15Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-family-20261006-b","kind":"run","title":"CA3 PC 1: family step costs on the 9070 XT (run b, by name)","status":"failed","ok":false,"exit":1,"secs":26,"started_at":"2026-10-06T17:10:18Z","finished_at":"2026-10-06T17:10:44Z","summary":"script exited with code 1","errors":[],"stage":null},{"job":"fetch-ca3-pc1-amd-family-20261006","kind":"fetch","title":"CA3 PC 1 family kit","status":"done","ok":true,"exit":0,"secs":1,"started_at":"2026-10-06T17:09:15Z","finished_at":"2026-10-06T17:09:16Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-pc1-amd-family-20261006\\igneum-ca3-pc1-amd-family-kit.zip (26007 bytes, sha256 ok), extracted into C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-pc1-amd-family-20261006","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-identities-20261006-c","kind":"run","title":"CA3 PC 1: the 9070 XT back to 8 identities (c)","status":"done","ok":true,"exit":0,"secs":120,"started_at":"2026-10-06T17:05:53Z","finished_at":"2026-10-06T17:07:53Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-identities-20261006-b","kind":"run","title":"CA3 PC 1: the 9070 XT back to 8 identities (b)","status":"failed","ok":false,"exit":1,"secs":1,"started_at":"2026-10-06T17:04:10Z","finished_at":"2026-10-06T17:04:11Z","summary":"script exited with code 1","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-identities-20261006","kind":"run","title":"CA3 PC 1: the 9070 XT back to 8 identities","status":"failed","ok":false,"exit":1,"secs":0,"started_at":"2026-10-06T17:02:10Z","finished_at":"2026-10-06T17:02:10Z","summary":"script exited with code 1","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-family-20261006","kind":"run","title":"CA3 PC 1: reserve-family step costs on the 9070 XT","status":"done","ok":true,"exit":0,"secs":150,"started_at":"2026-10-06T16:56:44Z","finished_at":"2026-10-06T16:59:14Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-reset-20261006","kind":"run","title":"CA3 PC 1: 9070 XT ADLX back to factory, identities read","status":"done","ok":true,"exit":0,"secs":4,"started_at":"2026-10-06T16:55:16Z","finished_at":"2026-10-06T16:55:20Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"ember-proof-pc1-2","kind":"run","title":"Ember proof (reads only): power cap lines, helper log, task action, nvidia-smi, UAC count, card state","status":"done","ok":true,"exit":0,"secs":1,"started_at":"2026-10-06T16:53:02Z","finished_at":"2026-10-06T16:53:03Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"ember-proof-collect-1","kind":"collect","title":"Ember proof: the installed app's power cap lines, the helper log, the task action, nvidia-smi limits","status":"failed","ok":false,"exit":1,"secs":0,"started_at":"2026-10-06T16:51:19Z","finished_at":"2026-10-06T16:51:19Z","summary":"0 files uploaded, command exited with code 1, output in the report","errors":[],"stage":null},{"job":"ember-repoint-pc1-1","kind":"run","title":"Ember: re-point the Power Helper at the installed exe (kit-7 reregister verb) and prove the no-prompt cap","status":"done","ok":true,"exit":0,"secs":73,"started_at":"2026-10-06T16:48:31Z","finished_at":"2026-10-06T16:49:44Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"ember-kit-7","kind":"fetch","title":"Ember kit-7 (185c333: reregister verb, task_exe fix, 45% floor, AMD apply on ack)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T16:47:27Z","finished_at":"2026-10-06T16:47:27Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\ember-kit-7\\igneum-app-ember.exe (3334144 bytes, sha256 ok)","errors":[],"stage":null},{"job":"build-20261006-164418","kind":"build","title":"ember-tune bd7fcf4: Windows app build (kit-7)","status":"done","ok":true,"exit":0,"secs":103,"started_at":"2026-10-06T16:44:56Z","finished_at":"2026-10-06T16:46:39Z","summary":"node devnet-v4 3bfe346f app 0.3.12: every stage ok; 6 files uploaded (27 MB); 1 min of 40","errors":[],"stage":"pack"},{"job":"ember-tune-pc1-6","kind":"run","title":"Ember Tune table run on PC 1 (0.3.13, kit-6): 5090 + 4070 two-knob tune, 9070 XT power ladder, one administrator prompt (registers the Power Helper on the installed exe)","status":"done","ok":true,"exit":0,"secs":2317,"started_at":"2026-10-06T16:01:00Z","finished_at":"2026-10-06T16:39:37Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"ember-kit-6","kind":"fetch","title":"Ember Tune engine (ember-tune 564bdea: the cmd quote fix, the self-registering elevated engine) for the table run","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T15:59:33Z","finished_at":"2026-10-06T15:59:33Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\ember-kit-6\\igneum-app-ember.exe (3354112 bytes, sha256 ok)","errors":[],"stage":null}],"last_report_at":"2026-10-06T17:48:10Z"},{"id":"1ccfe586","name":"PC 2","role":"test suites","note":"cargo test suites and shard runs","machine":"DESKTOP-KMCV30N-1ccfe586","running":null,"queue":[{"job":"build-20261006-174823","kind":"build","title":"Counter ASIC 3.0 G6 (signalling) 3 of 3: the five crates and the app on fork 0562a7f2","expires_at":"2026-10-08T17:48:24Z"}],"recent":[{"job":"build-20261006-174027","kind":"build","title":"Counter ASIC 3.0 G6 (signalling) 2 of 3: kaspa-consensus alone on fork 0562a7f2 (the ban_is_decided flake re-run, the 2.0 rule)","status":"done","ok":true,"exit":0,"secs":345,"started_at":"2026-10-06T17:41:02Z","finished_at":"2026-10-06T17:46:47Z","summary":"node ca3-v4-node 0562a7f2 app : every stage ok; 7 files uploaded (44 MB); 5 min of 40","errors":[],"stage":"pack"},{"job":"build-20261006-173017","kind":"build","title":"Counter ASIC 3.0 G6 (signalling): node suites on fork ca3-v4-node 0562a7f2 (main f39a8eb)","status":"failed","ok":false,"exit":1,"secs":385,"started_at":"2026-10-06T17:32:15Z","finished_at":"2026-10-06T17:38:40Z","summary":"node ca3-v4-node 0562a7f2 app 0.3.11: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40","errors":["test processes::finality::tests::ban_is_decided_by_the_carrying_block_so_nodes_agree_on_every_voter_list ... FAILED","thread 'processes::finality::tests::ban_is_decided_by_the_carrying_block_so_nodes_agree_on_every_voter_list' (20805) panicked at consensus/src/processes/finality.rs:1914:82:","test result: FAILED. 97 passed; 1 failed; 3 ignored; 0 measured; 0 filtered out; finished in 4.26s","RESULT build 2026-10-06T17:38:40Z failed node ca3-v4-node 0562a7f2 app 0.3.11: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40","job build-20261006-173017: failed (exit 1) after 385 s: node ca3-v4-node 0562a7f2 app 0.3.11: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40"],"stage":"pack"},{"job":"build-20261006-171423","kind":"build","title":"build node and app (release-0.3.14-node 4c6b129d)","status":"failed","ok":false,"exit":1,"secs":622,"started_at":"2026-10-06T17:15:09Z","finished_at":"2026-10-06T17:25:31Z","summary":"node release-0.3.14-node 4c6b129d app : FAILED: upload incomplete; 7 files uploaded (44 MB); 10 min of 40","errors":["RESULT upload 2026-10-06T17:25:31Z build-outputs.json FAILED: blob PUT: no url in the reply: {\"error\":{\"code\":\"service_unavailable\",\"message\":\"Blob service is currently unavailable. Please try again.\"","RESULT build 2026-10-06T17:25:31Z failed node release-0.3.14-node 4c6b129d app : FAILED: upload incomplete; 7 files uploaded (44 MB); 10 min of 40","job build-20261006-171423: failed (exit 1) after 622 s: node release-0.3.14-node 4c6b129d app : FAILED: upload incomplete; 7 files uploaded (44 MB); 10 min of 40"],"stage":"pack"},{"job":"build-20261006-165701","kind":"build","title":"build node and app (release-0.3.14-node 4c6b129d)","status":"failed","ok":false,"exit":1,"secs":387,"started_at":"2026-10-06T16:57:31Z","finished_at":"2026-10-06T17:03:58Z","summary":"node release-0.3.14-node 4c6b129d app 0.3.14: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40","errors":["test processes::finality::tests::ban_is_decided_by_the_carrying_block_so_nodes_agree_on_every_voter_list ... FAILED","thread 'processes::finality::tests::ban_is_decided_by_the_carrying_block_so_nodes_agree_on_every_voter_list' (20838) panicked at consensus/src/processes/finality.rs:1914:82:","test result: FAILED. 97 passed; 1 failed; 3 ignored; 0 measured; 0 filtered out; finished in 4.22s","RESULT build 2026-10-06T17:03:58Z failed node release-0.3.14-node 4c6b129d app 0.3.14: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40","job build-20261006-165701: failed (exit 1) after 387 s: node release-0.3.14-node 4c6b129d app 0.3.14: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40"],"stage":"pack"},{"job":"collect-ca3-v4-g1only-20261006","kind":"collect","title":"Counter ASIC 3.0 gate run (hash): the header, G1 lines and two packs G2 lines of run-ca3-v4-gates-pc2-20261006 (PC 2)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T16:12:06Z","finished_at":"2026-10-06T16:12:06Z","summary":"0 files uploaded, command exit 0, output in the report","errors":[],"stage":null},{"job":"collect-ca3-v4-g1lines-20261006","kind":"collect","title":"Counter ASIC 3.0 gate run (hash): the G1 lines and four packs G2 lines of run-ca3-v4-gates-pc2-20261006 from its job.log (PC 2)","status":"done","ok":true,"exit":0,"secs":1,"started_at":"2026-10-06T16:09:59Z","finished_at":"2026-10-06T16:10:00Z","summary":"0 files uploaded, command exit 0, output in the report","errors":[],"stage":null},{"job":"collect-ca3-v4-joblog2-20261006","kind":"collect","title":"Counter ASIC 3.0 gate run (hash): the full job.log of run-ca3-v4-gates-pc2-20261006, app/jobs path (PC 2)","status":"done","ok":true,"exit":0,"secs":1,"started_at":"2026-10-06T16:08:37Z","finished_at":"2026-10-06T16:08:38Z","summary":"1 file uploaded","errors":[],"stage":null},{"job":"build-20261006-155958","kind":"build","title":"Counter ASIC 3.0 G6: node suites on fork ca3-v4-node 5f7e0543 (main a522d04)","status":"done","ok":true,"exit":0,"secs":392,"started_at":"2026-10-06T16:01:02Z","finished_at":"2026-10-06T16:07:34Z","summary":"node ca3-v4-node 5f7e0543 app 0.3.11: every stage ok; 9 files uploaded (46 MB); 6 min of 40","errors":[],"stage":"pack"},{"job":"collect-ca3-v4-joblog-20261006","kind":"collect","title":"Counter ASIC 3.0 gate run (hash): the full job.log of run-ca3-v4-gates-pc2-20261006 (PC 2)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T15:57:16Z","finished_at":"2026-10-06T15:57:16Z","summary":"0 files uploaded","errors":[],"stage":null},{"job":"run-ca3-v4-gates-pc2-20261006","kind":"run","title":"Counter ASIC 3.0 gate run (hash): G1 and G2 on the RTX 5090 for mx8+sh256x27 (PC 2)","status":"done","ok":true,"exit":0,"secs":26,"started_at":"2026-10-06T15:52:23Z","finished_at":"2026-10-06T15:52:49Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"fetch-ca3-v4-20261006","kind":"fetch","title":"Counter ASIC 3.0 gate run (hash): the class v4 packs (PC 2)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T15:52:21Z","finished_at":"2026-10-06T15:52:21Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-v4-20261006\\fetch-ca3-v4-20261006 (497673 bytes, sha256 ok), extracted into C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-v4-20261006","errors":[],"stage":null},{"job":"update-now-0313-switch-1ccfe586","kind":"update-now","title":"Exec restart: re-read the manifest (exec_restart_number 27276, exec_restart_hash, exec_restart_trust_daa 200000) and restart the node with it","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T15:37:01Z","finished_at":"2026-10-06T15:37:01Z","summary":"update check and install asked","errors":[],"stage":null},{"job":"update-now-0313-1ccfe586","kind":"update-now","title":"0.3.13 is published: re-read the manifest and install now (the execution layer follows again, the finality route fix)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T15:25:54Z","finished_at":"2026-10-06T15:25:54Z","summary":"update check and install asked","errors":[],"stage":null},{"job":"floor-build-7","kind":"run","title":"prover floor: build patched sp1-gpu-server, run 7 (patch v5: abort on panic, the stage named; CPU only; live server untouched)","status":"done","ok":true,"exit":0,"secs":125,"started_at":"2026-10-06T13:02:39Z","finished_at":"2026-10-06T13:04:44Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"floor-pc2-verify","kind":"run","title":"prover floor: the packaged server's verification run (manifest check, tampered copy refused, install into ~/.sp1/bin, one shard on the 12 GB profile, live server put back)","status":"done","ok":true,"exit":0,"secs":58,"started_at":"2026-10-06T12:50:53Z","finished_at":"2026-10-06T12:51:51Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"floor-pc2-manifest","kind":"fetch","title":"prover floor: the signed manifest for PC 2's v4 server","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T12:49:37Z","finished_at":"2026-10-06T12:49:37Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\floor-pc2-manifest\\floor-pc2-manifest (750 bytes, sha256 ok), extracted into C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\floor-pc2-manifest","errors":[],"stage":null},{"job":"bench-a-pc2","kind":"run","title":"PC 2: proof-system bench job A (SP1 stock and patched, v1 and empty shard, beside the miner; prover off then on)","status":"done","ok":true,"exit":0,"secs":379,"started_at":"2026-10-06T11:59:46Z","finished_at":"2026-10-06T12:06:05Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"fetch-bench-ps","kind":"fetch","title":"PC 2: the proof-system bench kit (run.sh and two fixtures)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T11:58:31Z","finished_at":"2026-10-06T11:58:31Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\prove\\bench-ps.zip (36598 bytes, sha256 ok), extracted into C:\\Users\\Admin\\AppData\\Local\\igneum\\prove\\bench-ps","errors":[],"stage":null},{"job":"update-now-0312-switch-1ccfe586","kind":"update-now","title":"Proving v1 fresh-record rule: re-read the manifest (proving_v1_fresh_rule_daa 198000) and restart the node with it","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T11:41:23Z","finished_at":"2026-10-06T11:41:23Z","summary":"update check and install asked","errors":[],"stage":null},{"job":"update-now-0312-1ccfe586","kind":"update-now","title":"0.3.12 is published: re-read the manifest and install now (the fresh-record rule behind its height, the segment-aligned prover, the GPU list in performance order, Ember Tune, Power control off)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T11:33:28Z","finished_at":"2026-10-06T11:33:28Z","summary":"update check and install asked","errors":[],"stage":null}],"last_report_at":"2026-10-06T17:46:48Z"}]},"sources":{"mac":{"ok":true,"at":"2026-10-06T17:49:20Z"},"pcs":{"ok":true,"at":"2026-10-06T17:49:21Z"},"box":{"ok":true,"at":"2026-10-06T17:48:54Z","age_s":27}},"baselines":{"node-linux":{"secs":690,"source":"approximate: infra/cross/build-linux.sh on this Mac, 8 to 15 min per target at nice 19 with 4 to 6 jobs (docs/plans/build-job.md, 4 Oct 2026)"},"node-windows":{"secs":690,"source":"approximate: proto-cuda/windows-node/cross-build.sh on this Mac, the same 8 to 15 min (docs/plans/build-job.md, 4 Oct 2026)"}},"headline":{"clean_node":{"box_s":87,"measured_at":"2026-10-06T17:32:28Z","note":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow, fork 3bfe346f, cold target dir and cold sccache (0 hits / 993 misses), 563 crates, -j 90. Mac not measured on this tree; the logs hold 12 min 36 s to 17 min 53 s for other commits at -j 4 under load."},"incremental_under_load":{"box_s":7,"measured_at":"2026-10-06T17:33:20Z","note":"one line changed in kaspad/src/main.rs (uncommitted, carried by the overlay), same target dir, 1 crate compiled and igneumd relinked; box load 12 from the clean build a minute earlier, no other build. Mac not measured on this tree; logs: 2 min 08 s to 15 min 18 s for other commits under load."},"windows_cross":{"box_s":104,"measured_at":"2026-10-06T17:35:44Z","note":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu, mingw GCC 13 posix, cold (995 compiles); 8 s on a repeat. Mac not measured on this tree; logs: 4 min 49 s warm to 12 min 28 s for other commits."}}} \ No newline at end of file diff --git a/tools/workers/launchd/com.igneum.workers-push.plist b/tools/workers/launchd/com.igneum.workers-push.plist new file mode 100644 index 00000000..dfe3f01a --- /dev/null +++ b/tools/workers/launchd/com.igneum.workers-push.plist @@ -0,0 +1,16 @@ + + + + + Labelcom.igneum.workers-push + ProgramArguments/opt/homebrew/bin/node__ROOT__/tools/workers/push.mjs + StartInterval60 + RunAtLoad + WorkingDirectory__ROOT__ + EnvironmentVariablesPATH/opt/homebrew/bin:/usr/local/bin:/usr/bin:/binHOME__HOME__ + StandardOutPath__HOME__/Library/Logs/igneum-workers-push.log + StandardErrorPath__HOME__/Library/Logs/igneum-workers-push.log + Nice10 + ProcessTypeBackground + diff --git a/tools/workers/launchd/install.sh b/tools/workers/launchd/install.sh new file mode 100755 index 00000000..67fe6fce --- /dev/null +++ b/tools/workers/launchd/install.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Install or refresh the Mac pusher's launchd agent (every 60 s). Run from any checkout: the plist points at the MAIN +# checkout's tools/workers/push.mjs (/Users/joshm/Projects/igneum) unless IGNEUM_WORKERS_ROOT names another, so the agent +# keeps working after this worktree is removed. +# tools/workers/launchd/install.sh install and (re)load +# tools/workers/launchd/install.sh --remove unload and delete +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="${IGNEUM_WORKERS_ROOT:-/Users/joshm/Projects/igneum}" +LABEL=com.igneum.workers-push; DEST="$HOME/Library/LaunchAgents/$LABEL.plist" +if [ "${1:-}" = --remove ]; then launchctl bootout "gui/$(id -u)/$LABEL" 2>/dev/null || true; rm -f "$DEST"; echo "removed $LABEL"; exit 0; fi +[ -f "$ROOT/tools/workers/push.mjs" ] || { echo "no $ROOT/tools/workers/push.mjs (merge workers-dash to master first, or set IGNEUM_WORKERS_ROOT)" >&2; exit 1; } +mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs" +sed -e "s#__ROOT__#$ROOT#g" -e "s#__HOME__#$HOME#g" "$HERE/$LABEL.plist" > "$DEST" +plutil -lint "$DEST" >/dev/null +launchctl bootout "gui/$(id -u)/$LABEL" 2>/dev/null || true +launchctl bootstrap "gui/$(id -u)" "$DEST" +launchctl kickstart -k "gui/$(id -u)/$LABEL" +sleep 3; launchctl print "gui/$(id -u)/$LABEL" | grep -E 'state|last exit|runs' | head -4; tail -2 "$HOME/Library/Logs/igneum-workers-push.log" | sed 's/dl\/[A-Za-z0-9_-]\{8,\}/dl\//g' diff --git a/tools/workers/lib.mjs b/tools/workers/lib.mjs new file mode 100644 index 00000000..71f4ad55 --- /dev/null +++ b/tools/workers/lib.mjs @@ -0,0 +1,221 @@ +// Parsers shared by tools/workers/collect.mjs (on igneum-build-1) and tools/workers/push.mjs (on the Mac). Pure functions +// over text the machine gives us (/proc files, lock files, the builds JSONL), so node --test can run them against the +// fixtures in tools/workers/fixtures without a box. Every reader here takes the machine's own facts, never a reported number. + +/// /proc/stat -> { total: [{user, nice, system, idle, iowait, irq, softirq, steal}], cores: [...] } (jiffies). +export function parseProcStat(text) { + const cores = []; let total = null; + for (const line of String(text).split('\n')) { + if (!line.startsWith('cpu')) continue; + const f = line.trim().split(/\s+/); + const n = f.slice(1, 9).map(Number); + const row = { user: n[0], nice: n[1], system: n[2], idle: n[3], iowait: n[4] || 0, irq: n[5] || 0, softirq: n[6] || 0, steal: n[7] || 0 }; + if (f[0] === 'cpu') total = row; else cores.push(row); + } + return { total, cores }; +} + +/// Busy percentage between two /proc/stat samples: 0 to 100 for the whole box and per core, in core order. +export function cpuBusy(before, after) { + const pct = (a, b) => { + if (!a || !b) return 0; + const sum = r => r.user + r.nice + r.system + r.idle + r.iowait + r.irq + r.softirq + r.steal; + const idle = r => r.idle + r.iowait; + const dt = sum(b) - sum(a); const di = idle(b) - idle(a); + if (dt <= 0) return 0; + return Math.max(0, Math.min(100, Math.round((1 - di / dt) * 1000) / 10)); + }; + const n = Math.min(before.cores.length, after.cores.length); + const per = []; for (let i = 0; i < n; i++) per.push(pct(before.cores[i], after.cores[i])); + return { busy_pct: pct(before.total, after.total), per_core: per, cores_busy: per.filter(p => p >= 50).length }; +} + +/// /proc/meminfo -> kB numbers and a used percentage (used = total minus available, what a build can still take). +export function parseMeminfo(text) { + const m = {}; for (const line of String(text).split('\n')) { const r = /^(\w+):\s+(\d+)/.exec(line); if (r) m[r[1]] = Number(r[2]); } + const total = m.MemTotal || 0, avail = m.MemAvailable ?? m.MemFree ?? 0; + return { total_kb: total, available_kb: avail, used_kb: Math.max(0, total - avail), used_pct: total ? Math.round(((total - avail) / total) * 1000) / 10 : 0, swap_total_kb: m.SwapTotal || 0, swap_free_kb: m.SwapFree || 0 }; +} + +/// /proc/net/dev -> [{ iface, rx_bytes, tx_bytes }] without lo. +export function parseNetDev(text) { + const out = []; + for (const line of String(text).split('\n').slice(2)) { + const r = /^\s*([^:\s]+):\s*(.*)$/.exec(line); if (!r) continue; + const f = r[2].trim().split(/\s+/).map(Number); + if (r[1] === 'lo') continue; + out.push({ iface: r[1], rx_bytes: f[0] || 0, tx_bytes: f[8] || 0 }); + } + return out; +} + +/// /proc/loadavg -> { load: [1, 5, 15], running, threads }. +export function parseLoadavg(text) { + const f = String(text).trim().split(/\s+/); + const rq = (f[3] || '0/0').split('/'); + return { load: [Number(f[0]) || 0, Number(f[1]) || 0, Number(f[2]) || 0], running: Number(rq[0]) || 0, threads: Number(rq[1]) || 0 }; +} + +/// The slot-file line both with-lock.sh and bs_remote_run write: "pid N since HH:MM:SSZ waited S s: