diff --git a/infra/build-server/workers/igneum-workers.service b/infra/build-server/workers/igneum-workers.service new file mode 100644 index 00000000..ce963fec --- /dev/null +++ b/infra/build-server/workers/igneum-workers.service @@ -0,0 +1,21 @@ +# The worker dashboard's collector on igneum-build-1: one pass, written by igneum-workers.timer every 30 s. +# Reads /proc, /sys, /srv/builds/_locks, the process table and /srv/builds/_log/builds.jsonl; writes /srv/workers/workers.json. +# Installed by infra/build-server/workers/install.sh (copies tools/workers/{lib,collect}.mjs to /srv/workers/bin). +[Unit] +Description=Igneum worker dashboard collector (one pass) +After=local-fs.target + +[Service] +Type=oneshot +User=build +Group=build +Environment=HOME=/home/build +Environment=PATH=/home/build/.cargo/bin:/usr/local/bin:/usr/bin:/bin +Environment=SCCACHE_DIR=/srv/sccache +Environment=IGNEUM_WORKERS_DIR=/srv/workers +Environment=IGNEUM_BUILD_SLOTS_DIR=/srv/builds/_locks +Environment=IGNEUM_BUILD_ROOT=/srv/builds +WorkingDirectory=/srv/workers/bin +ExecStart=/usr/local/bin/node /srv/workers/bin/collect.mjs +TimeoutStartSec=25 +Nice=10 diff --git a/infra/build-server/workers/igneum-workers.timer b/infra/build-server/workers/igneum-workers.timer new file mode 100644 index 00000000..a6a6e5cb --- /dev/null +++ b/infra/build-server/workers/igneum-workers.timer @@ -0,0 +1,13 @@ +# Fires the collector every 30 s from boot. `systemctl list-timers igneum-workers.timer` shows the next pass; +# `journalctl -u igneum-workers.service -n 20` the last errors. +[Unit] +Description=Igneum worker dashboard collector, every 30 s + +[Timer] +OnBootSec=20s +OnUnitActiveSec=30s +AccuracySec=1s +Unit=igneum-workers.service + +[Install] +WantedBy=timers.target diff --git a/infra/build-server/workers/install.sh b/infra/build-server/workers/install.sh new file mode 100755 index 00000000..6842091e --- /dev/null +++ b/infra/build-server/workers/install.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Install or refresh the worker dashboard collector on igneum-build-1 from this Mac. +# infra/build-server/workers/install.sh copies tools/workers/{lib,collect}.mjs and the two units, enables the timer +# Needs root over ssh (root@ with ~/.ssh/igneum_ed25519); the host ip comes from ~/.config/igneum/build-server (build@). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../../.." && pwd)" +KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}" +HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')" +IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server" >&2; exit 1; } +SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10) +"${SSH[@]}" "root@$IP" 'install -d -o build -g build /srv/workers /srv/workers/bin /srv/workers/sources /srv/builds/_log; chown build:build /srv/builds/_log' +scp -q -i "$KEY" "$ROOT/tools/workers/lib.mjs" "$ROOT/tools/workers/collect.mjs" "build@$IP:/srv/workers/bin/" +scp -q -i "$KEY" "$HERE/igneum-workers.service" "$HERE/igneum-workers.timer" "root@$IP:/etc/systemd/system/" +"${SSH[@]}" "root@$IP" 'systemctl daemon-reload && systemctl enable --now igneum-workers.timer >/dev/null 2>&1; systemctl start igneum-workers.service; systemctl is-active igneum-workers.timer; systemctl list-timers igneum-workers.timer --no-pager | sed -n 2p; ls -la /srv/workers/workers.json' diff --git a/tools/workers/collect.mjs b/tools/workers/collect.mjs new file mode 100644 index 00000000..03cede18 --- /dev/null +++ b/tools/workers/collect.mjs @@ -0,0 +1,160 @@ +#!/usr/bin/env node +// The worker dashboard's collector. Runs on igneum-build-1 every 30 s (infra/build-server/workers/igneum-workers.timer) +// as the build user and writes /srv/workers/workers.json: what the MACHINE says (uptime, load, /proc/stat deltas per core, +// memory, the /srv disk, network bytes, hwmon temperatures), the slot and lock state under /srv/builds/_locks, the cargo +// processes with their worktree and target, the slot waiters, the recent builds from /srv/builds/_log/builds.jsonl and +// sccache's own counters. Nothing here is a reported number: every field is read from /proc, /sys, the lock files, the +// process table or the log the build itself appended. +// +// node tools/workers/collect.mjs write /srv/workers/workers.json (IGNEUM_WORKERS_DIR overrides) +// node tools/workers/collect.mjs --stdout print instead of writing +// node tools/workers/collect.mjs --sample-ms 1000 the /proc/stat and network delta window (default 1000) +// +// The Mac side (tools/workers/push.mjs) drops its own facts at /srv/workers/sources/mac.json (the Mac's with-lock slots) and +// /srv/workers/sources/pcs.json (the two PCs' relay job states); when present and under 10 minutes old they are merged +// into the same workers.json, so one file answers for the whole shop whichever side serves it. Needs Node 22, no deps. +import { readFileSync, readdirSync, writeFileSync, renameSync, mkdirSync, existsSync, statSync, readlinkSync, openSync, closeSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import { hostname } from 'node:os'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseProcStat, cpuBusy, parseMeminfo, parseNetDev, parseLoadavg, parseLockDir, parseBuildsJsonl, parseSccacheStats, pickTemps, cargoProcess, deriveKind, BASELINES } from './lib.mjs'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const argv = process.argv.slice(2); +const flag = (n, d) => { const i = argv.indexOf(`--${n}`); return i >= 0 ? (argv[i + 1] ?? true) : d; }; +const OUT_DIR = process.env.IGNEUM_WORKERS_DIR || '/srv/workers'; +const LOCKS = process.env.IGNEUM_BUILD_SLOTS_DIR || '/srv/builds/_locks'; +const BUILDS = process.env.IGNEUM_BUILD_ROOT || '/srv/builds'; +const LOG = process.env.IGNEUM_BUILDS_LOG || join(BUILDS, '_log', 'builds.jsonl'); +const SAMPLE_MS = Number(flag('sample-ms', 1000)); +const iso = ms => new Date(ms).toISOString().replace(/\.\d{3}Z$/, 'Z'); +const read = p => { try { return readFileSync(p, 'utf8'); } catch { return ''; } }; +const sh = (cmd, args, opts = {}) => { const r = spawnSync(cmd, args, { encoding: 'utf8', timeout: 8000, ...opts }); return r.status === 0 ? r.stdout : ''; }; + +// ---- machine facts ---------------------------------------------------------------------------------------------------- +function machine() { + const stat0 = parseProcStat(read('/proc/stat')); const net0 = parseNetDev(read('/proc/net/dev')); const t0 = Date.now(); + const end = t0 + SAMPLE_MS; while (Date.now() < end) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, end - Date.now()); + const stat1 = parseProcStat(read('/proc/stat')); const net1 = parseNetDev(read('/proc/net/dev')); const dt = (Date.now() - t0) / 1000; + const cpu = cpuBusy(stat0, stat1); + const up = Number(read('/proc/uptime').split(' ')[0]) || 0; + const la = parseLoadavg(read('/proc/loadavg')); + const mem = parseMeminfo(read('/proc/meminfo')); + const df = sh('df', ['-B1', '--output=target,size,used,avail', BUILDS]).trim().split('\n').pop() || ''; + const dfp = df.trim().split(/\s+/); + const disk = dfp.length >= 4 ? { mount: dfp[0], total_bytes: Number(dfp[1]), used_bytes: Number(dfp[2]), avail_bytes: Number(dfp[3]), used_pct: Number(dfp[1]) ? Math.round((Number(dfp[2]) / Number(dfp[1])) * 1000) / 10 : 0 } : null; + // the busiest interface carries the build traffic (rsync in, artefacts out) + const nets = net1.map(n => { const b = net0.find(x => x.iface === n.iface) || n; return { ...n, rx_bps: Math.max(0, Math.round((n.rx_bytes - b.rx_bytes) * 8 / dt)), tx_bps: Math.max(0, Math.round((n.tx_bytes - b.tx_bytes) * 8 / dt)) }; }).filter(n => n.rx_bytes + n.tx_bytes > 0).sort((a, b) => (b.rx_bytes + b.tx_bytes) - (a.rx_bytes + a.tx_bytes)); + const sensors = []; + for (const h of safeList('/sys/class/hwmon')) { + const dir = join('/sys/class/hwmon', h); const name = read(join(dir, 'name')).trim(); + for (const f of safeList(dir)) { const m = /^(temp\d+)_input$/.exec(f); if (!m) continue; sensors.push({ name, label: read(join(dir, `${m[1]}_label`)).trim() || null, millic: Number(read(join(dir, f))) }); } + } + const osr = read('/etc/os-release'); const pretty = (/^PRETTY_NAME="?([^"\n]+)"?/m.exec(osr) || [])[1] || null; + const clk = Number(sh('getconf', ['CLK_TCK']).trim()) || 100; + return { name: hostname(), os: pretty, cores: stat1.cores.length, uptime_s: Math.round(up), boot_at: iso(Date.now() - up * 1000), ...la, cpu, mem, disk, net: nets[0] || null, temps: pickTemps(sensors), clk, kernel: read('/proc/sys/kernel/osrelease').trim() || null }; +} +const safeList = d => { try { return readdirSync(d); } catch { return []; } }; +const alive = pid => { try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; } }; + +// flock probe without python: `flock -n true` fails while the slot is held (the slot is held by another process's fd) +const flockHeld = p => { const r = spawnSync('flock', ['-n', p, 'true'], { timeout: 3000 }); return r.status !== 0; }; + +// ---- slots, waiters, running builds --------------------------------------------------------------------------------------- +function slots(now) { + const files = safeList(LOCKS).map(n => ({ name: n, text: read(join(LOCKS, n)), held: /^(build-\d+|run-\d+|build|measure)$/.test(n) ? flockHeld(join(LOCKS, n)) : undefined })); + return parseLockDir(files, { now, aliveFn: alive }); +} +function procs(bootSec, clk, now) { + const running = [], waiters = []; let compilers = 0; + for (const d of safeList('/proc')) { + if (!/^\d+$/.test(d)) continue; + const pid = Number(d); const comm = read(`/proc/${d}/comm`).trim(); + if (['rustc', 'cc1plus', 'cc1', 'ld', 'ld.lld', 'x86_64-w64-mingw32-ld'].includes(comm)) compilers++; + if (comm === 'cargo') { + let cwd = null; try { cwd = readlinkSync(`/proc/${d}/cwd`); } catch { cwd = null; } + const p = cargoProcess({ pid, cwd, cmdline: read(`/proc/${d}/cmdline`), stat: read(`/proc/${d}/stat`), bootSec, clk, now, root: BUILDS }); + if (p) running.push(p); + } + if (comm === 'flock') { + const args = read(`/proc/${d}/cmdline`).split('\0').filter(Boolean); + if (args[1] === '-w' && /^\d+$/.test(args[2] || '')) { + const stat = read(`/proc/${d}/stat`); const rest = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + const ppid = Number(rest[1]); const startTicks = Number(rest[19]); + const since = Number.isFinite(startTicks) ? iso((bootSec + startTicks / clk) * 1000) : null; + const wait = read(join(LOCKS, `wait-${ppid}`)).trim(); // a waiter's own line, when the appender writes one + waiters.push({ pid: ppid, since, waiting_s: since ? Math.max(0, Math.round((now - Date.parse(since)) / 1000)) : null, label: wait || null, ...(wait ? { kind: deriveKind(parseLabelSafe(wait)) } : {}) }); + } + } + } + // the innermost cargo per worktree: a `cargo build` spawns no nested cargo, but a `cargo test` runs test binaries, not cargo + running.sort((a, b) => String(a.started_at).localeCompare(String(b.started_at))); + return { running, waiters, compilers }; +} +function parseLabelSafe(line) { const r = /: (.*)$/.exec(line); return { command: r ? r[1] : line }; } + +function sccache() { + const bin = existsSync('/home/build/.cargo/bin/sccache') ? '/home/build/.cargo/bin/sccache' : 'sccache'; + const out = sh(bin, ['--show-stats'], { env: { ...process.env, SCCACHE_DIR: process.env.SCCACHE_DIR || '/srv/sccache' } }); + return out ? parseSccacheStats(out) : null; +} + +// ---- sources dropped by the Mac ------------------------------------------------------------------------------------------ +function source(name, now) { + const p = join(OUT_DIR, 'sources', `${name}.json`); + try { + const st = statSync(p); const age = Math.round((now - st.mtimeMs) / 1000); + const j = JSON.parse(readFileSync(p, 'utf8')); + return { data: j, meta: { at: iso(st.mtimeMs), age_s: age, stale: age > 600, ok: true } }; + } catch (e) { return { data: null, meta: { ok: false, error: e.code === 'ENOENT' ? 'the Mac has not pushed yet' : String(e.message || e) } }; } +} + +// ---- main ------------------------------------------------------------------------------------------------------------------- +export function collect() { + const now = Date.now(); + const m = machine(); + const bootSec = Math.round((now - m.uptime_s * 1000) / 1000); + const s = slots(now); + const p = procs(bootSec, m.clk, now); + // attach the slot holder to its build (the label names the worktree and crate the cargo runs in) + for (const r of p.running) { + const h = s.held.find(h => h.worktree === r.worktree && (!h.crate || h.crate === r.crate || !r.crate)); + r.slot = h ? h.slot : null; r.agent = h ? h.agent : null; r.queued_at = h ? h.since : null; r.waited_s = h ? h.waited_s : null; + } + const logText = read(LOG); + const recent = parseBuildsJsonl(logText, { limit: 200 }); + const mac = source('mac', now), pcs = source('pcs', now); + let headline = null; try { headline = JSON.parse(readFileSync(join(OUT_DIR, 'headline.json'), 'utf8')); } catch { headline = null; } + const doc = { + v: 1, + generated_at: iso(now), + generated_by: 'collect.mjs on ' + m.name, + box: { + ...m, clk: undefined, + collected_at: iso(now), + sccache: sccache(), + slots: { count: s.count, held: s.held, free: s.free }, + running: p.running, + queue: p.waiters, + compilers: p.compilers, + recent: recent.rows, + log: { path: LOG, present: !!logText, bad_lines: recent.bad, total: recent.total, note: logText ? null : 'builds.jsonl is not written yet: the build-server agent adds the append to bs_remote_run; until then this lane fills from nothing' }, + }, + mac: mac.data, pcs: pcs.data, + headline, baselines: BASELINES, + sources: { box: { ok: true, at: iso(now) }, mac: mac.meta, pcs: pcs.meta }, + }; + return doc; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { + const doc = collect(); + const text = JSON.stringify(doc); + if (flag('stdout', false)) { process.stdout.write(text + '\n'); } + else { + mkdirSync(OUT_DIR, { recursive: true }); mkdirSync(join(OUT_DIR, 'sources'), { recursive: true }); + const tmp = join(OUT_DIR, '.workers.json.tmp'); writeFileSync(tmp, text); renameSync(tmp, join(OUT_DIR, 'workers.json')); + const fd = openSync(join(OUT_DIR, 'last-run'), 'w'); closeSync(fd); + } +} diff --git a/tools/workers/fixtures/builds.jsonl b/tools/workers/fixtures/builds.jsonl new file mode 100644 index 00000000..b00e98ae --- /dev/null +++ b/tools/workers/fixtures/builds.jsonl @@ -0,0 +1,6 @@ +{"v":1,"id":"igneum-build-1-1759770840-36992","host":"igneum-build-1","tool":"cross-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu -j 90","target":"x86_64-pc-windows-gnu","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:33:58Z","start":"2026-10-06T17:34:00Z","end":"2026-10-06T17:51:12Z","secs":1032,"exit":0,"compiles":1954,"sccache":{"hits":0,"misses":1886,"hits_total":0,"misses_total":1886},"load_end":[13.86,9.61,4.32],"artefacts":[{"path":"target/x86_64-pc-windows-gnu/release/igneumd.exe","bytes":52911104,"sha256":"0f3a9c2e1d8b7a6f5e4d3c2b1a0f9e8d7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f"},{"path":"target/x86_64-pc-windows-gnu/release/igneum-miner.exe","bytes":9811456,"sha256":"1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b"}]} +{"v":1,"id":"igneum-build-1-1759772000-37001","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow -j 90","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","slot":0,"wait_s":0,"start":"2026-10-06T17:53:20Z","end":"2026-10-06T18:06:41Z","secs":801,"exit":0,"compiles":1102,"sccache":{"hits":790,"misses":312,"hits_total":790,"misses_total":2198},"load_end":[61.2,40.1,22.0],"artefacts":[{"path":"target/release/igneumd","bytes":61120512,"sha256":"2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c"},{"path":"target/release/igneum-miner","bytes":11010048,"sha256":"3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d"}]} +{"v":1,"id":"igneum-build-1-1759773000-37100","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ship0314","crate":"app/igneum-app","command":"cargo build --release -j 90","branch":"release-0.3.14","sha":"952ce8f","label":"igneum-wt-ship0314/app/igneum-app cargo build --release","slot":0,"wait_s":412,"queued_at":"2026-10-06T18:03:10Z","start":"2026-10-06T18:10:02Z","end":"2026-10-06T18:13:48Z","secs":226,"exit":0,"compiles":388,"sccache":{"hits":301,"misses":87,"hits_total":1091,"misses_total":2285},"load_end":[70.3,55.8,30.2],"artefacts":[{"path":"target/release/igneum-app","bytes":30801920,"sha256":"4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e"}]} +{"v":1,"id":"igneum-build-1-1759773600-37200","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ca3-v4-node","crate":"vendor/igneum-node-v4","command":"cargo test --release -p kaspa-consensus-core --lib -j 90","branch":"ca3-v4-node","sha":"f39a8eb","label":"igneum-wt-ca3-v4-node/vendor/igneum-node-v4 cargo test --release -p kaspa-consensus-core --lib","slot":0,"wait_s":0,"start":"2026-10-06T18:14:00Z","end":"2026-10-06T18:19:33Z","secs":333,"exit":101,"compiles":210,"sccache":{"hits":180,"misses":30,"hits_total":1271,"misses_total":2315},"load_end":[55.0,50.2,33.1],"artefacts":[]} +not json at all +{"v":1,"id":"igneum-build-1-1759774200-37300","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-proving-v2","crate":"proving/igneum-prove","command":"cargo build --release -j 90","branch":"proving-v2","sha":"621f258","label":"igneum-wt-proving-v2/proving/igneum-prove cargo build --release","slot":0,"start":"2026-10-06T18:21:00Z","end":"2026-10-06T18:29:10Z","secs":490,"exit":0,"compiles":640,"sccache":{"hits":600,"misses":40,"hits_total":1871,"misses_total":2355},"load_end":[60.1,52.0,38.4],"artefacts":[{"path":"target/release/igneum-prove-host","bytes":80000000,"sha256":"5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f"}]} diff --git a/tools/workers/fixtures/locks/build-0 b/tools/workers/fixtures/locks/build-0 new file mode 100644 index 00000000..65a76e6a --- /dev/null +++ b/tools/workers/fixtures/locks/build-0 @@ -0,0 +1 @@ +pid 36992 since 17:34:00Z waited 0 s: igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server diff --git a/tools/workers/fixtures/locks/build-1 b/tools/workers/fixtures/locks/build-1 new file mode 100644 index 00000000..144af4e9 --- /dev/null +++ b/tools/workers/fixtures/locks/build-1 @@ -0,0 +1 @@ +pid 1 since 09:00:00Z waited 3 s: igneum-wt-old/app/igneum-app cargo build --release diff --git a/tools/workers/fixtures/locks/build-2 b/tools/workers/fixtures/locks/build-2 new file mode 100644 index 00000000..e69de29b diff --git a/tools/workers/fixtures/locks/slots b/tools/workers/fixtures/locks/slots new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/tools/workers/fixtures/locks/slots @@ -0,0 +1 @@ +1 diff --git a/tools/workers/fixtures/proc-stat-0.txt b/tools/workers/fixtures/proc-stat-0.txt new file mode 100644 index 00000000..a2e7c285 --- /dev/null +++ b/tools/workers/fixtures/proc-stat-0.txt @@ -0,0 +1,8 @@ +cpu 513633 175 77715 5794769 1804 0 128 0 0 0 +cpu0 10000 0 1000 50000 10 0 1 0 0 0 +cpu1 20000 0 2000 40000 10 0 1 0 0 0 +cpu2 100 0 10 60000 10 0 1 0 0 0 +cpu3 30000 0 500 30000 10 0 1 0 0 0 +intr 1 2 3 +ctxt 99 +btime 1759772000 diff --git a/tools/workers/fixtures/proc-stat-1.txt b/tools/workers/fixtures/proc-stat-1.txt new file mode 100644 index 00000000..cf0b599a --- /dev/null +++ b/tools/workers/fixtures/proc-stat-1.txt @@ -0,0 +1,8 @@ +cpu 514633 175 77915 5795569 1804 0 128 0 0 0 +cpu0 10100 0 1000 50000 10 0 1 0 0 0 +cpu1 20000 0 2000 40100 10 0 1 0 0 0 +cpu2 150 0 60 60000 10 0 1 0 0 0 +cpu3 30025 0 500 30075 10 0 1 0 0 0 +intr 1 2 3 +ctxt 99 +btime 1759772000 diff --git a/tools/workers/fixtures/sccache-stats.txt b/tools/workers/fixtures/sccache-stats.txt new file mode 100644 index 00000000..b31ae8b5 --- /dev/null +++ b/tools/workers/fixtures/sccache-stats.txt @@ -0,0 +1,27 @@ +Compile requests 2294 +Compile requests executed 1954 +Cache hits 790 +Cache misses 1886 +Cache misses (Assembler) 27 +Cache misses (C/C++) 777 +Cache misses (Rust) 1082 +Cache hits rate 29.52 % +Cache timeouts 0 +Cache read errors 0 +Forced recaches 0 +Cache write errors 0 +Compilation failures 0 +Cache errors 0 +Non-cacheable compilations 0 +Non-cacheable calls 340 +Non-compilation calls 0 +Unsupported compiler calls 0 +Average cache write 0.002 s +Average compiler 1.624 s +Average cache read hit 0.001 s +Failed distributed compilations 0 +Cache location Local disk: "/srv/sccache" +Use direct/preprocessor mode? yes +Version (client) 0.10.0 +Cache size 12 GiB +Max cache size 100 GiB diff --git a/tools/workers/fixtures/workers.demo.json b/tools/workers/fixtures/workers.demo.json new file mode 100644 index 00000000..f959d092 --- /dev/null +++ b/tools/workers/fixtures/workers.demo.json @@ -0,0 +1 @@ +{"v":1,"generated_at":"2026-10-06T17:49:21Z","generated_by":"demo document (fixtures over a live snapshot)","box":{"name":"igneum-build-1","os":"Ubuntu 24.04.5 LTS","cores":96,"uptime_s":1518,"boot_at":"2026-10-06T17:23:37Z","load":[68.2,41.3,19.7],"running":[{"pid":37210,"cwd":"/srv/builds/igneum-wt-ship0314/vendor/igneum-node-0314","worktree":"igneum-wt-ship0314","crate":"vendor/igneum-node-0314","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu -j 90","target":"x86_64-pc-windows-gnu","kind":"node-windows","started_at":"2026-10-06T17:42:31Z","elapsed_s":410,"slot":"build-0","agent":"release-0.3.14"}],"threads":1229,"cpu":{"busy_pct":71.4,"per_core":[55,63,71,12,84,89,92,94,95,94,12,87,82,76,68,61,57,12,73,79,85,90,93,95,12,93,90,86,80,74,66,12,60,67,75,81,87,91,12,95,95,93,89,84,78,12,64,56,62,70,77,83,12,92,94,95,94,92,88,12,76,69,62,56,64,72,12,85,89,93,95,95,94,12,87,81,74,67,59,59,12,74,81,86,91,93,95,12,93,90,85,79,72,65,12,61],"cores_busy":82},"mem":{"total_kb":131556268,"available_kb":122888768,"used_kb":8667500,"used_pct":6.6,"swap_total_kb":0,"swap_free_kb":0},"disk":{"mount":"/","total_bytes":3777582440448,"used_bytes":17311846400,"avail_bytes":3568303419392,"used_pct":0.5},"net":{"iface":"enp193s0f0np0","rx_bytes":1000335434,"tx_bytes":321279277,"rx_bps":528,"tx_bps":0},"temps":[{"name":"cpu","label":"Tctl","c":63},{"name":"nvme","label":"Sensor 2","c":46.9},{"name":"nic","label":"bnxt_en","c":54}],"kernel":"6.8.0-139-generic","collected_at":"2026-10-06T17:48:54Z","sccache":{"requests":4416,"executed":3862,"hits":864,"misses":2977,"hit_rate_pct":22.5,"cache_size":"2 GiB","max_size":"100 GiB"},"slots":{"count":1,"held":[{"slot":"build-0","pid":36992,"since":"2026-10-06T17:42:29Z","waited_s":0,"label":"igneum-wt-ship0314/vendor/igneum-node-0314 cross x86_64-pc-windows-gnu; agent=release-0.3.14","agent":"release-0.3.14","worktree":"igneum-wt-ship0314","crate":"vendor/igneum-node-0314","command":"cargo build --release --target x86_64-pc-windows-gnu","target":"x86_64-pc-windows-gnu","kind":"node-windows"}],"free":[]},"queue":[{"pid":38001,"since":"2026-10-06T17:47:46Z","waiting_s":95,"label":"pid 38001 since 00:00:00Z waited 0 s: igneum-wt-ca3-v4-node/vendor/igneum-node-v4 cargo test --release -p kaspa-consensus-core --lib","kind":"suite"}],"compilers":86,"recent":[{"v":1,"id":"igneum-build-1-1759774200-37300","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-proving-v2","crate":"proving/igneum-prove","command":"cargo build --release -j 90","branch":"proving-v2","sha":"621f258","label":"igneum-wt-proving-v2/proving/igneum-prove cargo build --release","slot":0,"start":"2026-10-06T17:11:11Z","end":"2026-10-06T17:19:21Z","secs":490,"exit":0,"compiles":640,"sccache":{"hits":600,"misses":40,"hit_rate_pct":93.8,"hits_total":1871,"misses_total":2355},"load_end":[60.1,52,38.4],"artefacts":[{"path":"target/release/igneum-prove-host","bytes":80000000,"sha256":"5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f"}],"kind":"prove","target":"x86_64-unknown-linux-gnu","ok":true},{"v":1,"id":"igneum-build-1-1759773600-37200","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ca3-v4-node","crate":"vendor/igneum-node-v4","command":"cargo test --release -p kaspa-consensus-core --lib -j 90","branch":"ca3-v4-node","sha":"f39a8eb","label":"igneum-wt-ca3-v4-node/vendor/igneum-node-v4 cargo test --release -p kaspa-consensus-core --lib","slot":0,"wait_s":0,"start":"2026-10-06T17:04:11Z","end":"2026-10-06T17:09:44Z","secs":333,"exit":101,"compiles":210,"sccache":{"hits":180,"misses":30,"hit_rate_pct":85.7,"hits_total":1271,"misses_total":2315},"load_end":[55,50.2,33.1],"artefacts":[],"kind":"suite","target":"x86_64-unknown-linux-gnu","ok":false},{"v":1,"id":"igneum-build-1-1759773000-37100","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-ship0314","crate":"app/igneum-app","command":"cargo build --release -j 90","branch":"release-0.3.14","sha":"952ce8f","label":"igneum-wt-ship0314/app/igneum-app cargo build --release","slot":0,"wait_s":412,"queued_at":"2026-10-06T18:03:10Z","start":"2026-10-06T17:00:13Z","end":"2026-10-06T17:03:59Z","secs":226,"exit":0,"compiles":388,"sccache":{"hits":301,"misses":87,"hit_rate_pct":77.6,"hits_total":1091,"misses_total":2285},"load_end":[70.3,55.8,30.2],"artefacts":[{"path":"target/release/igneum-app","bytes":30801920,"sha256":"4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e"}],"kind":"app","target":"x86_64-unknown-linux-gnu","ok":true},{"v":1,"id":"igneum-build-1-1759772000-37001","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow -j 90","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","slot":0,"wait_s":0,"start":"2026-10-06T16:43:31Z","end":"2026-10-06T16:56:52Z","secs":801,"exit":0,"compiles":1102,"sccache":{"hits":790,"misses":312,"hit_rate_pct":71.7,"hits_total":790,"misses_total":2198},"load_end":[61.2,40.1,22],"artefacts":[{"path":"target/release/igneumd","bytes":61120512,"sha256":"2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c"},{"path":"target/release/igneum-miner","bytes":11010048,"sha256":"3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d"}],"kind":"node-linux","target":"x86_64-unknown-linux-gnu","ok":true},{"v":1,"id":"igneum-build-1-1759770840-36992","host":"igneum-build-1","tool":"cross-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu -j 90","target":"x86_64-pc-windows-gnu","branch":"build-server","sha":"c07ae7e","label":"igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:33:58Z","start":"2026-10-06T16:24:11Z","end":"2026-10-06T16:41:23Z","secs":1032,"exit":0,"compiles":1954,"sccache":{"hits":0,"misses":1886,"hit_rate_pct":0,"hits_total":0,"misses_total":1886},"load_end":[13.86,9.61,4.32],"artefacts":[{"path":"target/x86_64-pc-windows-gnu/release/igneumd.exe","bytes":52911104,"sha256":"0f3a9c2e1d8b7a6f5e4d3c2b1a0f9e8d7c6b5a4f3e2d1c0b9a8f7e6d5c4b3a2f"},{"path":"target/x86_64-pc-windows-gnu/release/igneum-miner.exe","bytes":9811456,"sha256":"1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b"}],"kind":"node-windows","ok":true},{"v":1,"id":"igneum-build-1-1791308825-86828","host":"igneum-build-1","tool":"cross-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","kind":"node-windows","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu","target":"x86_64-pc-windows-gnu","branch":"detached-3bfe346f","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:47:05Z","start":"2026-10-06T17:47:05Z","end":"2026-10-06T17:48:46Z","secs":101,"exit":0,"compiles":867,"sccache":{"hits":259,"misses":605,"hit_rate_pct":30,"hits_total":863,"misses_total":2977},"load_end":[28.91,15.83,10.29],"artefacts":[{"path":"target/x86_64-pc-windows-gnu/release/igneumd.exe","bytes":49436672,"sha256":"c38b7570fbe49fad6c3437efdca2571734f7bc1fe79731492611f0b5448274f2"},{"path":"target/x86_64-pc-windows-gnu/release/igneum-miner.exe","bytes":10201600,"sha256":"c3a0fc2be734cb61f1afe71ad01f44103ef6651ed2092e9a1a63a839d0749e77"}],"ok":true},{"v":1,"id":"igneum-build-1-1791308605-67585","host":"igneum-build-1","tool":"build-remote","worktree":"igneum","crate":"vendor/igneum-node-v4","kind":"node-linux","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","target":"x86_64-unknown-linux-gnu","branch":"devnet-v4","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum/vendor/igneum-node-v4 cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:43:25Z","start":"2026-10-06T17:43:25Z","end":"2026-10-06T17:44:44Z","secs":79,"exit":0,"compiles":999,"sccache":{"hits":604,"misses":389,"hit_rate_pct":60.8,"hits_total":604,"misses_total":2372},"load_end":[22.32,13.04,8.65],"artefacts":[{"path":"target/release/igneumd","bytes":48222304,"sha256":"f236e6cf9d5b92a9253c30504a83293c0d05b095d6a963168fc95e86d2a3285a"},{"path":"target/release/igneum-miner","bytes":9106976,"sha256":"6b580e6401523b6a77815dd2c0e5b63bcf36392c8a59c5af5133d7373ac7fd5c"}],"ok":true},{"v":1,"id":"igneum-build-1-1791308425-64707","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","kind":"node-linux","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","target":"x86_64-unknown-linux-gnu","branch":"detached-3bfe346f","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:40:25Z","start":"2026-10-06T17:40:25Z","end":"2026-10-06T17:40:35Z","secs":10,"exit":0,"compiles":1,"sccache":{"hits":0,"misses":1,"hit_rate_pct":0,"hits_total":0,"misses_total":1983},"load_end":[16.06,12.32,7.7],"artefacts":[{"path":"target/release/igneumd","bytes":48221920,"sha256":"c35899ee6b1e95abf72c522be693a62c8f576b83aa94e59f3a0a9e147a029f2e"},{"path":"target/release/igneum-miner","bytes":9107232,"sha256":"c752893f78e1d284daa2f9e389fed266aaa7d5497edab299a93d1537dc0bb075"}],"ok":true},{"v":1,"id":"igneum-build-1-1791308374-62696","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","kind":"node-linux","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","target":"x86_64-unknown-linux-gnu","branch":"detached-3bfe346f","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:39:34Z","start":"2026-10-06T17:39:34Z","end":"2026-10-06T17:39:41Z","secs":7,"exit":0,"compiles":0,"sccache":{"hits":0,"misses":0,"hit_rate_pct":null,"hits_total":0,"misses_total":1982},"load_end":[15.75,12.32,7.45],"artefacts":[{"path":"target/release/igneumd","bytes":48220896,"sha256":"f9e0a4d198658b34b3d530e2f8395b3de1a96ebc928553a8d709d2b14cbcdfb0"},{"path":"target/release/igneum-miner","bytes":9107232,"sha256":"c752893f78e1d284daa2f9e389fed266aaa7d5497edab299a93d1537dc0bb075"}],"ok":true},{"v":1,"id":"igneum-build-1-1791308317-60832","host":"igneum-build-1","tool":"cross-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","kind":"node-windows","command":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu","target":"x86_64-pc-windows-gnu","branch":"detached-3bfe346f","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum-wt-build-server/vendor/igneum-node-bs cross x86_64-pc-windows-gnu; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:38:37Z","start":"2026-10-06T17:38:37Z","end":"2026-10-06T17:38:46Z","secs":9,"exit":0,"compiles":0,"sccache":{"hits":0,"misses":0,"hit_rate_pct":null,"hits_total":0,"misses_total":1982},"load_end":[18.57,12.67,7.3],"artefacts":[{"path":"target/x86_64-pc-windows-gnu/release/igneumd.exe","bytes":49434624,"sha256":"bfbff01565f21408e4d5df80a02dc979684839497517b4c84b50d0e8bf5da715"},{"path":"target/x86_64-pc-windows-gnu/release/igneum-miner.exe","bytes":10201600,"sha256":"5812b74a3eb0672e9872e00f58b9c517e0a0861cef6ac0854598971fe556e8bb"}],"ok":true},{"v":1,"id":"igneum-build-1-1791308306-58965","host":"igneum-build-1","tool":"build-remote","worktree":"igneum-wt-build-server","crate":"vendor/igneum-node-bs","kind":"node-linux","command":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow","target":"x86_64-unknown-linux-gnu","branch":"detached-3bfe346f","sha":"3bfe346f4de0d9a982627270c791b7855abc5af6","label":"igneum-wt-build-server/vendor/igneum-node-bs cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow; agent=build-server","agent":"build-server","slot":0,"wait_s":0,"queued_at":"2026-10-06T17:38:26Z","start":"2026-10-06T17:38:26Z","end":"2026-10-06T17:38:33Z","secs":7,"exit":0,"compiles":0,"sccache":{"hits":0,"misses":0,"hit_rate_pct":null,"hits_total":0,"misses_total":1982},"load_end":[9.75,10.89,6.68],"artefacts":[{"path":"target/release/igneumd","bytes":48220896,"sha256":"f9e0a4d198658b34b3d530e2f8395b3de1a96ebc928553a8d709d2b14cbcdfb0"},{"path":"target/release/igneum-miner","bytes":9107232,"sha256":"c752893f78e1d284daa2f9e389fed266aaa7d5497edab299a93d1537dc0bb075"}],"ok":true}],"log":{"path":"/srv/builds/_log/builds.jsonl","present":true,"bad_lines":0,"total":6,"note":null}},"mac":{"name":"MacBook-Pro","collected_at":"2026-10-06T17:49:20Z","build_slots":3,"slots":{"count":3,"held":[{"slot":"build-0","pid":4242,"since":"2026-10-06T17:27:41Z","waited_s":0,"label":"cargo build --release -p igneum-app","agent":null,"worktree":"igneum-wt-ship0314","crate":null,"command":"cargo build --release -p igneum-app","target":"x86_64-unknown-linux-gnu","kind":"app"}],"free":["build","build-0","build-1","build-2","measure","run-0","run-1","run-2"]},"queue":[{"pid":4300,"mode":"measure","waiting_s":240,"command":"node infra/gpu-bench/hashrate.mjs --secs 120","kind":"other"}]},"pcs":{"collected_at":"2026-10-06T17:49:21Z","machines":[{"id":"ae432dc7","name":"PC 1","role":"builds","note":"Linux and Windows binaries, the project lead's desk","machine":"DESKTOP-KMCV30N-ae432dc7","running":null,"queue":[],"recent":[{"job":"run-ca3-pc1-4070-shadow-20261006","kind":"run","title":"CA3 PC 1: the shadow ladder on the RTX 4070","status":"done","ok":true,"exit":0,"secs":725,"started_at":"2026-10-06T17:36:05Z","finished_at":"2026-10-06T17:48:10Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-family-20261006-e","kind":"run","title":"CA3 PC 1: family step costs on the 9070 XT (run e, the card alone)","status":"done","ok":true,"exit":0,"secs":170,"started_at":"2026-10-06T17:31:57Z","finished_at":"2026-10-06T17:34:47Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-derive-20261006","kind":"run","title":"CA3 PC 1: dr736 build, compile and fingerprints on the 9070 XT","status":"done","ok":true,"exit":0,"secs":317,"started_at":"2026-10-06T17:23:56Z","finished_at":"2026-10-06T17:29:13Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-g2-20261006","kind":"run","title":"CA3 PC 1: gate G2 on the 9070 XT","status":"done","ok":true,"exit":0,"secs":4,"started_at":"2026-10-06T17:22:33Z","finished_at":"2026-10-06T17:22:37Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"fetch-ca3-pc1-amd-g2-20261006","kind":"fetch","title":"CA3 PC 1 G2 kit","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T17:21:33Z","finished_at":"2026-10-06T17:21:33Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-pc1-amd-g2-20261006\\igneum-ca3-pc1-amd-g2-kit.zip (198597 bytes, sha256 ok), extracted into C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-pc1-amd-g2-20261006","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-family-20261006-d","kind":"run","title":"CA3 PC 1: family step costs on the 9070 XT (run d, by name, fixed splat)","status":"done","ok":true,"exit":0,"secs":161,"started_at":"2026-10-06T17:17:34Z","finished_at":"2026-10-06T17:20:15Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-family-20261006-b","kind":"run","title":"CA3 PC 1: family step costs on the 9070 XT (run b, by name)","status":"failed","ok":false,"exit":1,"secs":26,"started_at":"2026-10-06T17:10:18Z","finished_at":"2026-10-06T17:10:44Z","summary":"script exited with code 1","errors":[],"stage":null},{"job":"fetch-ca3-pc1-amd-family-20261006","kind":"fetch","title":"CA3 PC 1 family kit","status":"done","ok":true,"exit":0,"secs":1,"started_at":"2026-10-06T17:09:15Z","finished_at":"2026-10-06T17:09:16Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-pc1-amd-family-20261006\\igneum-ca3-pc1-amd-family-kit.zip (26007 bytes, sha256 ok), extracted into C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-pc1-amd-family-20261006","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-identities-20261006-c","kind":"run","title":"CA3 PC 1: the 9070 XT back to 8 identities (c)","status":"done","ok":true,"exit":0,"secs":120,"started_at":"2026-10-06T17:05:53Z","finished_at":"2026-10-06T17:07:53Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-identities-20261006-b","kind":"run","title":"CA3 PC 1: the 9070 XT back to 8 identities (b)","status":"failed","ok":false,"exit":1,"secs":1,"started_at":"2026-10-06T17:04:10Z","finished_at":"2026-10-06T17:04:11Z","summary":"script exited with code 1","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-identities-20261006","kind":"run","title":"CA3 PC 1: the 9070 XT back to 8 identities","status":"failed","ok":false,"exit":1,"secs":0,"started_at":"2026-10-06T17:02:10Z","finished_at":"2026-10-06T17:02:10Z","summary":"script exited with code 1","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-family-20261006","kind":"run","title":"CA3 PC 1: reserve-family step costs on the 9070 XT","status":"done","ok":true,"exit":0,"secs":150,"started_at":"2026-10-06T16:56:44Z","finished_at":"2026-10-06T16:59:14Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"run-ca3-pc1-amd-reset-20261006","kind":"run","title":"CA3 PC 1: 9070 XT ADLX back to factory, identities read","status":"done","ok":true,"exit":0,"secs":4,"started_at":"2026-10-06T16:55:16Z","finished_at":"2026-10-06T16:55:20Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"ember-proof-pc1-2","kind":"run","title":"Ember proof (reads only): power cap lines, helper log, task action, nvidia-smi, UAC count, card state","status":"done","ok":true,"exit":0,"secs":1,"started_at":"2026-10-06T16:53:02Z","finished_at":"2026-10-06T16:53:03Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"ember-proof-collect-1","kind":"collect","title":"Ember proof: the installed app's power cap lines, the helper log, the task action, nvidia-smi limits","status":"failed","ok":false,"exit":1,"secs":0,"started_at":"2026-10-06T16:51:19Z","finished_at":"2026-10-06T16:51:19Z","summary":"0 files uploaded, command exited with code 1, output in the report","errors":[],"stage":null},{"job":"ember-repoint-pc1-1","kind":"run","title":"Ember: re-point the Power Helper at the installed exe (kit-7 reregister verb) and prove the no-prompt cap","status":"done","ok":true,"exit":0,"secs":73,"started_at":"2026-10-06T16:48:31Z","finished_at":"2026-10-06T16:49:44Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"ember-kit-7","kind":"fetch","title":"Ember kit-7 (185c333: reregister verb, task_exe fix, 45% floor, AMD apply on ack)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T16:47:27Z","finished_at":"2026-10-06T16:47:27Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\ember-kit-7\\igneum-app-ember.exe (3334144 bytes, sha256 ok)","errors":[],"stage":null},{"job":"build-20261006-164418","kind":"build","title":"ember-tune bd7fcf4: Windows app build (kit-7)","status":"done","ok":true,"exit":0,"secs":103,"started_at":"2026-10-06T16:44:56Z","finished_at":"2026-10-06T16:46:39Z","summary":"node devnet-v4 3bfe346f app 0.3.12: every stage ok; 6 files uploaded (27 MB); 1 min of 40","errors":[],"stage":"pack"},{"job":"ember-tune-pc1-6","kind":"run","title":"Ember Tune table run on PC 1 (0.3.13, kit-6): 5090 + 4070 two-knob tune, 9070 XT power ladder, one administrator prompt (registers the Power Helper on the installed exe)","status":"done","ok":true,"exit":0,"secs":2317,"started_at":"2026-10-06T16:01:00Z","finished_at":"2026-10-06T16:39:37Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"ember-kit-6","kind":"fetch","title":"Ember Tune engine (ember-tune 564bdea: the cmd quote fix, the self-registering elevated engine) for the table run","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T15:59:33Z","finished_at":"2026-10-06T15:59:33Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\ember-kit-6\\igneum-app-ember.exe (3354112 bytes, sha256 ok)","errors":[],"stage":null}],"last_report_at":"2026-10-06T17:48:10Z"},{"id":"1ccfe586","name":"PC 2","role":"test suites","note":"cargo test suites and shard runs","machine":"DESKTOP-KMCV30N-1ccfe586","running":null,"queue":[{"job":"build-20261006-174823","kind":"build","title":"Counter ASIC 3.0 G6 (signalling) 3 of 3: the five crates and the app on fork 0562a7f2","expires_at":"2026-10-08T17:48:24Z"}],"recent":[{"job":"build-20261006-174027","kind":"build","title":"Counter ASIC 3.0 G6 (signalling) 2 of 3: kaspa-consensus alone on fork 0562a7f2 (the ban_is_decided flake re-run, the 2.0 rule)","status":"done","ok":true,"exit":0,"secs":345,"started_at":"2026-10-06T17:41:02Z","finished_at":"2026-10-06T17:46:47Z","summary":"node ca3-v4-node 0562a7f2 app : every stage ok; 7 files uploaded (44 MB); 5 min of 40","errors":[],"stage":"pack"},{"job":"build-20261006-173017","kind":"build","title":"Counter ASIC 3.0 G6 (signalling): node suites on fork ca3-v4-node 0562a7f2 (main f39a8eb)","status":"failed","ok":false,"exit":1,"secs":385,"started_at":"2026-10-06T17:32:15Z","finished_at":"2026-10-06T17:38:40Z","summary":"node ca3-v4-node 0562a7f2 app 0.3.11: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40","errors":["test processes::finality::tests::ban_is_decided_by_the_carrying_block_so_nodes_agree_on_every_voter_list ... FAILED","thread 'processes::finality::tests::ban_is_decided_by_the_carrying_block_so_nodes_agree_on_every_voter_list' (20805) panicked at consensus/src/processes/finality.rs:1914:82:","test result: FAILED. 97 passed; 1 failed; 3 ignored; 0 measured; 0 filtered out; finished in 4.26s","RESULT build 2026-10-06T17:38:40Z failed node ca3-v4-node 0562a7f2 app 0.3.11: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40","job build-20261006-173017: failed (exit 1) after 385 s: node ca3-v4-node 0562a7f2 app 0.3.11: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40"],"stage":"pack"},{"job":"build-20261006-171423","kind":"build","title":"build node and app (release-0.3.14-node 4c6b129d)","status":"failed","ok":false,"exit":1,"secs":622,"started_at":"2026-10-06T17:15:09Z","finished_at":"2026-10-06T17:25:31Z","summary":"node release-0.3.14-node 4c6b129d app : FAILED: upload incomplete; 7 files uploaded (44 MB); 10 min of 40","errors":["RESULT upload 2026-10-06T17:25:31Z build-outputs.json FAILED: blob PUT: no url in the reply: {\"error\":{\"code\":\"service_unavailable\",\"message\":\"Blob service is currently unavailable. Please try again.\"","RESULT build 2026-10-06T17:25:31Z failed node release-0.3.14-node 4c6b129d app : FAILED: upload incomplete; 7 files uploaded (44 MB); 10 min of 40","job build-20261006-171423: failed (exit 1) after 622 s: node release-0.3.14-node 4c6b129d app : FAILED: upload incomplete; 7 files uploaded (44 MB); 10 min of 40"],"stage":"pack"},{"job":"build-20261006-165701","kind":"build","title":"build node and app (release-0.3.14-node 4c6b129d)","status":"failed","ok":false,"exit":1,"secs":387,"started_at":"2026-10-06T16:57:31Z","finished_at":"2026-10-06T17:03:58Z","summary":"node release-0.3.14-node 4c6b129d app 0.3.14: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40","errors":["test processes::finality::tests::ban_is_decided_by_the_carrying_block_so_nodes_agree_on_every_voter_list ... FAILED","thread 'processes::finality::tests::ban_is_decided_by_the_carrying_block_so_nodes_agree_on_every_voter_list' (20838) panicked at consensus/src/processes/finality.rs:1914:82:","test result: FAILED. 97 passed; 1 failed; 3 ignored; 0 measured; 0 filtered out; finished in 4.22s","RESULT build 2026-10-06T17:03:58Z failed node release-0.3.14-node 4c6b129d app 0.3.14: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40","job build-20261006-165701: failed (exit 1) after 387 s: node release-0.3.14-node 4c6b129d app 0.3.14: FAILED: test exit 101; 9 files uploaded (46 MB); 6 min of 40"],"stage":"pack"},{"job":"collect-ca3-v4-g1only-20261006","kind":"collect","title":"Counter ASIC 3.0 gate run (hash): the header, G1 lines and two packs G2 lines of run-ca3-v4-gates-pc2-20261006 (PC 2)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T16:12:06Z","finished_at":"2026-10-06T16:12:06Z","summary":"0 files uploaded, command exit 0, output in the report","errors":[],"stage":null},{"job":"collect-ca3-v4-g1lines-20261006","kind":"collect","title":"Counter ASIC 3.0 gate run (hash): the G1 lines and four packs G2 lines of run-ca3-v4-gates-pc2-20261006 from its job.log (PC 2)","status":"done","ok":true,"exit":0,"secs":1,"started_at":"2026-10-06T16:09:59Z","finished_at":"2026-10-06T16:10:00Z","summary":"0 files uploaded, command exit 0, output in the report","errors":[],"stage":null},{"job":"collect-ca3-v4-joblog2-20261006","kind":"collect","title":"Counter ASIC 3.0 gate run (hash): the full job.log of run-ca3-v4-gates-pc2-20261006, app/jobs path (PC 2)","status":"done","ok":true,"exit":0,"secs":1,"started_at":"2026-10-06T16:08:37Z","finished_at":"2026-10-06T16:08:38Z","summary":"1 file uploaded","errors":[],"stage":null},{"job":"build-20261006-155958","kind":"build","title":"Counter ASIC 3.0 G6: node suites on fork ca3-v4-node 5f7e0543 (main a522d04)","status":"done","ok":true,"exit":0,"secs":392,"started_at":"2026-10-06T16:01:02Z","finished_at":"2026-10-06T16:07:34Z","summary":"node ca3-v4-node 5f7e0543 app 0.3.11: every stage ok; 9 files uploaded (46 MB); 6 min of 40","errors":[],"stage":"pack"},{"job":"collect-ca3-v4-joblog-20261006","kind":"collect","title":"Counter ASIC 3.0 gate run (hash): the full job.log of run-ca3-v4-gates-pc2-20261006 (PC 2)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T15:57:16Z","finished_at":"2026-10-06T15:57:16Z","summary":"0 files uploaded","errors":[],"stage":null},{"job":"run-ca3-v4-gates-pc2-20261006","kind":"run","title":"Counter ASIC 3.0 gate run (hash): G1 and G2 on the RTX 5090 for mx8+sh256x27 (PC 2)","status":"done","ok":true,"exit":0,"secs":26,"started_at":"2026-10-06T15:52:23Z","finished_at":"2026-10-06T15:52:49Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"fetch-ca3-v4-20261006","kind":"fetch","title":"Counter ASIC 3.0 gate run (hash): the class v4 packs (PC 2)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T15:52:21Z","finished_at":"2026-10-06T15:52:21Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-v4-20261006\\fetch-ca3-v4-20261006 (497673 bytes, sha256 ok), extracted into C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\fetch-ca3-v4-20261006","errors":[],"stage":null},{"job":"update-now-0313-switch-1ccfe586","kind":"update-now","title":"Exec restart: re-read the manifest (exec_restart_number 27276, exec_restart_hash, exec_restart_trust_daa 200000) and restart the node with it","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T15:37:01Z","finished_at":"2026-10-06T15:37:01Z","summary":"update check and install asked","errors":[],"stage":null},{"job":"update-now-0313-1ccfe586","kind":"update-now","title":"0.3.13 is published: re-read the manifest and install now (the execution layer follows again, the finality route fix)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T15:25:54Z","finished_at":"2026-10-06T15:25:54Z","summary":"update check and install asked","errors":[],"stage":null},{"job":"floor-build-7","kind":"run","title":"prover floor: build patched sp1-gpu-server, run 7 (patch v5: abort on panic, the stage named; CPU only; live server untouched)","status":"done","ok":true,"exit":0,"secs":125,"started_at":"2026-10-06T13:02:39Z","finished_at":"2026-10-06T13:04:44Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"floor-pc2-verify","kind":"run","title":"prover floor: the packaged server's verification run (manifest check, tampered copy refused, install into ~/.sp1/bin, one shard on the 12 GB profile, live server put back)","status":"done","ok":true,"exit":0,"secs":58,"started_at":"2026-10-06T12:50:53Z","finished_at":"2026-10-06T12:51:51Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"floor-pc2-manifest","kind":"fetch","title":"prover floor: the signed manifest for PC 2's v4 server","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T12:49:37Z","finished_at":"2026-10-06T12:49:37Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\floor-pc2-manifest\\floor-pc2-manifest (750 bytes, sha256 ok), extracted into C:\\Users\\Admin\\AppData\\Local\\igneum\\app\\jobs\\floor-pc2-manifest","errors":[],"stage":null},{"job":"bench-a-pc2","kind":"run","title":"PC 2: proof-system bench job A (SP1 stock and patched, v1 and empty shard, beside the miner; prover off then on)","status":"done","ok":true,"exit":0,"secs":379,"started_at":"2026-10-06T11:59:46Z","finished_at":"2026-10-06T12:06:05Z","summary":"script finished, exit 0","errors":[],"stage":null},{"job":"fetch-bench-ps","kind":"fetch","title":"PC 2: the proof-system bench kit (run.sh and two fixtures)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T11:58:31Z","finished_at":"2026-10-06T11:58:31Z","summary":"C:\\Users\\Admin\\AppData\\Local\\igneum\\prove\\bench-ps.zip (36598 bytes, sha256 ok), extracted into C:\\Users\\Admin\\AppData\\Local\\igneum\\prove\\bench-ps","errors":[],"stage":null},{"job":"update-now-0312-switch-1ccfe586","kind":"update-now","title":"Proving v1 fresh-record rule: re-read the manifest (proving_v1_fresh_rule_daa 198000) and restart the node with it","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T11:41:23Z","finished_at":"2026-10-06T11:41:23Z","summary":"update check and install asked","errors":[],"stage":null},{"job":"update-now-0312-1ccfe586","kind":"update-now","title":"0.3.12 is published: re-read the manifest and install now (the fresh-record rule behind its height, the segment-aligned prover, the GPU list in performance order, Ember Tune, Power control off)","status":"done","ok":true,"exit":0,"secs":0,"started_at":"2026-10-06T11:33:28Z","finished_at":"2026-10-06T11:33:28Z","summary":"update check and install asked","errors":[],"stage":null}],"last_report_at":"2026-10-06T17:46:48Z"}]},"sources":{"mac":{"ok":true,"at":"2026-10-06T17:49:20Z"},"pcs":{"ok":true,"at":"2026-10-06T17:49:21Z"},"box":{"ok":true,"at":"2026-10-06T17:48:54Z","age_s":27}},"baselines":{"node-linux":{"secs":690,"source":"approximate: infra/cross/build-linux.sh on this Mac, 8 to 15 min per target at nice 19 with 4 to 6 jobs (docs/plans/build-job.md, 4 Oct 2026)"},"node-windows":{"secs":690,"source":"approximate: proto-cuda/windows-node/cross-build.sh on this Mac, the same 8 to 15 min (docs/plans/build-job.md, 4 Oct 2026)"}},"headline":{"clean_node":{"box_s":87,"measured_at":"2026-10-06T17:32:28Z","note":"cargo build --release -p kaspad -p igneum-miner --features kaspad/igneum-pow, fork 3bfe346f, cold target dir and cold sccache (0 hits / 993 misses), 563 crates, -j 90. Mac not measured on this tree; the logs hold 12 min 36 s to 17 min 53 s for other commits at -j 4 under load."},"incremental_under_load":{"box_s":7,"measured_at":"2026-10-06T17:33:20Z","note":"one line changed in kaspad/src/main.rs (uncommitted, carried by the overlay), same target dir, 1 crate compiled and igneumd relinked; box load 12 from the clean build a minute earlier, no other build. Mac not measured on this tree; logs: 2 min 08 s to 15 min 18 s for other commits under load."},"windows_cross":{"box_s":104,"measured_at":"2026-10-06T17:35:44Z","note":"cargo build --release -p kaspad -p igneum-miner --features igneum-pow --target x86_64-pc-windows-gnu, mingw GCC 13 posix, cold (995 compiles); 8 s on a repeat. Mac not measured on this tree; logs: 4 min 49 s warm to 12 min 28 s for other commits."}}} \ No newline at end of file diff --git a/tools/workers/launchd/com.igneum.workers-push.plist b/tools/workers/launchd/com.igneum.workers-push.plist new file mode 100644 index 00000000..dfe3f01a --- /dev/null +++ b/tools/workers/launchd/com.igneum.workers-push.plist @@ -0,0 +1,16 @@ + + + + + Labelcom.igneum.workers-push + ProgramArguments/opt/homebrew/bin/node__ROOT__/tools/workers/push.mjs + StartInterval60 + RunAtLoad + WorkingDirectory__ROOT__ + EnvironmentVariablesPATH/opt/homebrew/bin:/usr/local/bin:/usr/bin:/binHOME__HOME__ + StandardOutPath__HOME__/Library/Logs/igneum-workers-push.log + StandardErrorPath__HOME__/Library/Logs/igneum-workers-push.log + Nice10 + ProcessTypeBackground + diff --git a/tools/workers/launchd/install.sh b/tools/workers/launchd/install.sh new file mode 100755 index 00000000..67fe6fce --- /dev/null +++ b/tools/workers/launchd/install.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Install or refresh the Mac pusher's launchd agent (every 60 s). Run from any checkout: the plist points at the MAIN +# checkout's tools/workers/push.mjs (/Users/joshm/Projects/igneum) unless IGNEUM_WORKERS_ROOT names another, so the agent +# keeps working after this worktree is removed. +# tools/workers/launchd/install.sh install and (re)load +# tools/workers/launchd/install.sh --remove unload and delete +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="${IGNEUM_WORKERS_ROOT:-/Users/joshm/Projects/igneum}" +LABEL=com.igneum.workers-push; DEST="$HOME/Library/LaunchAgents/$LABEL.plist" +if [ "${1:-}" = --remove ]; then launchctl bootout "gui/$(id -u)/$LABEL" 2>/dev/null || true; rm -f "$DEST"; echo "removed $LABEL"; exit 0; fi +[ -f "$ROOT/tools/workers/push.mjs" ] || { echo "no $ROOT/tools/workers/push.mjs (merge workers-dash to master first, or set IGNEUM_WORKERS_ROOT)" >&2; exit 1; } +mkdir -p "$HOME/Library/LaunchAgents" "$HOME/Library/Logs" +sed -e "s#__ROOT__#$ROOT#g" -e "s#__HOME__#$HOME#g" "$HERE/$LABEL.plist" > "$DEST" +plutil -lint "$DEST" >/dev/null +launchctl bootout "gui/$(id -u)/$LABEL" 2>/dev/null || true +launchctl bootstrap "gui/$(id -u)" "$DEST" +launchctl kickstart -k "gui/$(id -u)/$LABEL" +sleep 3; launchctl print "gui/$(id -u)/$LABEL" | grep -E 'state|last exit|runs' | head -4; tail -2 "$HOME/Library/Logs/igneum-workers-push.log" | sed 's/dl\/[A-Za-z0-9_-]\{8,\}/dl\//g' diff --git a/tools/workers/lib.mjs b/tools/workers/lib.mjs new file mode 100644 index 00000000..71f4ad55 --- /dev/null +++ b/tools/workers/lib.mjs @@ -0,0 +1,221 @@ +// Parsers shared by tools/workers/collect.mjs (on igneum-build-1) and tools/workers/push.mjs (on the Mac). Pure functions +// over text the machine gives us (/proc files, lock files, the builds JSONL), so node --test can run them against the +// fixtures in tools/workers/fixtures without a box. Every reader here takes the machine's own facts, never a reported number. + +/// /proc/stat -> { total: [{user, nice, system, idle, iowait, irq, softirq, steal}], cores: [...] } (jiffies). +export function parseProcStat(text) { + const cores = []; let total = null; + for (const line of String(text).split('\n')) { + if (!line.startsWith('cpu')) continue; + const f = line.trim().split(/\s+/); + const n = f.slice(1, 9).map(Number); + const row = { user: n[0], nice: n[1], system: n[2], idle: n[3], iowait: n[4] || 0, irq: n[5] || 0, softirq: n[6] || 0, steal: n[7] || 0 }; + if (f[0] === 'cpu') total = row; else cores.push(row); + } + return { total, cores }; +} + +/// Busy percentage between two /proc/stat samples: 0 to 100 for the whole box and per core, in core order. +export function cpuBusy(before, after) { + const pct = (a, b) => { + if (!a || !b) return 0; + const sum = r => r.user + r.nice + r.system + r.idle + r.iowait + r.irq + r.softirq + r.steal; + const idle = r => r.idle + r.iowait; + const dt = sum(b) - sum(a); const di = idle(b) - idle(a); + if (dt <= 0) return 0; + return Math.max(0, Math.min(100, Math.round((1 - di / dt) * 1000) / 10)); + }; + const n = Math.min(before.cores.length, after.cores.length); + const per = []; for (let i = 0; i < n; i++) per.push(pct(before.cores[i], after.cores[i])); + return { busy_pct: pct(before.total, after.total), per_core: per, cores_busy: per.filter(p => p >= 50).length }; +} + +/// /proc/meminfo -> kB numbers and a used percentage (used = total minus available, what a build can still take). +export function parseMeminfo(text) { + const m = {}; for (const line of String(text).split('\n')) { const r = /^(\w+):\s+(\d+)/.exec(line); if (r) m[r[1]] = Number(r[2]); } + const total = m.MemTotal || 0, avail = m.MemAvailable ?? m.MemFree ?? 0; + return { total_kb: total, available_kb: avail, used_kb: Math.max(0, total - avail), used_pct: total ? Math.round(((total - avail) / total) * 1000) / 10 : 0, swap_total_kb: m.SwapTotal || 0, swap_free_kb: m.SwapFree || 0 }; +} + +/// /proc/net/dev -> [{ iface, rx_bytes, tx_bytes }] without lo. +export function parseNetDev(text) { + const out = []; + for (const line of String(text).split('\n').slice(2)) { + const r = /^\s*([^:\s]+):\s*(.*)$/.exec(line); if (!r) continue; + const f = r[2].trim().split(/\s+/).map(Number); + if (r[1] === 'lo') continue; + out.push({ iface: r[1], rx_bytes: f[0] || 0, tx_bytes: f[8] || 0 }); + } + return out; +} + +/// /proc/loadavg -> { load: [1, 5, 15], running, threads }. +export function parseLoadavg(text) { + const f = String(text).trim().split(/\s+/); + const rq = (f[3] || '0/0').split('/'); + return { load: [Number(f[0]) || 0, Number(f[1]) || 0, Number(f[2]) || 0], running: Number(rq[0]) || 0, threads: Number(rq[1]) || 0 }; +} + +/// The slot-file line both with-lock.sh and bs_remote_run write: "pid N since HH:MM:SSZ waited S s: