igneum/tools/build-job.mjs

#!/usr/bin/env node
// Mac side of the `build` job (app/igneum-app/src/jobbuild.rs): a PC builds the node and the app engine for Linux
// and Windows inside its WSL2 Ubuntu and sends the binaries to the relay; this tool packs the sources, publishes
// the signed job, watches the report, brings the binaries back, checks every sha256 and the Windows PE headers,
// and puts them where the packaging scripts look.
//   node tools/build-job.mjs run     [--node vendor/igneum-node-v4] [--target ae432dc7] [--budget-minutes 40]
//                                    [--stage-minutes '{"linux":20}'] [--targets linux,windows] [--no-tests]
//                                    [--title "..."] [--no-place] [--out dir]        pack + publish + watch + fetch
//   node tools/build-job.mjs publish [the same flags]                                pack + publish, prints the id
//   node tools/build-job.mjs watch <job id>                                          STAGE and RESULT lines as they land
//   node tools/build-job.mjs fetch <job id> [--out dir] [--no-place]                 download, zstd -d, sha256, PE check, place
//   node tools/build-job.mjs verify <exe...>                                         the PE check alone
// Where the binaries go (--no-place keeps them in --out only):
//   igneumd.exe, igneum-miner.exe -> $IGNEUM_WIN_RELEASE or vendor/igneum-node/target-integration/x86_64-pc-windows-gnu/release/
//                                    (packaging/windows/push-inputs.sh and make-payload.sh read them there)
//   igneum-app.exe                -> app/igneum-app/target/x86_64-pc-windows-gnu/release/ (make-payload.sh's IGNEUM_APP_EXE default)
//   igneumd, igneum-miner, igneum-app (Linux) -> infra/cross/out/ with version.txt (where infra/cross/build-linux.sh leaves them)
// Reads ~/.config/igneum/env (DATABASE_URL, the intake, as tools/jobs.mjs), relay-token, log-intake-key, dl-token.
// Needs zstd and python3 on the PATH. No dependencies.
import { readFileSync, writeFileSync, mkdirSync, existsSync, copyFileSync, chmodSync, statSync } from 'node:fs';
import { homedir, tmpdir } from 'node:os';
import { join, resolve, dirname, basename } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createHash } from 'node:crypto';
import { spawnSync } from 'node:child_process';

process.stdout.on('error', e => { if (e.code === 'EPIPE') process.exit(0); throw e; });
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
const cfg = n => { try { return readFileSync(join(homedir(), '.config', 'igneum', n), 'utf8').trim(); } catch { return ''; } };
const RELAY_BASE = (cfg('relay-url') || 'https://relay.igneum.network').replace(/\/$/, '');

const argv = process.argv.slice(2);
const flags = {}; const pos = [];
const BOOL = new Set(['no-tests', 'no-place', 'no-deploy', 'help']);
for (let i = 0; i < argv.length; i++) {
  const a = argv[i];
  if (a.startsWith('--')) { const k = a.slice(2); const next = argv[i + 1]; if (!BOOL.has(k) && next !== undefined && !next.startsWith('--')) { flags[k] = next; i++; } else flags[k] = true; }
  else pos.push(a);
}
const cmd = pos[0] || 'help';
const usage = () => { console.log(readFileSync(fileURLToPath(import.meta.url), 'utf8').split('\n').slice(1, 20).map(l => l.replace(/^\/\/ ?/, '')).join('\n')); };

// ---- the intake (Neon HTTP SQL, as tools/jobs.mjs and tools/logs.mjs) ----------------------------------------------------
function db() {
  const m = /^DATABASE_URL=(.*)$/m.exec(cfg('env'));
  if (!m) { console.error('DATABASE_URL not found in ~/.config/igneum/env'); process.exit(1); }
  const url = m[1].trim().replace(/^['"]|['"]$/g, '');
  const host = new URL(url).hostname.replace('-pooler', '');
  return async (query, params = []) => {
    const r = await fetch(`https://${host}/sql`, { method: 'POST', headers: { 'Neon-Connection-String': url, 'Content-Type': 'application/json' }, body: JSON.stringify({ query, params }) });
    const j = await r.json();
    if (!r.ok) throw new Error(j.message || JSON.stringify(j));
    return j.rows;
  };
}
const summaryOf = lines => { const l = (lines || '').split('\n')[0]; if (!l.startsWith('SUMMARY ')) return null; try { return JSON.parse(l.slice(8)); } catch { return null; } };
const FINAL = new Set(['done', 'failed', 'timeout', 'aborted']);

/// The newest report per machine for the job: { machine, summary, lines[] }.
async function reports(sql, id) {
  const rows = await sql(`SELECT DISTINCT ON (run_id) run_id, machine, received_at, lines FROM miner_logs WHERE run_id LIKE $1 AND label LIKE 'job-%' ORDER BY run_id, received_at DESC`, [`job-${id}-%`]);
  return rows.map(r => ({ run_id: r.run_id, machine: r.machine, received_at: r.received_at, summary: summaryOf(r.lines), lines: (r.lines || '').split('\n') }));
}

async function watch(id, quiet = false) {
  const sql = db();
  const seen = new Set();
  let waited = 0;
  for (;;) {
    const rs = await reports(sql, id);
    for (const r of rs) {
      for (const l of r.lines) {
        const t = l.trimEnd();
        if (!/^(STAGE|RESULT|BUILD FAILED)/.test(t) || seen.has(t)) continue;
        seen.add(t); if (!quiet) console.log(`${r.machine}: ${t}`);
      }
    }
    const finals = rs.filter(r => r.summary && FINAL.has(r.summary.status));
    if (rs.length && finals.length === rs.length) {
      for (const r of finals) console.log(`${r.machine}: ${r.summary.status} exit ${r.summary.exit} after ${r.summary.duration_s} s: ${r.summary.summary}`);
      return finals;
    }
    waited += 20;
    if (!quiet && waited % 120 === 0) console.log(`${rs.length ? 'still running' : 'nothing from the PC yet (it polls the jobs file every 10 min)'}; ${Math.floor(waited / 60)} min waited`);
    await new Promise(r => setTimeout(r, 20000));
  }
}

// ---- the relay (file download by item id, feed search by title) ------------------------------------------------------
function relayApi() {
  const token = cfg('relay-token'); const key = cfg('log-intake-key');
  if (!token && !key) { console.error('no ~/.config/igneum/relay-token and no log-intake-key'); process.exit(1); }
  const api = `${RELAY_BASE}/r/${token || '-'}/api/`;
  const headers = key ? { 'x-igneum-key': key } : {};
  return {
    async get(fn, q) { const r = await fetch(api + fn + (q ? '?' + new URLSearchParams(q) : ''), { headers }); const j = await r.json(); if (!r.ok || j.ok === false) throw new Error(j.error || `http ${r.status}`); return j; },
    async download(id, to) { const r = await fetch(`${api}file?id=${id}`, { headers, redirect: 'follow' }); if (!r.ok) throw new Error(`download of relay item ${id}: http ${r.status}`); writeFileSync(to, Buffer.from(await r.arrayBuffer())); return statSync(to).size; },
  };
}
const sha256 = p => createHash('sha256').update(readFileSync(p)).digest('hex');

/// A Windows exe: MZ, a PE signature, x86-64, a .text section, at least 1 MB. Prints what it finds.
export function peCheck(path, minBytes = 1024 * 1024) {
  const d = readFileSync(path);
  const problems = [];
  if (d.length < minBytes) problems.push(`only ${d.length} bytes (under ${minBytes})`);
  if (d.length < 0x40 || d.toString('latin1', 0, 2) !== 'MZ') { problems.push('no MZ header'); return { ok: false, problems }; }
  const pe = d.readUInt32LE(0x3c);
  if (pe + 24 > d.length || d.toString('latin1', pe, pe + 4) !== 'PE\0\0') { problems.push('no PE signature'); return { ok: false, problems }; }
  const machine = d.readUInt16LE(pe + 4);
  if (machine !== 0x8664) problems.push(`machine 0x${machine.toString(16)} is not x86-64 (0x8664)`);
  const nsec = d.readUInt16LE(pe + 6); const optSize = d.readUInt16LE(pe + 20);
  const magic = d.readUInt16LE(pe + 24);
  if (magic !== 0x20b) problems.push(`optional header magic 0x${magic.toString(16)} is not PE32+`);
  const subsystem = optSize >= 70 ? d.readUInt16LE(pe + 24 + 68) : 0;
  const names = [];
  for (let i = 0; i < nsec; i++) { const s = pe + 24 + optSize + 40 * i; if (s + 8 > d.length) break; names.push(d.toString('latin1', s, s + 8).replace(/\0+$/, '')); }
  if (!names.includes('.text')) problems.push('no .text section');
  return { ok: !problems.length, problems, bytes: d.length, machine, subsystem, sections: names };
}

// ---- fetch: the outputs of a finished job -----------------------------------------------------------------------------------
async function fetchOutputs(id, finals) {
  const out = resolve(flags.out || join(tmpdir(), 'igneum-build-job', id));
  mkdirSync(out, { recursive: true });
  const relay = relayApi();
  // the SUMMARY carries outputs[] with relay ids; the feed is the fallback (titles "build-job <id> <file>")
  let outputs = [];
  let meta = {};
  for (const f of finals || []) { if (f.summary && Array.isArray(f.summary.outputs) && f.summary.outputs.length) { outputs = f.summary.outputs; meta = f.summary; break; } }
  if (!outputs.length) {
    const feed = await relay.get('feed', { limit: 500 });
    for (const it of feed.items || []) {
      const m = new RegExp(`^build-job ${id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')} (.+)$`).exec(it.title || '');
      if (!m || !it.has_file || m[1] === 'build-outputs.json') continue;
      const body = it.body || '';
      const unpacked = /^(linux|windows) (\S+) (\d+) bytes sha256 ([0-9a-f]{64})$/m.exec(body);
      const packed = /^(\S+\.zst) (\d+) bytes sha256 ([0-9a-f]{64})$/m.exec(body);
      outputs.push({ name: unpacked ? unpacked[2] : m[1].replace(/\.zst$/, '').replace(/\.linux$/, ''), target: unpacked ? unpacked[1] : (m[1].endsWith('.exe.zst') ? 'windows' : 'linux'), bytes: unpacked ? Number(unpacked[3]) : 0, sha256: unpacked ? unpacked[4] : '', zst: m[1], zst_bytes: packed ? Number(packed[2]) : it.size, zst_sha256: packed ? packed[3] : '', relay_id: it.id });
      const nb = /^node (\S+) (\S+)$/m.exec(body); if (nb) { meta.node_branch = nb[1]; meta.node_commit = nb[2]; }
      const av = /^app (\S+)$/m.exec(body); if (av) meta.app_version = av[1];
    }
  }
  if (!outputs.length) { console.error(`no outputs for job ${id}: nothing in the SUMMARY and nothing titled "build-job ${id} ..." on the relay`); process.exit(1); }
  const placed = []; let bad = 0;
  for (const o of outputs) {
    if (!o.relay_id) { console.log(`${o.zst}: not uploaded (no relay item); skipped`); bad++; continue; }
    const zst = join(out, o.zst);
    const got = await relay.download(o.relay_id, zst);
    if (o.zst_sha256 && sha256(zst) !== o.zst_sha256) { console.log(`${o.zst}: sha256 of the download does not match the RESULT line (${got} bytes); refused`); bad++; continue; }
    const plain = join(out, o.target === 'windows' ? o.name : `${o.name}`);
    const r = spawnSync('zstd', ['-d', '-q', '-f', zst, '-o', plain], { stdio: 'inherit' });
    if (r.status !== 0) { console.log(`${o.zst}: zstd -d failed (exit ${r.status}); is zstd installed? (brew install zstd)`); bad++; continue; }
    const sum = sha256(plain);
    if (o.sha256 && sum !== o.sha256) { console.log(`${o.name} (${o.target}): sha256 after decompression ${sum.slice(0, 16)}... is not the PC's ${o.sha256.slice(0, 16)}...; refused`); bad++; continue; }
    if (!o.sha256) console.log(`${o.name} (${o.target}): WARNING no sha256 from the PC for this file (no SUMMARY and no body on the relay item); only the download and the PE check stand`);
    if (o.target === 'linux') { try { chmodSync(plain, 0o755); } catch {} }
    let note = '';
    if (o.target === 'windows') {
      const pe = peCheck(plain);
      if (!pe.ok) { console.log(`${o.name}: PE check FAILED: ${pe.problems.join('; ')}`); bad++; continue; }
      note = ` PE ok (${pe.sections.join(' ')}, subsystem ${pe.subsystem})`;
      if (o.name === 'igneum-app.exe') {
        const v = spawnSync('python3', [join(ROOT, 'packaging/windows/resources/verify-exe.py'), ...(meta.app_version ? ['--version', meta.app_version] : []), plain], { encoding: 'utf8' });
        if (v.status !== 0) { console.log(`${o.name}: verify-exe.py FAILED:\n${(v.stdout || '') + (v.stderr || '')}`); bad++; continue; }
        note += ', coin icon and version block ok';
      }
    }
    console.log(`${o.name} (${o.target}): ${statSync(plain).size} bytes, sha256 ${sum.slice(0, 16)}...${o.sha256 ? ' matches the PC' : ' (unchecked)'}${note}`);
    if (!flags['no-place']) {
      const dest = placeFor(o);
      mkdirSync(dirname(dest), { recursive: true });
      copyFileSync(plain, dest);
      if (o.target === 'linux') { try { chmodSync(dest, 0o755); } catch {} }
      placed.push(dest);
    }
  }
  if (placed.some(p => p.startsWith(join(ROOT, 'infra/cross/out')))) {
    const v = `igneumd ${meta.node_commit || ''}\nbuilt by build job ${id} on a PC (WSL2, native x86_64 Linux) from ${meta.node_commit || '?'} (${meta.node_branch || '?'}), app ${meta.app_version || '?'}\n${meta.node_commit || ''}\n`;
    writeFileSync(join(ROOT, 'infra/cross/out/version.txt'), v);
    placed.push(join(ROOT, 'infra/cross/out/version.txt'));
  }
  console.log(`\n${outputs.length - bad} of ${outputs.length} outputs verified${bad ? `, ${bad} refused` : ''}; downloads in ${out}`);
  for (const p of placed) console.log(`  placed ${p.replace(ROOT + '/', '')}`);
  if (placed.length) console.log('Next: packaging/windows/push-inputs.sh (the Windows payload inputs), then the installer round; Linux binaries: infra/cross/out/');
  return bad === 0;
}

function placeFor(o) {
  if (o.target === 'windows') {
    if (o.name === 'igneum-app.exe') return join(ROOT, 'app/igneum-app/target/x86_64-pc-windows-gnu/release/igneum-app.exe');
    return join(process.env.IGNEUM_WIN_RELEASE || join(ROOT, 'vendor/igneum-node/target-integration/x86_64-pc-windows-gnu/release'), o.name);
  }
  return join(ROOT, 'infra/cross/out', o.name);
}

// ---- publish: pack, then the signed job ---------------------------------------------------------------------------------------
function publish() {
  const pack = ['packaging/windows/push-build-inputs.sh'];
  if (flags.node) pack.push('--node', flags.node);
  if (flags['no-deploy']) pack.push('--no-deploy');
  console.log(`$ ${pack.join(' ')}`);
  const r = spawnSync('bash', pack, { cwd: ROOT, stdio: 'inherit' });
  if (r.status !== 0) { console.error('push-build-inputs.sh failed'); process.exit(1); }
  const add = ['packaging/ota/publish-jobs.sh', 'add', '--kind', 'build', '--target', flags.target || 'ae432dc7'];
  if (flags['budget-minutes']) add.push('--budget-minutes', String(flags['budget-minutes']));
  if (flags['stage-minutes']) add.push('--stage-minutes', String(flags['stage-minutes']));
  if (flags.targets) add.push('--targets', String(flags.targets));
  if (flags['no-tests']) add.push('--no-tests');
  if (flags['min-free-gb']) add.push('--min-free-gb', String(flags['min-free-gb']));
  if (flags.title) add.push('--title', String(flags.title));
  if (flags.id) add.push('--id', String(flags.id));
  if (!flags['no-deploy']) add.push('--deploy');
  console.log(`$ ${add.join(' ')}`);
  const a = spawnSync('bash', add, { cwd: ROOT, encoding: 'utf8' });
  process.stdout.write(a.stdout || ''); process.stderr.write(a.stderr || '');
  if (a.status !== 0) { console.error('publish-jobs.sh failed'); process.exit(1); }
  const m = /^added: (\S+) build/m.exec(a.stderr || '');
  if (!m) { console.error('could not read the job id from the publisher output'); process.exit(1); }
  console.log(`job ${m[1]} published${flags['no-deploy'] ? ' (not deployed)' : ''}`);
  return m[1];
}

if (cmd === 'help' || flags.help) { usage(); process.exit(0); }
if (cmd === 'verify') {
  if (pos.length < 2) { console.error('verify <exe...>'); process.exit(1); }
  let ok = true;
  for (const p of pos.slice(1)) { const r = peCheck(resolve(p)); console.log(`${p}: ${r.ok ? `ok, ${r.bytes} bytes, sections ${r.sections.join(' ')}, subsystem ${r.subsystem}` : 'FAIL: ' + r.problems.join('; ')}`); ok = ok && r.ok; }
  process.exit(ok ? 0 : 1);
}
if (cmd === 'publish') { publish(); process.exit(0); }
if (cmd === 'watch') { if (!pos[1]) { console.error('watch <job id>'); process.exit(1); } await watch(pos[1]); process.exit(0); }
if (cmd === 'fetch') { if (!pos[1]) { console.error('fetch <job id>'); process.exit(1); } const finals = await reports(db(), pos[1]).then(rs => rs.filter(r => r.summary && FINAL.has(r.summary.status))); process.exit((await fetchOutputs(pos[1], finals)) ? 0 : 1); }
if (cmd === 'run') {
  const id = publish();
  if (flags['no-deploy']) { console.log('not deployed, so nothing to watch'); process.exit(0); }
  console.log(`watching job ${id} (the PC polls every 10 minutes; Settings > remote jobs > Check now on the PC runs it at once)`);
  const finals = await watch(id);
  if (!finals.some(f => f.summary.status === 'done' || (f.summary.outputs || []).length)) { console.error('the job produced nothing to fetch'); process.exit(1); }
  process.exit((await fetchOutputs(id, finals)) ? 0 : 1);
}
console.error(`unknown command ${cmd}`); usage(); process.exit(2);