igneum/tools/ci/red-watch.mjs

#!/usr/bin/env node
// The red-master watcher. One line per failed master or release-* run, so nobody opens the Actions page to learn CI is red.
// Node 22, standard library only.
//
//   node tools/ci/red-watch.mjs record --file <red.jsonl>            in the workflow's `red` job (runs on igneum-build-1 after a
//                                                                     failed run): reads the run from the GitHub environment and
//                                                                     the failed jobs and steps from the API with the job's own
//                                                                     token, appends ONE JSON line for this run id (idempotent)
//   node tools/ci/red-watch.mjs post --file <red.jsonl> [--live]      on the box, every minute as `build` (igneum-ci-red.timer):
//                                                                     every recorded run not yet posted goes as one line to the
//                                                                     hidden updates channel (DISCORD_WEBHOOK_UPDATES in the
//                                                                     credentials file), then is marked posted in the state file;
//                                                                     without --live the line is printed, not sent
//   node tools/ci/red-watch.mjs --self-test                           record twice = one line; post = one send; post again = none
//
// Files: the record file is written by the runner user (one object per line: run_id, workflow, branch, sha, title, failed,
// url, at); the poster's state (which run ids were posted, when) is $IGNEUM_CI_RED_STATE, default
// ~/.config/igneum/ci-red-posted.json, so the two users never write the same file. Credentials: $IGNEUM_DISCORD_ENV
// (default ~/.config/igneum/discord), KEY=VALUE lines, mode 600, never printed: a webhook URL never appears in any output,
// only the key's name. The orchestrator reads the record file (ssh build@<box> cat /srv/ci-red/red.jsonl) or the channel.
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';

const args = process.argv.slice(2);
const flag = (name) => { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; };
const has = (name) => args.includes(name);
const CRED_FILE = process.env.IGNEUM_DISCORD_ENV || path.join(os.homedir(), '.config', 'igneum', 'discord');
const STATE_FILE = process.env.IGNEUM_CI_RED_STATE || path.join(os.homedir(), '.config', 'igneum', 'ci-red-posted.json');
const WEBHOOK_KEY = 'DISCORD_WEBHOOK_UPDATES';

export function readLines(file) {
  if (!fs.existsSync(file)) return [];
  return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
}

export function runFromEnv(env = process.env) {
  const need = ['GITHUB_RUN_ID', 'GITHUB_REPOSITORY', 'GITHUB_REF_NAME', 'GITHUB_SHA', 'GITHUB_WORKFLOW'];
  for (const k of need) if (!env[k]) throw new Error(`record: ${k} is not set (this command runs inside a GitHub Actions job)`);
  const server = env.GITHUB_SERVER_URL || 'https://github.com';
  return {
    run_id: String(env.GITHUB_RUN_ID), attempt: Number(env.GITHUB_RUN_ATTEMPT || 1), workflow: env.GITHUB_WORKFLOW,
    branch: env.GITHUB_REF_NAME, sha: env.GITHUB_SHA.slice(0, 7), event: env.GITHUB_EVENT_NAME || '',
    url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`, at: new Date().toISOString(),
  };
}

// The failed jobs and their first failed step, from the run's jobs API with the job's own token. The `red` job itself
// (the caller) is skipped by name. Any API trouble gives an empty list and a note, never a thrown error: the line is
// the thing that must land.
export async function failedJobs(env = process.env, fetchImpl = fetch) {
  const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; const id = env.GITHUB_RUN_ID;
  const api = env.GITHUB_API_URL || 'https://api.github.com';
  if (!token) return { failed: [], note: 'no GITHUB_TOKEN; failed steps not read' };
  try {
    const r = await fetchImpl(`${api}/repos/${repo}/actions/runs/${id}/jobs?per_page=100`, {
      headers: { Authorization: `Bearer ${token}`, Accept: 'application/vnd.github+json', 'User-Agent': 'igneum-red-watch' },
    });
    if (!r.ok) return { failed: [], note: `jobs API ${r.status}` };
    const j = await r.json();
    const failed = [];
    for (const job of j.jobs || []) {
      if (job.name === (env.GITHUB_JOB_NAME || 'red watcher') || /^red watcher/.test(job.name)) continue;
      if (job.conclusion === 'success' || job.conclusion === 'skipped' || job.conclusion === null) continue;
      const step = (job.steps || []).find((s) => s.conclusion && s.conclusion !== 'success' && s.conclusion !== 'skipped');
      const zeroSteps = !(job.steps || []).length;
      failed.push({ job: job.name, conclusion: job.conclusion, step: step ? step.name : (zeroSteps ? '(job never started: runner or billing)' : '(no step)') });
    }
    return { failed, note: '' };
  } catch (e) {
    return { failed: [], note: `jobs API: ${e.message}` };
  }
}

export async function record(file, env = process.env, fetchImpl = fetch, title = '') {
  const run = runFromEnv(env);
  const existing = readLines(file);
  if (existing.some((l) => l.run_id === run.run_id && l.attempt === run.attempt)) {
    return { written: false, run };   // one line per run attempt, however many times the job is re-run or retried
  }
  const { failed, note } = await failedJobs(env, fetchImpl);
  const line = { ...run, title: (title || env.RED_WATCH_TITLE || '').slice(0, 100), failed, note };
  fs.mkdirSync(path.dirname(file), { recursive: true });
  fs.appendFileSync(file, JSON.stringify(line) + '\n');
  return { written: true, run: line };
}

export function formatLine(l) {
  const where = l.failed.length ? l.failed.map((f) => `${f.job.replace(/,.*$/, '')} at "${f.step}"`).join('; ') : (l.note || 'no step detail');
  const title = l.title ? ` "${l.title}"` : '';
  return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}: ${where} ${l.url}`;
}

function readCredentials(file) {
  if (!fs.existsSync(file)) return {};
  const out = {};
  for (const raw of fs.readFileSync(file, 'utf8').split('\n')) {
    const line = raw.trim(); if (!line || line.startsWith('#')) continue;
    const i = line.indexOf('='); if (i < 0) continue;
    out[line.slice(0, i).trim()] = line.slice(i + 1).trim();
  }
  return out;
}

function readState(file) { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return { posted: {} }; } }
function writeState(file, state) { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, JSON.stringify(state, null, 1) + '\n', { mode: 0o600 }); }

export async function post(file, { live = false, stateFile = STATE_FILE, credFile = CRED_FILE, fetchImpl = fetch, log = console.log } = {}) {
  const lines = readLines(file);
  const state = readState(stateFile);
  const pending = lines.filter((l) => !state.posted[`${l.run_id}.${l.attempt || 1}`]);
  if (!pending.length) { log(`ci-red: nothing to post (${lines.length} recorded, all posted)`); return { sent: 0, pending: 0 }; }
  const creds = readCredentials(credFile);
  const hook = creds[WEBHOOK_KEY];
  let sent = 0;
  for (const l of pending) {
    const text = formatLine(l);
    if (!live) { log(`ci-red (dry run, not sent): ${text}`); continue; }
    if (!hook) { log(`ci-red: ${WEBHOOK_KEY} is not in the credentials file; ${pending.length} line(s) wait (the line itself is in ${file})`); return { sent: 0, pending: pending.length, missingKey: true }; }
    try {
      const r = await fetchImpl(hook, { method: 'POST', headers: { 'Content-Type': 'application/json', 'User-Agent': 'igneum-red-watch' },
        body: JSON.stringify({ username: 'Igneum CI', content: text.slice(0, 1900), allowed_mentions: { parse: [] } }) });
      if (!r.ok && r.status !== 204) { log(`ci-red: the webhook answered ${r.status} for run ${l.run_id}; retried next tick`); continue; }
      state.posted[`${l.run_id}.${l.attempt || 1}`] = new Date().toISOString(); sent += 1;
      log(`ci-red: posted run ${l.run_id} (${l.workflow} on ${l.branch} @${l.sha})`);
    } catch (e) {
      log(`ci-red: send failed for run ${l.run_id}: ${e.message.replace(/https?:\/\/\S+/g, '<url>')}; retried next tick`);
    }
  }
  if (live) writeState(stateFile, state);
  return { sent, pending: pending.length - sent };
}

async function selfTest() {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'red-watch-'));
  const file = path.join(dir, 'red.jsonl'); const stateFile = path.join(dir, 'posted.json'); const credFile = path.join(dir, 'discord');
  const env = { GITHUB_RUN_ID: '424242', GITHUB_RUN_ATTEMPT: '1', GITHUB_REPOSITORY: 'igneum-network/igneum', GITHUB_REF_NAME: 'master',
    GITHUB_SHA: '0f0abc6deadbeef', GITHUB_WORKFLOW: 'ci', GITHUB_TOKEN: 'x', RED_WATCH_TITLE: 'Merge box-work 2bd3bec: the repro script re-stamps its clones' };
  const jobs = { jobs: [
    { name: 'site build, link check, identity grep', conclusion: 'failure', steps: [{ name: 'site build', conclusion: 'success' }, { name: 'identity grep of the public export list', conclusion: 'failure' }] },
    { name: 'igneum-pow tests, igneum-census build', conclusion: 'success', steps: [] },
    { name: 'simulators, quick modes', conclusion: 'failure', steps: [] },
    { name: 'red watcher (master and release-* only)', conclusion: null, steps: [] },
  ] };
  const fakeFetch = async () => ({ ok: true, status: 200, json: async () => jobs });
  const fails = [];
  const a = await record(file, env, fakeFetch); const b = await record(file, env, fakeFetch);
  if (!a.written || b.written) fails.push('record: the second call for the same run wrote a second line');
  const lines = readLines(file);
  if (lines.length !== 1) fails.push(`record: ${lines.length} lines, expected 1`);
  if (lines[0].failed.length !== 2) fails.push(`record: ${lines[0].failed.length} failed jobs, expected 2 (the watcher itself and the green job skipped)`);
  if (lines[0].failed[0].step !== 'identity grep of the public export list') fails.push('record: the failed step was not the first non-success step');
  if (lines[0].failed[1].step !== '(job never started: runner or billing)') fails.push('record: a job with no steps was not named as never started');
  const text = formatLine(lines[0]);
  if (!/^CI red: ci on master @0f0abc6 "Merge box-work/.test(text) || !text.includes('actions/runs/424242')) fails.push(`format: ${text}`);
  // post, dry run: prints, sends nothing, marks nothing
  let printed = []; const log = (s) => printed.push(s);
  const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; };
  await post(file, { live: false, stateFile, credFile, fetchImpl: hookFetch, log });
  if (sends.length !== 0 || !printed.some((s) => s.includes('dry run'))) fails.push('post: the dry run sent or did not print');
  // post, live, no key: says which key is missing, names no URL, sends nothing
  fs.writeFileSync(credFile, 'DISCORD_WEBHOOK_NUMBERS=https://discord.example/api/webhooks/1/secret\n', { mode: 0o600 });
  printed = [];
  const r0 = await post(file, { live: true, stateFile, credFile, fetchImpl: hookFetch, log });
  if (!r0.missingKey || sends.length !== 0 || !printed.some((s) => s.includes(WEBHOOK_KEY))) fails.push('post: a missing updates key was not reported by name');
  if (printed.some((s) => s.includes('secret'))) fails.push('post: a webhook URL leaked into the log');
  // post, live, with the key: one send with the line, then marked posted; a second pass sends nothing
  fs.writeFileSync(credFile, `${WEBHOOK_KEY}=https://discord.example/api/webhooks/2/secret2\n`, { mode: 0o600 });
  printed = [];
  const r1 = await post(file, { live: true, stateFile, credFile, fetchImpl: hookFetch, log });
  if (r1.sent !== 1 || sends.length !== 1 || sends[0].body.content !== text) fails.push(`post: expected one send of the line, got ${sends.length}`);
  if (sends[0].body.allowed_mentions?.parse?.length !== 0) fails.push('post: mentions are not disabled');
  if (printed.some((s) => s.includes('secret2'))) fails.push('post: the webhook URL leaked into the log');
  const r2 = await post(file, { live: true, stateFile, credFile, fetchImpl: hookFetch, log });
  if (r2.sent !== 0 || sends.length !== 1) fails.push('post: the second pass sent the same run again');
  // a failing webhook leaves the run pending for the next tick
  const env2 = { ...env, GITHUB_RUN_ID: '424243' };
  await record(file, env2, fakeFetch);
  const badFetch = async () => ({ ok: false, status: 500 });
  const r3 = await post(file, { live: true, stateFile, credFile, fetchImpl: badFetch, log });
  if (r3.sent !== 0 || r3.pending !== 1) fails.push('post: a 500 from the webhook did not keep the run pending');
  fs.rmSync(dir, { recursive: true, force: true });
  if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); }
  console.log('self-test passed: one line per run however often record runs; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending');
}

const cmd = args[0];
if (cmd === '--self-test') {
  await selfTest();
} else if (cmd === 'record') {
  const file = flag('--file'); if (!file) { console.error('record: --file <red.jsonl> is required'); process.exit(2); }
  const r = await record(file, process.env, fetch, flag('--title') || '');
  console.log(r.written ? `ci-red: recorded ${formatLine(r.run)}` : `ci-red: run ${r.run.run_id} attempt ${r.run.attempt} already recorded`);
} else if (cmd === 'post') {
  const file = flag('--file'); if (!file) { console.error('post: --file <red.jsonl> is required'); process.exit(2); }
  await post(file, { live: has('--live') });
} else {
  console.error('usage: red-watch.mjs record --file <red.jsonl> | post --file <red.jsonl> [--live] | --self-test'); process.exit(2);
}