igneum/tools/ci/red-watch.mjs

#!/usr/bin/env node
// The red watcher. One line per failed run on ANY branch (master and release-* only until 7 October 2026: eight red runs on
// ca3-v4-node went unseen that morning), naming the branch, the commit, the red check and the pushing author, so nobody
// opens the Actions page to learn a branch is red.
// Node 22, standard library only.
//
//   node tools/ci/red-watch.mjs record --file <red.jsonl>            in .github/workflows/ci-red.yml (a workflow_run job on
//                                                                     igneum-build-1 after a failed ci run on any branch): reads the
//                                                                     FAILED run from RED_WATCH_* (the workflow_run payload; the
//                                                                     GITHUB_* variables there describe the watcher's own run) and
//                                                                     the failed jobs and steps from the API with the job's own
//                                                                     token, appends ONE JSON line for that run id (idempotent)
//   node tools/ci/red-watch.mjs post --file <red.jsonl> [--live]      on the box, every minute as `build` (igneum-ci-red.timer):
//                                                                     every recorded run not yet posted goes as one line to the
//                                                                     hidden updates channel (DISCORD_WEBHOOK_UPDATES in the
//                                                                     credentials file), then is marked posted in the state file;
//                                                                     without --live the line is printed, not sent
//   node tools/ci/red-watch.mjs digest --file <red.jsonl> [--live] [--now]
//                                                                     the daily line: at the first pass at or after 09:00 London
//                                                                     (or at once with --now), one line to the updates channel
//                                                                     counting the last 24 h of red rows, CI and box, per class,
//                                                                     with the guard each class has; once per London day
//   node tools/ci/red-watch.mjs tick --file <red.jsonl> [--live]      post, then digest (what igneum-ci-red.timer runs every minute)
//   node tools/ci/red-watch.mjs --self-test                           record twice = one line; post = one send; post again = none;
//                                                                     a box row is counted, never posted alone; the digest once a day
//
// Box rows: infra/build-server/remote-run.sh appends a line with "source":"box" for every red build, suite or check on
// igneum-build-1 (class instant, compile-error, test-failure, no-test-matched, preflight-*, slot-timeout, no-dir, other).
// Those are not posted one by one (an agent iterating red to green would flood the channel); the digest counts them.
//
// Files: the record file is written by the runner user (one object per line: run_id, workflow, branch, sha, title, failed,
// url, at); the poster's state (which run ids were posted, when) is $IGNEUM_CI_RED_STATE, default
// ~/.config/igneum/ci-red-posted.json, so the two users never write the same file. Credentials: $IGNEUM_DISCORD_ENV
// (default ~/.config/igneum/discord), KEY=VALUE lines, mode 600, never printed: a webhook URL never appears in any output,
// only the key's name. The orchestrator reads the record file (ssh build@<box> cat /srv/ci-red/red.jsonl) or the channel.
import fs from 'node:fs';
import path from 'node:path';
import os from 'node:os';

const args = process.argv.slice(2);
const flag = (name) => { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; };
const has = (name) => args.includes(name);
const CRED_FILE = process.env.IGNEUM_DISCORD_ENV || path.join(os.homedir(), '.config', 'igneum', 'discord');
const STATE_FILE = process.env.IGNEUM_CI_RED_STATE || path.join(os.homedir(), '.config', 'igneum', 'ci-red-posted.json');
const WEBHOOK_KEY = 'DISCORD_WEBHOOK_UPDATES';
// what stops each class now (named in the digest so the line teaches, not just counts); docs/analysis/ci-failures-2026-10-06.md
export const GUARDS = {
  'ci': 'the pre-push gate (tools/ci/pre-push.sh: the full gate before a push to master or release-*, the never-push checks before a push to any branch)',
  'instant': 'pre-flight in remote-run.sh (manifest, -p package, feature, subcommand checked in a second) and the kept run log',
  'preflight-manifest': 'refused before the slot: the manifest did not parse',
  'preflight-package': 'refused before the slot: the -p package does not exist',
  'preflight-feature': 'refused before the slot: the feature does not exist on that package',
  'preflight-subcommand': 'refused before the slot: cargo has no such subcommand on the box (provision.sh installs it)',
  'no-test-matched': 'refused after the run: the test filter matched nothing (exit 3 instead of a green "0 tests")',
  'compile-error': 'iteration: the box is the pre-check (a Mac check takes 12 to 18 min); the run log is kept on the box',
  'link-error': 'iteration; the run log is kept on the box',
  'test-failure': 'iteration; the run log is kept on the box',
  'slot-timeout': 'two slots since 20:3x UK on 6 October; the queue is visible on the workers page',
  'no-dir': 'one run per worktree at a time (the per-worktree lock in remote-run.sh)',
  'other': 'read the kept run log (/srv/builds/_log/runs/<id>.log)',
};

export function readLines(file) {
  if (!fs.existsSync(file)) return [];
  return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
}

// The run being recorded: the FAILED run from RED_WATCH_* when the watcher runs as a workflow_run job (ci-red.yml), else
// the job's own run from GITHUB_* (the inline shape, kept for a branch whose ci.yml still carries the old `red` job).
export const watchedRunId = (env = process.env) => env.RED_WATCH_RUN_ID || env.GITHUB_RUN_ID;
export function runFromEnv(env = process.env) {
  const need = ['GITHUB_REPOSITORY', 'GITHUB_RUN_ID'];
  for (const k of need) if (!env[k]) throw new Error(`record: ${k} is not set (this command runs inside a GitHub Actions job)`);
  const pick = (own, fallback) => env[own] || env[fallback] || '';
  const server = env.GITHUB_SERVER_URL || 'https://github.com';
  const id = String(watchedRunId(env));
  const sha = pick('RED_WATCH_SHA', 'GITHUB_SHA');
  if (!sha) throw new Error('record: neither RED_WATCH_SHA nor GITHUB_SHA is set');
  return {
    run_id: id, attempt: Number(pick('RED_WATCH_ATTEMPT', 'GITHUB_RUN_ATTEMPT') || 1), workflow: pick('RED_WATCH_WORKFLOW', 'GITHUB_WORKFLOW'),
    conclusion: env.RED_WATCH_CONCLUSION || 'failure',   // failure, cancelled or timed_out (ci-red.yml fires on all three since 7 October 2026)
    branch: pick('RED_WATCH_BRANCH', 'GITHUB_REF_NAME'), sha: sha.slice(0, 7), event: pick('RED_WATCH_EVENT', 'GITHUB_EVENT_NAME'),
    actor: pick('RED_WATCH_ACTOR', 'GITHUB_ACTOR'), author: env.RED_WATCH_AUTHOR || '',   // who pushed (the GitHub login), who the head commit names
    url: env.RED_WATCH_URL || `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${id}`, at: new Date().toISOString(),
  };
}

// The failed jobs and their first failed step, from the run's jobs API with the job's own token. The `red` job itself
// (the caller) is skipped by name. Any API trouble gives an empty list and a note, never a thrown error: the line is
// the thing that must land.
export async function failedJobs(env = process.env, fetchImpl = fetch) {
  const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; const id = watchedRunId(env);
  const api = env.GITHUB_API_URL || 'https://api.github.com';
  if (!token) return { failed: [], note: 'no GITHUB_TOKEN; failed steps not read' };
  try {
    const r = await fetchImpl(`${api}/repos/${repo}/actions/runs/${id}/jobs?per_page=100`, {
      headers: { Authorization: `Bearer ${token}`, Accept: 'application/vnd.github+json', 'User-Agent': 'igneum-red-watch' },
    });
    if (!r.ok) return { failed: [], note: `jobs API ${r.status}` };
    const j = await r.json();
    const failed = [];
    for (const job of j.jobs || []) {
      if (job.name === (env.GITHUB_JOB_NAME || 'red watcher') || /^red watcher/.test(job.name)) continue;
      if (job.conclusion === 'success' || job.conclusion === 'skipped' || job.conclusion === null) continue;
      const step = (job.steps || []).find((s) => s.conclusion && s.conclusion !== 'success' && s.conclusion !== 'skipped');
      const zeroSteps = !(job.steps || []).length;
      failed.push({ job: job.name, conclusion: job.conclusion, step: step ? step.name : (zeroSteps ? '(job never started: runner or billing)' : '(no step)') });
    }
    return { failed, note: '' };
  } catch (e) {
    return { failed: [], note: `jobs API: ${e.message}` };
  }
}

export async function record(file, env = process.env, fetchImpl = fetch, title = '') {
  const run = runFromEnv(env);
  const existing = readLines(file);
  if (existing.some((l) => l.run_id === run.run_id && l.attempt === run.attempt)) {
    return { written: false, run };   // one line per run attempt, however many times the job is re-run or retried
  }
  const { failed, note } = await failedJobs(env, fetchImpl);
  const line = { ...run, title: (title || env.RED_WATCH_TITLE || '').slice(0, 100), failed, note };
  fs.mkdirSync(path.dirname(file), { recursive: true });
  fs.appendFileSync(file, JSON.stringify(line) + '\n');
  return { written: true, run: line };
}

// The kind of line: a failed run is "CI red"; a cancelled run "CI cancelled" (a hand on the run, or a job past its timeout-minutes
// under GitHub's older runner, which reports cancelled); a timed-out run "CI timed out". All three are read like a red.
export const KINDS = { failure: 'CI red', cancelled: 'CI cancelled', timed_out: 'CI timed out' };
export function formatLine(l) {
  const where = l.failed.length ? l.failed.map((f) => `${f.job.replace(/,.*$/, '')} at "${f.step}"`).join('; ') : (l.note || 'no step detail');
  const title = l.title ? ` "${l.title}"` : '';
  const who = l.actor ? ` pushed by ${l.actor}${l.author && l.author !== l.actor ? ` (commit by ${l.author})` : ''};` : '';
  const kind = KINDS[l.conclusion || 'failure'] || `CI ${l.conclusion}`;
  return `${kind}: ${l.workflow} on ${l.branch} @${l.sha}${title}:${who} ${where} ${l.url}`;
}

function readCredentials(file) {
  if (!fs.existsSync(file)) return {};
  const out = {};
  for (const raw of fs.readFileSync(file, 'utf8').split('\n')) {
    const line = raw.trim(); if (!line || line.startsWith('#')) continue;
    const i = line.indexOf('='); if (i < 0) continue;
    out[line.slice(0, i).trim()] = line.slice(i + 1).trim();
  }
  return out;
}

function readState(file) { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return { posted: {} }; } }
function writeState(file, state) { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, JSON.stringify(state, null, 1) + '\n', { mode: 0o600 }); }

export async function post(file, { live = false, stateFile = STATE_FILE, credFile = CRED_FILE, fetchImpl = fetch, log = console.log } = {}) {
  const lines = readLines(file);
  const state = readState(stateFile);
  const pending = lines.filter((l) => l.source !== 'box' && !state.posted[`${l.run_id}.${l.attempt || 1}`]);
  const boxRows = lines.filter((l) => l.source === 'box').length;
  if (!pending.length) { log(`ci-red: nothing to post (${lines.length} recorded, ${boxRows} box rows for the digest, the rest posted)`); return { sent: 0, pending: 0 }; }
  const creds = readCredentials(credFile);
  const hook = creds[WEBHOOK_KEY];
  let sent = 0;
  for (const l of pending) {
    const text = formatLine(l);
    if (!live) { log(`ci-red (dry run, not sent): ${text}`); continue; }
    if (!hook) { log(`ci-red: ${WEBHOOK_KEY} is not in the credentials file; ${pending.length} line(s) wait (the line itself is in ${file})`); return { sent: 0, pending: pending.length, missingKey: true }; }
    try {
      const r = await fetchImpl(hook, { method: 'POST', headers: { 'Content-Type': 'application/json', 'User-Agent': 'igneum-red-watch' },
        body: JSON.stringify({ username: 'Igneum CI', content: text.slice(0, 1900), allowed_mentions: { parse: [] } }) });
      if (!r.ok && r.status !== 204) { log(`ci-red: the webhook answered ${r.status} for run ${l.run_id}; retried next tick`); continue; }
      state.posted[`${l.run_id}.${l.attempt || 1}`] = new Date().toISOString(); sent += 1;
      log(`ci-red: posted run ${l.run_id} (${l.workflow} on ${l.branch} @${l.sha})`);
    } catch (e) {
      log(`ci-red: send failed for run ${l.run_id}: ${e.message.replace(/https?:\/\/\S+/g, '<url>')}; retried next tick`);
    }
  }
  if (live) writeState(stateFile, state);
  return { sent, pending: pending.length - sent };
}

// The London day of a Date (YYYY-MM-DD) and its hour, without a time zone library
function london(d) {
  const parts = new Intl.DateTimeFormat('en-GB', { timeZone: 'Europe/London', year: 'numeric', month: '2-digit', day: '2-digit', hour: '2-digit', hour12: false }).formatToParts(d);
  const get = (t) => parts.find((p) => p.type === t).value;
  return { day: `${get('year')}-${get('month')}-${get('day')}`, hour: Number(get('hour')) % 24 };
}

export function digestText(lines, now = new Date()) {
  const since = now.getTime() - 24 * 3600 * 1000;
  const recent = lines.filter((l) => Date.parse(l.at) >= since);
  const ci = recent.filter((l) => l.source !== 'box'); const box = recent.filter((l) => l.source === 'box');
  const byClass = {};
  for (const l of box) byClass[l.class || 'other'] = (byClass[l.class || 'other'] || 0) + 1;
  const ciSteps = {};
  for (const l of ci) for (const f of (l.failed || [])) ciSteps[f.step] = (ciSteps[f.step] || 0) + 1;
  const parts = [`CI red digest, last 24 h: ${recent.length} red run(s): ${ci.length} CI, ${box.length} on the box.`];
  if (ci.length) parts.push('CI: ' + Object.entries(ciSteps).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${v} x "${k}"`).join(', ') + ` (guard: ${GUARDS.ci}).`);
  if (box.length) parts.push('Box, by class: ' + Object.entries(byClass).sort((a, b) => b[1] - a[1]).map(([k, v]) => `${v} ${k} (${GUARDS[k] || GUARDS.other})`).join('; ') + '.');
  if (!recent.length) parts.push('Nothing was red.');
  return parts.join(' ').slice(0, 1900);
}

export async function digest(file, { live = false, now = new Date(), force = false, stateFile = STATE_FILE, credFile = CRED_FILE, fetchImpl = fetch, log = console.log } = {}) {
  const state = readState(stateFile);
  const { day, hour } = london(now);
  if (!force) {
    if (hour < 9) return { sent: false, why: 'before 09:00 London' };
    if (state.digest_day === day) return { sent: false, why: 'already sent today' };
  }
  const text = digestText(readLines(file), now);
  if (!live) { log(`ci-red digest (dry run, not sent): ${text}`); return { sent: false, why: 'dry run', text }; }
  const hook = readCredentials(credFile)[WEBHOOK_KEY];
  if (!hook) { log(`ci-red digest: ${WEBHOOK_KEY} is not in the credentials file; the digest waits`); return { sent: false, why: 'no key', text }; }
  try {
    const r = await fetchImpl(hook, { method: 'POST', headers: { 'Content-Type': 'application/json', 'User-Agent': 'igneum-red-watch' },
      body: JSON.stringify({ username: 'Igneum CI', content: text, allowed_mentions: { parse: [] } }) });
    if (!r.ok && r.status !== 204) { log(`ci-red digest: the webhook answered ${r.status}; retried next pass`); return { sent: false, why: `webhook ${r.status}`, text }; }
  } catch (e) {
    log(`ci-red digest: send failed: ${e.message.replace(/https?:\/\/\S+/g, '<url>')}; retried next pass`); return { sent: false, why: 'send failed', text };
  }
  state.digest_day = day; writeState(stateFile, state);
  log(`ci-red digest: posted for ${day}`);
  return { sent: true, text };
}

async function selfTest() {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'red-watch-'));
  const file = path.join(dir, 'red.jsonl'); const stateFile = path.join(dir, 'posted.json'); const credFile = path.join(dir, 'discord');
  const env = { GITHUB_RUN_ID: '424242', GITHUB_RUN_ATTEMPT: '1', GITHUB_REPOSITORY: 'igneum-network/igneum', GITHUB_REF_NAME: 'master',
    GITHUB_SHA: '0f0abc6deadbeef', GITHUB_WORKFLOW: 'ci', GITHUB_TOKEN: 'x', RED_WATCH_TITLE: 'Merge box-work 2bd3bec: the repro script re-stamps its clones',
    GITHUB_ACTOR: 'igneum-labs', RED_WATCH_AUTHOR: 'igneum-labs' };
  const jobs = { jobs: [
    { name: 'site build, link check, identity grep', conclusion: 'failure', steps: [{ name: 'site build', conclusion: 'success' }, { name: 'identity grep of the public export list', conclusion: 'failure' }] },
    { name: 'igneum-pow tests, igneum-census build', conclusion: 'success', steps: [] },
    { name: 'simulators, quick modes', conclusion: 'failure', steps: [] },
    { name: 'red watcher (master and release-* only)', conclusion: null, steps: [] },
  ] };
  const fakeFetch = async () => ({ ok: true, status: 200, json: async () => jobs });
  const fails = [];
  const a = await record(file, env, fakeFetch); const b = await record(file, env, fakeFetch);
  if (!a.written || b.written) fails.push('record: the second call for the same run wrote a second line');
  const lines = readLines(file);
  if (lines.length !== 1) fails.push(`record: ${lines.length} lines, expected 1`);
  if (lines[0].failed.length !== 2) fails.push(`record: ${lines[0].failed.length} failed jobs, expected 2 (the watcher itself and the green job skipped)`);
  if (lines[0].failed[0].step !== 'identity grep of the public export list') fails.push('record: the failed step was not the first non-success step');
  if (lines[0].failed[1].step !== '(job never started: runner or billing)') fails.push('record: a job with no steps was not named as never started');
  const text = formatLine(lines[0]);
  if (!/^CI red: ci on master @0f0abc6 "Merge box-work/.test(text) || !text.includes('actions/runs/424242')) fails.push(`format: ${text}`);
  if (!text.includes('pushed by igneum-labs (commit by igneum-labs);') || !text.includes('site build at "identity grep of the public export list"')) fails.push(`format: the line does not name the pushing author and the red check: ${text}`);
  // a feature branch is recorded and formatted like master (every branch since 7 October 2026)
  const envFeature = { ...env, GITHUB_RUN_ID: '424299', GITHUB_REF_NAME: 'ca3-v4-node', RED_WATCH_AUTHOR: 'igneum-labs' };
  const fileFeature = path.join(dir, 'feature.jsonl');
  await record(fileFeature, envFeature, fakeFetch);
  const textFeature = formatLine(readLines(fileFeature)[0]);
  if (!/^CI red: ci on ca3-v4-node @0f0abc6 "Merge box-work[^"]*": pushed by igneum-labs; site build at/.test(textFeature)) fails.push(`format on a feature branch: ${textFeature}`);
  // the workflow_run shape (ci-red.yml): GITHUB_* describe the watcher's own run, RED_WATCH_* the failed one; the line is the failed run's
  const envRun = { GITHUB_RUN_ID: '999', GITHUB_RUN_ATTEMPT: '1', GITHUB_REPOSITORY: 'igneum-network/igneum', GITHUB_REF_NAME: 'master', GITHUB_SHA: 'ffffffffffff', GITHUB_WORKFLOW: 'ci-red', GITHUB_ACTOR: 'igneum-labs', GITHUB_TOKEN: 'x',
    RED_WATCH_RUN_ID: '37620364667', RED_WATCH_ATTEMPT: '2', RED_WATCH_WORKFLOW: 'ci', RED_WATCH_BRANCH: 'ca3-v4-node', RED_WATCH_SHA: '26a4b0f1deadbeef', RED_WATCH_EVENT: 'push',
    RED_WATCH_URL: 'https://github.com/igneum-network/igneum/actions/runs/37620364667', RED_WATCH_ACTOR: 'igneum-labs', RED_WATCH_TITLE: 'Counter ASIC 3.0 node plan 6.7', RED_WATCH_AUTHOR: 'igneum-labs' };
  const fileRun = path.join(dir, 'workflow-run.jsonl'); const asked = [];
  const askingFetch = async (url) => { asked.push(url); return { ok: true, status: 200, json: async () => jobs }; };
  await record(fileRun, envRun, askingFetch);
  const lr = readLines(fileRun)[0];
  if (lr.run_id !== '37620364667' || lr.attempt !== 2 || lr.branch !== 'ca3-v4-node' || lr.sha !== '26a4b0f' || lr.workflow !== 'ci') fails.push(`workflow_run shape: recorded ${JSON.stringify({ run_id: lr.run_id, attempt: lr.attempt, branch: lr.branch, sha: lr.sha, workflow: lr.workflow })}, expected the failed run, not the watcher's`);
  if (!asked[0] || !asked[0].includes('/actions/runs/37620364667/jobs')) fails.push(`workflow_run shape: the jobs API was asked for ${asked[0]}, not the failed run`);
  const textRun = formatLine(lr);
  if (!/^CI red: ci on ca3-v4-node @26a4b0f "Counter ASIC 3.0 node plan 6.7": pushed by igneum-labs \(commit by igneum-labs\); site build at "identity grep of the public export list"; simulators at "\(job never started: runner or billing\)" https:\/\/github.com\/igneum-network\/igneum\/actions\/runs\/37620364667$/.test(textRun)) fails.push(`workflow_run line: ${textRun}`);
  // a cancelled run and a timed-out run are recorded with their kind in the line (a hung job past its timeout-minutes, a hand on the run)
  const cancelledJobs = { jobs: [{ name: 'site build, link check, identity grep', conclusion: 'cancelled', steps: [{ name: 'site build', conclusion: 'success' }, { name: 'the tree gate, tools/ci/pre-push.sh --ci', conclusion: 'cancelled' }] }] };
  const fileKinds = path.join(dir, 'kinds.jsonl');
  await record(fileKinds, { ...envRun, RED_WATCH_RUN_ID: '555', RED_WATCH_CONCLUSION: 'cancelled' }, async () => ({ ok: true, status: 200, json: async () => cancelledJobs }));
  await record(fileKinds, { ...envRun, RED_WATCH_RUN_ID: '556', RED_WATCH_CONCLUSION: 'timed_out' }, async () => ({ ok: true, status: 200, json: async () => cancelledJobs }));
  const [lc, lt] = readLines(fileKinds).map(formatLine);
  if (!/^CI cancelled: ci on ca3-v4-node @26a4b0f .*site build at "the tree gate, tools\/ci\/pre-push.sh --ci"/.test(lc)) fails.push(`cancelled line: ${lc}`);
  if (!/^CI timed out: ci on ca3-v4-node @26a4b0f /.test(lt)) fails.push(`timed-out line: ${lt}`);
  if (readLines(fileKinds)[0].conclusion !== 'cancelled') fails.push('record: the conclusion was not kept in the line');
  // the watcher workflow fires on all three conclusions and hands the conclusion to the record step
  const ymlPath = path.join(path.dirname(new URL(import.meta.url).pathname), '..', '..', '.github', 'workflows', 'ci-red.yml');
  if (fs.existsSync(ymlPath)) {
    const yml = fs.readFileSync(ymlPath, 'utf8');
    for (const c of ['failure', 'cancelled', 'timed_out']) if (!yml.includes(`github.event.workflow_run.conclusion == '${c}'`)) fails.push(`ci-red.yml: the job's if does not fire on ${c}`);
    if (!yml.includes('RED_WATCH_CONCLUSION: ${{ github.event.workflow_run.conclusion }}')) fails.push('ci-red.yml: RED_WATCH_CONCLUSION is not handed to the record step');
  }
  // post, dry run: prints, sends nothing, marks nothing
  let printed = []; const log = (s) => printed.push(s);
  const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; };
  await post(file, { live: false, stateFile, credFile, fetchImpl: hookFetch, log });
  if (sends.length !== 0 || !printed.some((s) => s.includes('dry run'))) fails.push('post: the dry run sent or did not print');
  // post, live, no key: says which key is missing, names no URL, sends nothing
  fs.writeFileSync(credFile, 'DISCORD_WEBHOOK_NUMBERS=https://discord.example/api/webhooks/1/secret\n', { mode: 0o600 });
  printed = [];
  const r0 = await post(file, { live: true, stateFile, credFile, fetchImpl: hookFetch, log });
  if (!r0.missingKey || sends.length !== 0 || !printed.some((s) => s.includes(WEBHOOK_KEY))) fails.push('post: a missing updates key was not reported by name');
  if (printed.some((s) => s.includes('secret'))) fails.push('post: a webhook URL leaked into the log');
  // post, live, with the key: one send with the line, then marked posted; a second pass sends nothing
  fs.writeFileSync(credFile, `${WEBHOOK_KEY}=https://discord.example/api/webhooks/2/secret2\n`, { mode: 0o600 });
  printed = [];
  const r1 = await post(file, { live: true, stateFile, credFile, fetchImpl: hookFetch, log });
  if (r1.sent !== 1 || sends.length !== 1 || sends[0].body.content !== text) fails.push(`post: expected one send of the line, got ${sends.length}`);
  if (sends[0].body.allowed_mentions?.parse?.length !== 0) fails.push('post: mentions are not disabled');
  if (printed.some((s) => s.includes('secret2'))) fails.push('post: the webhook URL leaked into the log');
  const r2 = await post(file, { live: true, stateFile, credFile, fetchImpl: hookFetch, log });
  if (r2.sent !== 0 || sends.length !== 1) fails.push('post: the second pass sent the same run again');
  // a failing webhook leaves the run pending for the next tick
  const env2 = { ...env, GITHUB_RUN_ID: '424243' };
  await record(file, env2, fakeFetch);
  const badFetch = async () => ({ ok: false, status: 500 });
  const r3 = await post(file, { live: true, stateFile, credFile, fetchImpl: badFetch, log });
  if (r3.sent !== 0 || r3.pending !== 1) fails.push('post: a 500 from the webhook did not keep the run pending');
  // a box row (remote-run.sh's shape) is never posted alone, and the digest counts it per class with its guard
  const boxLine = { source: 'box', run_id: 'igneum-build-1-1-1', attempt: 1, workflow: 'box:suite', branch: 'x', sha: 'abc1234', url: '', at: new Date().toISOString(),
    title: 'cargo test --release -p kaspa-consensus --lib finality', failed: [{ job: 'wt/crate', conclusion: 'failure', step: 'instant: exit 101 after 0 s' }], class: 'instant', note: '' };
  fs.appendFileSync(file, JSON.stringify(boxLine) + '\n');
  const before = sends.length;
  const r4 = await post(file, { live: true, stateFile, credFile, fetchImpl: hookFetch, log });
  // the pending CI run 424243 goes now (one send), the box row does not
  if (sends.length !== before + 1 || r4.pending !== 0 || !sends[sends.length - 1].body.content.includes('actions/runs/424243')) fails.push('post: a box row was posted alone or the pending CI run was not');
  const text2 = digestText(readLines(file), new Date());
  if (!/3 red run\(s\): 2 CI, 1 on the box/.test(text2) || !text2.includes('1 instant (pre-flight')) fails.push(`digest text: ${text2}`);
  const at0830 = new Date('2026-10-07T07:30:00Z');  // 08:30 London (BST)
  const d0 = await digest(file, { live: true, now: at0830, stateFile, credFile, fetchImpl: hookFetch, log });
  if (d0.sent || d0.why !== 'before 09:00 London') fails.push(`digest: sent before 09:00 London (${d0.why})`);
  const at0905 = new Date('2026-10-07T08:05:00Z');  // 09:05 London
  const d1 = await digest(file, { live: true, now: at0905, stateFile, credFile, fetchImpl: hookFetch, log });
  if (!d1.sent || sends[sends.length - 1].body.content !== d1.text) fails.push('digest: not sent at 09:05 London or the text differs');
  const d2 = await digest(file, { live: true, now: new Date('2026-10-07T15:00:00Z'), stateFile, credFile, fetchImpl: hookFetch, log });
  if (d2.sent || d2.why !== 'already sent today') fails.push('digest: sent twice in one London day');
  const d3 = await digest(file, { live: true, now: new Date('2026-10-08T08:05:00Z'), stateFile, credFile, fetchImpl: hookFetch, log });
  if (!d3.sent) fails.push('digest: not sent the next day');
  fs.rmSync(dir, { recursive: true, force: true });
  if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); }
  console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author, the failed run and not the watcher\'s own under workflow_run; a cancelled and a timed-out run are recorded with their kind and ci-red.yml fires on all three; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
}

const cmd = args[0];
if (cmd === '--self-test') {
  await selfTest();
} else if (cmd === 'record') {
  const file = flag('--file'); if (!file) { console.error('record: --file <red.jsonl> is required'); process.exit(2); }
  const r = await record(file, process.env, fetch, flag('--title') || '');
  console.log(r.written ? `ci-red: recorded ${formatLine(r.run)}` : `ci-red: run ${r.run.run_id} attempt ${r.run.attempt} already recorded`);
} else if (cmd === 'post') {
  const file = flag('--file'); if (!file) { console.error('post: --file <red.jsonl> is required'); process.exit(2); }
  await post(file, { live: has('--live') });
} else if (cmd === 'digest') {
  const file = flag('--file'); if (!file) { console.error('digest: --file <red.jsonl> is required'); process.exit(2); }
  const r = await digest(file, { live: has('--live'), force: has('--now') });
  if (!r.sent && r.why !== 'dry run') console.log(`ci-red digest: not sent (${r.why})`);
} else if (cmd === 'tick') {
  const file = flag('--file'); if (!file) { console.error('tick: --file <red.jsonl> is required'); process.exit(2); }
  await post(file, { live: has('--live') });
  const r = await digest(file, { live: has('--live') });
  if (!r.sent && r.why !== 'dry run' && r.why !== 'before 09:00 London' && r.why !== 'already sent today') console.log(`ci-red digest: not sent (${r.why})`);
} else {
  console.error('usage: red-watch.mjs record --file <red.jsonl> | post --file <red.jsonl> [--live] | digest --file <red.jsonl> [--live] [--now] | tick --file <red.jsonl> [--live] | --self-test'); process.exit(2);
}