CI red watcher: every branch, one line per failed run naming the branch, the commit, the red check and the pushing author
The red job's condition drops the master/release-* clause; record() keeps GITHUB_ACTOR and the head commit's author name (RED_WATCH_AUTHOR from the workflow) and formatLine prints "pushed by <actor> (commit by <author>)" before the failed job and step. The self-test covers a feature-branch run and the author in the line. Same updates channel, same box file, same one-line-per-run idempotence. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
parent
0412fa4ae5
commit
577c614b31
2 changed files with 26 additions and 12 deletions
16
.github/workflows/ci.yml
vendored
16
.github/workflows/ci.yml
vendored
|
|
@ -11,9 +11,10 @@
|
|||
# Where it runs: `pow` and `sims` go to the box's runner (igneum-build-1, rustc pinned, sccache read-only, 48 jobs)
|
||||
# when the repository variable IGNEUM_CI_RUNNER is `box`, else to ubuntu-latest (docs/plans/ci-self-hosted.md; GitHub
|
||||
# has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The `red` job
|
||||
# runs on the box after any failed master or release-* run and records the failure for the watcher
|
||||
# (tools/ci/red-watch.mjs; infra/build-server/ci-red): one line per run to the hidden updates channel and to
|
||||
# /srv/ci-red/red.jsonl, so nobody opens the Actions page to learn master is red.
|
||||
# runs on the box after any failed run on ANY branch and records the failure for the watcher
|
||||
# (tools/ci/red-watch.mjs; infra/build-server/ci-red): one line per run, naming the branch, the commit, the red check
|
||||
# and the pushing author, to the hidden updates channel and to /srv/ci-red/red.jsonl, so nobody opens the Actions page
|
||||
# to learn a branch is red (master and release-* only until 7 October 2026, when eight red runs on ca3-v4-node went unseen).
|
||||
#
|
||||
# What does not run, on purpose: the node fork (vendor/igneum-node*, a rusty-kaspa fork of about 500 crates with
|
||||
# rocksdb, blst and the execution layer) is gitignored here and too big for the free runners today (a cold build is
|
||||
|
|
@ -77,14 +78,14 @@ jobs:
|
|||
run: node tools/ci/public-api-check.mjs https://igneum.network
|
||||
|
||||
red:
|
||||
# Runs only when a master or release-* run has a failed job, on the box's own runner (not a GitHub-hosted machine:
|
||||
# Runs when a run on any branch has a failed job, on the box's own runner (not a GitHub-hosted machine:
|
||||
# the billing block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told).
|
||||
# tools/ci/red-watch.mjs record appends ONE line for this run to /srv/ci-red/red.jsonl (idempotent per run attempt);
|
||||
# the box's igneum-ci-red.timer posts each new line once to the hidden updates channel. Never blocks a release:
|
||||
# it reads the run, writes one line, and ends.
|
||||
name: red watcher (master and release-* only; one line per failed run to the updates channel and the box file)
|
||||
name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file)
|
||||
needs: [pow, sims, site]
|
||||
if: ${{ failure() && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) }}
|
||||
if: ${{ failure() }}
|
||||
runs-on: [self-hosted, linux, x64, igneum-build-1]
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
|
|
@ -94,8 +95,9 @@ jobs:
|
|||
- uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: tools/ci
|
||||
- name: record this run (one line, the failed jobs and their first failed step, from the run's own API)
|
||||
- name: record this run (one line, the branch, the commit, the failed jobs and their first failed step from the run's own API, the pushing author)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
RED_WATCH_TITLE: ${{ github.event.head_commit.message }}
|
||||
RED_WATCH_AUTHOR: ${{ github.event.head_commit.author.name }}
|
||||
run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
#!/usr/bin/env node
|
||||
// The red-master watcher. One line per failed master or release-* run, so nobody opens the Actions page to learn CI is red.
|
||||
// The red watcher. One line per failed run on ANY branch (master and release-* only until 7 October 2026: eight red runs on
|
||||
// ca3-v4-node went unseen that morning), naming the branch, the commit, the red check and the pushing author, so nobody
|
||||
// opens the Actions page to learn a branch is red.
|
||||
// Node 22, standard library only.
|
||||
//
|
||||
// node tools/ci/red-watch.mjs record --file <red.jsonl> in the workflow's `red` job (runs on igneum-build-1 after a
|
||||
|
|
@ -41,7 +43,7 @@ const STATE_FILE = process.env.IGNEUM_CI_RED_STATE || path.join(os.homedir(), '.
|
|||
const WEBHOOK_KEY = 'DISCORD_WEBHOOK_UPDATES';
|
||||
// what stops each class now (named in the digest so the line teaches, not just counts); docs/analysis/ci-failures-2026-10-06.md
|
||||
export const GUARDS = {
|
||||
'ci': 'the pre-push gate (tools/ci/pre-push.sh, the same checks CI runs, before any push to master or release-*)',
|
||||
'ci': 'the pre-push gate (tools/ci/pre-push.sh: the full gate before a push to master or release-*, the never-push checks before a push to any branch)',
|
||||
'instant': 'pre-flight in remote-run.sh (manifest, -p package, feature, subcommand checked in a second) and the kept run log',
|
||||
'preflight-manifest': 'refused before the slot: the manifest did not parse',
|
||||
'preflight-package': 'refused before the slot: the -p package does not exist',
|
||||
|
|
@ -68,6 +70,7 @@ export function runFromEnv(env = process.env) {
|
|||
return {
|
||||
run_id: String(env.GITHUB_RUN_ID), attempt: Number(env.GITHUB_RUN_ATTEMPT || 1), workflow: env.GITHUB_WORKFLOW,
|
||||
branch: env.GITHUB_REF_NAME, sha: env.GITHUB_SHA.slice(0, 7), event: env.GITHUB_EVENT_NAME || '',
|
||||
actor: env.GITHUB_ACTOR || '', author: env.RED_WATCH_AUTHOR || '', // who pushed (the GitHub login), who the head commit names
|
||||
url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`, at: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
|
@ -115,7 +118,8 @@ export async function record(file, env = process.env, fetchImpl = fetch, title =
|
|||
export function formatLine(l) {
|
||||
const where = l.failed.length ? l.failed.map((f) => `${f.job.replace(/,.*$/, '')} at "${f.step}"`).join('; ') : (l.note || 'no step detail');
|
||||
const title = l.title ? ` "${l.title}"` : '';
|
||||
return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}: ${where} ${l.url}`;
|
||||
const who = l.actor ? ` pushed by ${l.actor}${l.author && l.author !== l.actor ? ` (commit by ${l.author})` : ''};` : '';
|
||||
return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}:${who} ${where} ${l.url}`;
|
||||
}
|
||||
|
||||
function readCredentials(file) {
|
||||
|
|
@ -208,7 +212,8 @@ async function selfTest() {
|
|||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'red-watch-'));
|
||||
const file = path.join(dir, 'red.jsonl'); const stateFile = path.join(dir, 'posted.json'); const credFile = path.join(dir, 'discord');
|
||||
const env = { GITHUB_RUN_ID: '424242', GITHUB_RUN_ATTEMPT: '1', GITHUB_REPOSITORY: 'igneum-network/igneum', GITHUB_REF_NAME: 'master',
|
||||
GITHUB_SHA: '0f0abc6deadbeef', GITHUB_WORKFLOW: 'ci', GITHUB_TOKEN: 'x', RED_WATCH_TITLE: 'Merge box-work 2bd3bec: the repro script re-stamps its clones' };
|
||||
GITHUB_SHA: '0f0abc6deadbeef', GITHUB_WORKFLOW: 'ci', GITHUB_TOKEN: 'x', RED_WATCH_TITLE: 'Merge box-work 2bd3bec: the repro script re-stamps its clones',
|
||||
GITHUB_ACTOR: 'igneum-labs', RED_WATCH_AUTHOR: 'igneum-labs' };
|
||||
const jobs = { jobs: [
|
||||
{ name: 'site build, link check, identity grep', conclusion: 'failure', steps: [{ name: 'site build', conclusion: 'success' }, { name: 'identity grep of the public export list', conclusion: 'failure' }] },
|
||||
{ name: 'igneum-pow tests, igneum-census build', conclusion: 'success', steps: [] },
|
||||
|
|
@ -226,6 +231,13 @@ async function selfTest() {
|
|||
if (lines[0].failed[1].step !== '(job never started: runner or billing)') fails.push('record: a job with no steps was not named as never started');
|
||||
const text = formatLine(lines[0]);
|
||||
if (!/^CI red: ci on master @0f0abc6 "Merge box-work/.test(text) || !text.includes('actions/runs/424242')) fails.push(`format: ${text}`);
|
||||
if (!text.includes('pushed by igneum-labs (commit by igneum-labs);') || !text.includes('site build at "identity grep of the public export list"')) fails.push(`format: the line does not name the pushing author and the red check: ${text}`);
|
||||
// a feature branch is recorded and formatted like master (every branch since 7 October 2026)
|
||||
const envFeature = { ...env, GITHUB_RUN_ID: '424299', GITHUB_REF_NAME: 'ca3-v4-node', RED_WATCH_AUTHOR: 'igneum-labs' };
|
||||
const fileFeature = path.join(dir, 'feature.jsonl');
|
||||
await record(fileFeature, envFeature, fakeFetch);
|
||||
const textFeature = formatLine(readLines(fileFeature)[0]);
|
||||
if (!/^CI red: ci on ca3-v4-node @0f0abc6 "Merge box-work[^"]*": pushed by igneum-labs; site build at/.test(textFeature)) fails.push(`format on a feature branch: ${textFeature}`);
|
||||
// post, dry run: prints, sends nothing, marks nothing
|
||||
let printed = []; const log = (s) => printed.push(s);
|
||||
const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; };
|
||||
|
|
@ -274,7 +286,7 @@ async function selfTest() {
|
|||
if (!d3.sent) fails.push('digest: not sent the next day');
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); }
|
||||
console.log('self-test passed: one line per run however often record runs; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
|
||||
console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
|
||||
}
|
||||
|
||||
const cmd = args[0];
|
||||
|
|
|
|||
Loading…
Reference in a new issue