diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f080596..47d81673 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,9 +11,10 @@ # Where it runs: `pow` and `sims` go to the box's runner (igneum-build-1, rustc pinned, sccache read-only, 48 jobs) # when the repository variable IGNEUM_CI_RUNNER is `box`, else to ubuntu-latest (docs/plans/ci-self-hosted.md; GitHub # has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The `red` job -# runs on the box after any failed master or release-* run and records the failure for the watcher -# (tools/ci/red-watch.mjs; infra/build-server/ci-red): one line per run to the hidden updates channel and to -# /srv/ci-red/red.jsonl, so nobody opens the Actions page to learn master is red. +# runs on the box after any failed run on ANY branch and records the failure for the watcher +# (tools/ci/red-watch.mjs; infra/build-server/ci-red): one line per run, naming the branch, the commit, the red check +# and the pushing author, to the hidden updates channel and to /srv/ci-red/red.jsonl, so nobody opens the Actions page +# to learn a branch is red (master and release-* only until 7 October 2026, when eight red runs on ca3-v4-node went unseen). # # What does not run, on purpose: the node fork (vendor/igneum-node*, a rusty-kaspa fork of about 500 crates with # rocksdb, blst and the execution layer) is gitignored here and too big for the free runners today (a cold build is @@ -77,14 +78,14 @@ jobs: run: node tools/ci/public-api-check.mjs https://igneum.network red: - # Runs only when a master or release-* run has a failed job, on the box's own runner (not a GitHub-hosted machine: + # Runs when a run on any branch has a failed job, on the box's own runner (not a GitHub-hosted machine: # the billing block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told). # tools/ci/red-watch.mjs record appends ONE line for this run to /srv/ci-red/red.jsonl (idempotent per run attempt); # the box's igneum-ci-red.timer posts each new line once to the hidden updates channel. Never blocks a release: # it reads the run, writes one line, and ends. - name: red watcher (master and release-* only; one line per failed run to the updates channel and the box file) + name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file) needs: [pow, sims, site] - if: ${{ failure() && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) }} + if: ${{ failure() }} runs-on: [self-hosted, linux, x64, igneum-build-1] timeout-minutes: 5 permissions: @@ -94,8 +95,9 @@ jobs: - uses: actions/checkout@v4 with: sparse-checkout: tools/ci - - name: record this run (one line, the failed jobs and their first failed step, from the run's own API) + - name: record this run (one line, the branch, the commit, the failed jobs and their first failed step from the run's own API, the pushing author) env: GITHUB_TOKEN: ${{ github.token }} RED_WATCH_TITLE: ${{ github.event.head_commit.message }} + RED_WATCH_AUTHOR: ${{ github.event.head_commit.author.name }} run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl diff --git a/tools/ci/red-watch.mjs b/tools/ci/red-watch.mjs index 3f367735..3d401e21 100755 --- a/tools/ci/red-watch.mjs +++ b/tools/ci/red-watch.mjs @@ -1,5 +1,7 @@ #!/usr/bin/env node -// The red-master watcher. One line per failed master or release-* run, so nobody opens the Actions page to learn CI is red. +// The red watcher. One line per failed run on ANY branch (master and release-* only until 7 October 2026: eight red runs on +// ca3-v4-node went unseen that morning), naming the branch, the commit, the red check and the pushing author, so nobody +// opens the Actions page to learn a branch is red. // Node 22, standard library only. // // node tools/ci/red-watch.mjs record --file in the workflow's `red` job (runs on igneum-build-1 after a @@ -41,7 +43,7 @@ const STATE_FILE = process.env.IGNEUM_CI_RED_STATE || path.join(os.homedir(), '. const WEBHOOK_KEY = 'DISCORD_WEBHOOK_UPDATES'; // what stops each class now (named in the digest so the line teaches, not just counts); docs/analysis/ci-failures-2026-10-06.md export const GUARDS = { - 'ci': 'the pre-push gate (tools/ci/pre-push.sh, the same checks CI runs, before any push to master or release-*)', + 'ci': 'the pre-push gate (tools/ci/pre-push.sh: the full gate before a push to master or release-*, the never-push checks before a push to any branch)', 'instant': 'pre-flight in remote-run.sh (manifest, -p package, feature, subcommand checked in a second) and the kept run log', 'preflight-manifest': 'refused before the slot: the manifest did not parse', 'preflight-package': 'refused before the slot: the -p package does not exist', @@ -68,6 +70,7 @@ export function runFromEnv(env = process.env) { return { run_id: String(env.GITHUB_RUN_ID), attempt: Number(env.GITHUB_RUN_ATTEMPT || 1), workflow: env.GITHUB_WORKFLOW, branch: env.GITHUB_REF_NAME, sha: env.GITHUB_SHA.slice(0, 7), event: env.GITHUB_EVENT_NAME || '', + actor: env.GITHUB_ACTOR || '', author: env.RED_WATCH_AUTHOR || '', // who pushed (the GitHub login), who the head commit names url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`, at: new Date().toISOString(), }; } @@ -115,7 +118,8 @@ export async function record(file, env = process.env, fetchImpl = fetch, title = export function formatLine(l) { const where = l.failed.length ? l.failed.map((f) => `${f.job.replace(/,.*$/, '')} at "${f.step}"`).join('; ') : (l.note || 'no step detail'); const title = l.title ? ` "${l.title}"` : ''; - return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}: ${where} ${l.url}`; + const who = l.actor ? ` pushed by ${l.actor}${l.author && l.author !== l.actor ? ` (commit by ${l.author})` : ''};` : ''; + return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}:${who} ${where} ${l.url}`; } function readCredentials(file) { @@ -208,7 +212,8 @@ async function selfTest() { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'red-watch-')); const file = path.join(dir, 'red.jsonl'); const stateFile = path.join(dir, 'posted.json'); const credFile = path.join(dir, 'discord'); const env = { GITHUB_RUN_ID: '424242', GITHUB_RUN_ATTEMPT: '1', GITHUB_REPOSITORY: 'igneum-network/igneum', GITHUB_REF_NAME: 'master', - GITHUB_SHA: '0f0abc6deadbeef', GITHUB_WORKFLOW: 'ci', GITHUB_TOKEN: 'x', RED_WATCH_TITLE: 'Merge box-work 2bd3bec: the repro script re-stamps its clones' }; + GITHUB_SHA: '0f0abc6deadbeef', GITHUB_WORKFLOW: 'ci', GITHUB_TOKEN: 'x', RED_WATCH_TITLE: 'Merge box-work 2bd3bec: the repro script re-stamps its clones', + GITHUB_ACTOR: 'igneum-labs', RED_WATCH_AUTHOR: 'igneum-labs' }; const jobs = { jobs: [ { name: 'site build, link check, identity grep', conclusion: 'failure', steps: [{ name: 'site build', conclusion: 'success' }, { name: 'identity grep of the public export list', conclusion: 'failure' }] }, { name: 'igneum-pow tests, igneum-census build', conclusion: 'success', steps: [] }, @@ -226,6 +231,13 @@ async function selfTest() { if (lines[0].failed[1].step !== '(job never started: runner or billing)') fails.push('record: a job with no steps was not named as never started'); const text = formatLine(lines[0]); if (!/^CI red: ci on master @0f0abc6 "Merge box-work/.test(text) || !text.includes('actions/runs/424242')) fails.push(`format: ${text}`); + if (!text.includes('pushed by igneum-labs (commit by igneum-labs);') || !text.includes('site build at "identity grep of the public export list"')) fails.push(`format: the line does not name the pushing author and the red check: ${text}`); + // a feature branch is recorded and formatted like master (every branch since 7 October 2026) + const envFeature = { ...env, GITHUB_RUN_ID: '424299', GITHUB_REF_NAME: 'ca3-v4-node', RED_WATCH_AUTHOR: 'igneum-labs' }; + const fileFeature = path.join(dir, 'feature.jsonl'); + await record(fileFeature, envFeature, fakeFetch); + const textFeature = formatLine(readLines(fileFeature)[0]); + if (!/^CI red: ci on ca3-v4-node @0f0abc6 "Merge box-work[^"]*": pushed by igneum-labs; site build at/.test(textFeature)) fails.push(`format on a feature branch: ${textFeature}`); // post, dry run: prints, sends nothing, marks nothing let printed = []; const log = (s) => printed.push(s); const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; }; @@ -274,7 +286,7 @@ async function selfTest() { if (!d3.sent) fails.push('digest: not sent the next day'); fs.rmSync(dir, { recursive: true, force: true }); if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); } - console.log('self-test passed: one line per run however often record runs; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00'); + console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00'); } const cmd = args[0];