diff --git a/.github/workflows/ci-red.yml b/.github/workflows/ci-red.yml new file mode 100644 index 00000000..dddb758b --- /dev/null +++ b/.github/workflows/ci-red.yml @@ -0,0 +1,42 @@ +# The red watcher as its own workflow, on workflow_run, so the copy on master watches EVERY branch's ci run whatever +# ci.yml that branch carries: GitHub runs a workflow_run workflow from the default branch only, and the branch's own +# ci.yml never enters it (7 October 2026: the inline `red` job of ci.yml was conditioned on master and release-*, and +# a feature branch would have waited for a merge of master before its reds were posted at all). +# +# One line per failed run (tools/ci/red-watch.mjs record, idempotent per run attempt) to /srv/ci-red/red.jsonl on the +# box; the box's igneum-ci-red.timer posts each new line once to the hidden updates channel, naming the branch, the +# commit, the red check and the pushing author. Runs on the box's own runner (not a GitHub-hosted machine: the billing +# block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told). Never blocks a +# release: it reads the run, writes one line, and ends. +name: ci-red +on: + workflow_run: + workflows: [ci] + types: [completed] +jobs: + red: + name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file) + if: ${{ github.event.workflow_run.conclusion == 'failure' }} + runs-on: [self-hosted, linux, x64, igneum-build-1] + timeout-minutes: 5 + permissions: + actions: read # the failed run's jobs API (the first real red run, 21:19Z on 6 October: the default token answered 403 and the line carried no step) + contents: read + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: tools/ci + - name: record the failed run (one line, the branch, the commit, the failed jobs and their first failed step from the run's own API, the pushing author) + env: + GITHUB_TOKEN: ${{ github.token }} + RED_WATCH_RUN_ID: ${{ github.event.workflow_run.id }} + RED_WATCH_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + RED_WATCH_WORKFLOW: ${{ github.event.workflow_run.name }} + RED_WATCH_BRANCH: ${{ github.event.workflow_run.head_branch }} + RED_WATCH_SHA: ${{ github.event.workflow_run.head_sha }} + RED_WATCH_EVENT: ${{ github.event.workflow_run.event }} + RED_WATCH_URL: ${{ github.event.workflow_run.html_url }} + RED_WATCH_ACTOR: ${{ github.event.workflow_run.actor.login }} + RED_WATCH_TITLE: ${{ github.event.workflow_run.head_commit.message }} + RED_WATCH_AUTHOR: ${{ github.event.workflow_run.head_commit.author.name }} + run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47d81673..5311c6dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,11 +10,12 @@ # # Where it runs: `pow` and `sims` go to the box's runner (igneum-build-1, rustc pinned, sccache read-only, 48 jobs) # when the repository variable IGNEUM_CI_RUNNER is `box`, else to ubuntu-latest (docs/plans/ci-self-hosted.md; GitHub -# has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The `red` job -# runs on the box after any failed run on ANY branch and records the failure for the watcher -# (tools/ci/red-watch.mjs; infra/build-server/ci-red): one line per run, naming the branch, the commit, the red check -# and the pushing author, to the hidden updates channel and to /srv/ci-red/red.jsonl, so nobody opens the Actions page -# to learn a branch is red (master and release-* only until 7 October 2026, when eight red runs on ca3-v4-node went unseen). +# has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The red watcher +# is its own workflow, .github/workflows/ci-red.yml (workflow_run, so the copy on master watches every branch's run +# whatever ci.yml that branch carries): one line per failed run, naming the branch, the commit, the red check and the +# pushing author, to the hidden updates channel and to /srv/ci-red/red.jsonl (tools/ci/red-watch.mjs; +# infra/build-server/ci-red), so nobody opens the Actions page to learn a branch is red (the inline `red` job here +# watched master and release-* only until 7 October 2026, when eight red runs on ca3-v4-node went unseen). # # What does not run, on purpose: the node fork (vendor/igneum-node*, a rusty-kaspa fork of about 500 crates with # rocksdb, blst and the execution layer) is gitignored here and too big for the free runners today (a cold build is @@ -76,28 +77,3 @@ jobs: - name: public stats API answers with the documented fields (the live site; master only, the endpoints exist there after the merge) if: github.ref == 'refs/heads/master' run: node tools/ci/public-api-check.mjs https://igneum.network - - red: - # Runs when a run on any branch has a failed job, on the box's own runner (not a GitHub-hosted machine: - # the billing block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told). - # tools/ci/red-watch.mjs record appends ONE line for this run to /srv/ci-red/red.jsonl (idempotent per run attempt); - # the box's igneum-ci-red.timer posts each new line once to the hidden updates channel. Never blocks a release: - # it reads the run, writes one line, and ends. - name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file) - needs: [pow, sims, site] - if: ${{ failure() }} - runs-on: [self-hosted, linux, x64, igneum-build-1] - timeout-minutes: 5 - permissions: - actions: read # the run's jobs API (the first real red run, 21:19Z: the default token answered 403 and the line carried no step) - contents: read - steps: - - uses: actions/checkout@v4 - with: - sparse-checkout: tools/ci - - name: record this run (one line, the branch, the commit, the failed jobs and their first failed step from the run's own API, the pushing author) - env: - GITHUB_TOKEN: ${{ github.token }} - RED_WATCH_TITLE: ${{ github.event.head_commit.message }} - RED_WATCH_AUTHOR: ${{ github.event.head_commit.author.name }} - run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl diff --git a/tools/ci/red-watch.mjs b/tools/ci/red-watch.mjs index 3d401e21..4f585edd 100755 --- a/tools/ci/red-watch.mjs +++ b/tools/ci/red-watch.mjs @@ -4,10 +4,12 @@ // opens the Actions page to learn a branch is red. // Node 22, standard library only. // -// node tools/ci/red-watch.mjs record --file in the workflow's `red` job (runs on igneum-build-1 after a -// failed run): reads the run from the GitHub environment and +// node tools/ci/red-watch.mjs record --file in .github/workflows/ci-red.yml (a workflow_run job on +// igneum-build-1 after a failed ci run on any branch): reads the +// FAILED run from RED_WATCH_* (the workflow_run payload; the +// GITHUB_* variables there describe the watcher's own run) and // the failed jobs and steps from the API with the job's own -// token, appends ONE JSON line for this run id (idempotent) +// token, appends ONE JSON line for that run id (idempotent) // node tools/ci/red-watch.mjs post --file [--live] on the box, every minute as `build` (igneum-ci-red.timer): // every recorded run not yet posted goes as one line to the // hidden updates channel (DISCORD_WEBHOOK_UPDATES in the @@ -63,15 +65,22 @@ export function readLines(file) { return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean); } +// The run being recorded: the FAILED run from RED_WATCH_* when the watcher runs as a workflow_run job (ci-red.yml), else +// the job's own run from GITHUB_* (the inline shape, kept for a branch whose ci.yml still carries the old `red` job). +export const watchedRunId = (env = process.env) => env.RED_WATCH_RUN_ID || env.GITHUB_RUN_ID; export function runFromEnv(env = process.env) { - const need = ['GITHUB_RUN_ID', 'GITHUB_REPOSITORY', 'GITHUB_REF_NAME', 'GITHUB_SHA', 'GITHUB_WORKFLOW']; + const need = ['GITHUB_REPOSITORY', 'GITHUB_RUN_ID']; for (const k of need) if (!env[k]) throw new Error(`record: ${k} is not set (this command runs inside a GitHub Actions job)`); + const pick = (own, fallback) => env[own] || env[fallback] || ''; const server = env.GITHUB_SERVER_URL || 'https://github.com'; + const id = String(watchedRunId(env)); + const sha = pick('RED_WATCH_SHA', 'GITHUB_SHA'); + if (!sha) throw new Error('record: neither RED_WATCH_SHA nor GITHUB_SHA is set'); return { - run_id: String(env.GITHUB_RUN_ID), attempt: Number(env.GITHUB_RUN_ATTEMPT || 1), workflow: env.GITHUB_WORKFLOW, - branch: env.GITHUB_REF_NAME, sha: env.GITHUB_SHA.slice(0, 7), event: env.GITHUB_EVENT_NAME || '', - actor: env.GITHUB_ACTOR || '', author: env.RED_WATCH_AUTHOR || '', // who pushed (the GitHub login), who the head commit names - url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`, at: new Date().toISOString(), + run_id: id, attempt: Number(pick('RED_WATCH_ATTEMPT', 'GITHUB_RUN_ATTEMPT') || 1), workflow: pick('RED_WATCH_WORKFLOW', 'GITHUB_WORKFLOW'), + branch: pick('RED_WATCH_BRANCH', 'GITHUB_REF_NAME'), sha: sha.slice(0, 7), event: pick('RED_WATCH_EVENT', 'GITHUB_EVENT_NAME'), + actor: pick('RED_WATCH_ACTOR', 'GITHUB_ACTOR'), author: env.RED_WATCH_AUTHOR || '', // who pushed (the GitHub login), who the head commit names + url: env.RED_WATCH_URL || `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${id}`, at: new Date().toISOString(), }; } @@ -79,7 +88,7 @@ export function runFromEnv(env = process.env) { // (the caller) is skipped by name. Any API trouble gives an empty list and a note, never a thrown error: the line is // the thing that must land. export async function failedJobs(env = process.env, fetchImpl = fetch) { - const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; const id = env.GITHUB_RUN_ID; + const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; const id = watchedRunId(env); const api = env.GITHUB_API_URL || 'https://api.github.com'; if (!token) return { failed: [], note: 'no GITHUB_TOKEN; failed steps not read' }; try { @@ -238,6 +247,18 @@ async function selfTest() { await record(fileFeature, envFeature, fakeFetch); const textFeature = formatLine(readLines(fileFeature)[0]); if (!/^CI red: ci on ca3-v4-node @0f0abc6 "Merge box-work[^"]*": pushed by igneum-labs; site build at/.test(textFeature)) fails.push(`format on a feature branch: ${textFeature}`); + // the workflow_run shape (ci-red.yml): GITHUB_* describe the watcher's own run, RED_WATCH_* the failed one; the line is the failed run's + const envRun = { GITHUB_RUN_ID: '999', GITHUB_RUN_ATTEMPT: '1', GITHUB_REPOSITORY: 'igneum-network/igneum', GITHUB_REF_NAME: 'master', GITHUB_SHA: 'ffffffffffff', GITHUB_WORKFLOW: 'ci-red', GITHUB_ACTOR: 'igneum-labs', GITHUB_TOKEN: 'x', + RED_WATCH_RUN_ID: '37620364667', RED_WATCH_ATTEMPT: '2', RED_WATCH_WORKFLOW: 'ci', RED_WATCH_BRANCH: 'ca3-v4-node', RED_WATCH_SHA: '26a4b0f1deadbeef', RED_WATCH_EVENT: 'push', + RED_WATCH_URL: 'https://github.com/igneum-network/igneum/actions/runs/37620364667', RED_WATCH_ACTOR: 'igneum-labs', RED_WATCH_TITLE: 'Counter ASIC 3.0 node plan 6.7', RED_WATCH_AUTHOR: 'igneum-labs' }; + const fileRun = path.join(dir, 'workflow-run.jsonl'); const asked = []; + const askingFetch = async (url) => { asked.push(url); return { ok: true, status: 200, json: async () => jobs }; }; + await record(fileRun, envRun, askingFetch); + const lr = readLines(fileRun)[0]; + if (lr.run_id !== '37620364667' || lr.attempt !== 2 || lr.branch !== 'ca3-v4-node' || lr.sha !== '26a4b0f' || lr.workflow !== 'ci') fails.push(`workflow_run shape: recorded ${JSON.stringify({ run_id: lr.run_id, attempt: lr.attempt, branch: lr.branch, sha: lr.sha, workflow: lr.workflow })}, expected the failed run, not the watcher's`); + if (!asked[0] || !asked[0].includes('/actions/runs/37620364667/jobs')) fails.push(`workflow_run shape: the jobs API was asked for ${asked[0]}, not the failed run`); + const textRun = formatLine(lr); + if (!/^CI red: ci on ca3-v4-node @26a4b0f "Counter ASIC 3.0 node plan 6.7": pushed by igneum-labs \(commit by igneum-labs\); site build at "identity grep of the public export list"; simulators at "\(job never started: runner or billing\)" https:\/\/github.com\/igneum-network\/igneum\/actions\/runs\/37620364667$/.test(textRun)) fails.push(`workflow_run line: ${textRun}`); // post, dry run: prints, sends nothing, marks nothing let printed = []; const log = (s) => printed.push(s); const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; }; @@ -286,7 +307,7 @@ async function selfTest() { if (!d3.sent) fails.push('digest: not sent the next day'); fs.rmSync(dir, { recursive: true, force: true }); if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); } - console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00'); + console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author, the failed run and not the watcher\'s own under workflow_run; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00'); } const cmd = args[0];