diff --git a/.github/workflows/ci-red.yml b/.github/workflows/ci-red.yml index 48b3deff7..401005dbf 100644 --- a/.github/workflows/ci-red.yml +++ b/.github/workflows/ci-red.yml @@ -3,7 +3,7 @@ # ci.yml never enters it (7 October 2026: the inline `red` job of ci.yml was conditioned on master and release-*, and # a feature branch would have waited for a merge of master before its reds were posted at all). # -# One line per failed run (tools/ci/red-watch.mjs record, idempotent per run attempt) to /srv/ci-red/red.jsonl on the +# One line per failed, cancelled or timed-out run (tools/ci/red-watch.mjs record, idempotent per run attempt) to /srv/ci-red/red.jsonl on the # box; the box's igneum-ci-red.timer posts each new line once to the hidden updates channel, naming the branch, the # commit, the red check and the pushing author. Runs on the box's own runner (not a GitHub-hosted machine: the billing # block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told). Never blocks a @@ -16,7 +16,9 @@ on: jobs: red: name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file) - if: ${{ github.event.workflow_run.conclusion == 'failure' }} + # failure, and since 7 October 2026 (17:2x UK) cancelled and timed_out too: a job that hangs into its timeout-minutes or a run + # someone cancels is a run that never answered, and a lane reads it like a red (tools/ci/red-watch.mjs names the kind) + if: ${{ github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'cancelled' || github.event.workflow_run.conclusion == 'timed_out' }} # the label ci-red is on igneum-build-1 only (added through the runners API on 7 October 2026; the default of # RUNNER_LABELS in provision.sh carries it): the record file and the poster (igneum-ci-red.timer, the webhook file) # live on that box, and the pool label igneum-build-1 is shared with igneum-build-2 since the same day @@ -35,6 +37,7 @@ jobs: RED_WATCH_RUN_ID: ${{ github.event.workflow_run.id }} RED_WATCH_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} RED_WATCH_WORKFLOW: ${{ github.event.workflow_run.name }} + RED_WATCH_CONCLUSION: ${{ github.event.workflow_run.conclusion }} RED_WATCH_BRANCH: ${{ github.event.workflow_run.head_branch }} RED_WATCH_SHA: ${{ github.event.workflow_run.head_sha }} RED_WATCH_EVENT: ${{ github.event.workflow_run.event }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fefdd8b69..a09e40319 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: # code=true (no `before` to compare from), as does any error reading the compare API: when in doubt, run. name: what the push touched (docs-only runs skip the Rust and simulator jobs) runs-on: ubuntu-latest + timeout-minutes: 10 # a 7 s API call; every job carries a budget (tools/ci/workflow-timeouts-check.sh) outputs: code: ${{ steps.classify.outputs.code }} steps: @@ -62,6 +63,7 @@ jobs: needs: changes if: ${{ needs.changes.outputs.code == 'true' }} runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }} + timeout-minutes: 60 # the box's suite ran 45 s to 2 min 40 s on 7 October 2026; a hosted fallback compiles cold steps: - uses: actions/checkout@v4 - name: toolchain @@ -81,6 +83,7 @@ jobs: # queue read 22); a feature-branch code push runs the igneum-pow tests alone. tools/ci/sims-branch-check.sh holds this rule. if: ${{ needs.changes.outputs.code == 'true' && ((github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))) || (github.event_name == 'pull_request' && (github.base_ref == 'master' || startsWith(github.base_ref, 'release-')))) }} runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }} + timeout-minutes: 45 # two simulators under 120 s each by their own timeout, plus a hosted fallback's pip install steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -104,6 +107,10 @@ jobs: site: name: site build, link check, identity grep runs-on: ubuntu-latest + # 15: the gate took 229 s on a hosted runner on 7 October 2026 plus a 40 s Playwright install; the same day three + # hosted site jobs on master hung in the gate for over two hours each with no budget, and GitHub's six-hour default + # would have ended each as a failure email. A hung job is a red the watcher posts (ci-red.yml fires on timed_out). + timeout-minutes: 15 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -118,4 +125,4 @@ jobs: run: bash tools/ci/pre-push.sh --ci - name: public stats API answers with the documented fields (the live site; master only, the endpoints exist there after the merge) if: github.ref == 'refs/heads/master' - run: node tools/ci/public-api-check.mjs https://igneum.network + run: bash tools/ci/retry-once.sh public-api node tools/ci/public-api-check.mjs https://igneum.network # a live host: one retry before red diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 82aa2f60c..f4f486d5c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -216,8 +216,14 @@ jobs: run: | $payload = Resolve-Path 'packaging\windows\igneum-windows-app' function Run-Capture([string]$exe, [string]$flag) { + # Start-Process -Wait on an exe that exits in milliseconds can throw "Cannot process request because the process has + # exited" before it attaches (release-0.3.21 run 37653903394, 7 October 2026, 17:40 UK): one retry before the verdict. $out = Join-Path $env:RUNNER_TEMP ('smoke-' + [IO.Path]::GetRandomFileName() + '.txt') - $p = Start-Process -FilePath $exe -ArgumentList $flag -Wait -NoNewWindow -PassThru -RedirectStandardOutput $out + $p = $null + foreach ($try in 1, 2) { + try { $p = Start-Process -FilePath $exe -ArgumentList $flag -Wait -NoNewWindow -PassThru -RedirectStandardOutput $out; break } + catch { if ($try -eq 2) { throw }; Write-Host ("Start-Process on {0} {1} failed once ({2}); second try" -f (Split-Path -Leaf $exe), $flag, $_.Exception.Message); Start-Sleep -Milliseconds 500 } + } $text = if (Test-Path $out) { (Get-Content $out -Raw) } else { '' } Write-Host ("{0} {1} -> exit {2}: {3}" -f (Split-Path -Leaf $exe), $flag, $p.ExitCode, $text.Trim()) if ($p.ExitCode -ne 0) { throw "$exe $flag exited $($p.ExitCode)" } diff --git a/CLAUDE.md b/CLAUDE.md index 87ff4c078..830755af3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -132,6 +132,7 @@ The Mac crashed and rebooted under agent load (about twenty lanes, local cargo t - The pre-push gate is the same script CI runs: `tools/ci/pre-push.sh` (installed by `tools/ci/install-hooks.sh`; `--hook` before a push to master or release-*, `--ci` in the workflow). A check is added there, never only in ci.yml. On a feature branch the hook runs the light gate: the two structural checks plus the two never-push classes, the no-secrets check and the identity grep (7 October 2026). - After every push, the pushing lane reads the conclusion when it lands (`gh run list --branch --limit 1`; `gh run view --log-failed` on a red) and owns a red before the next push; pushes are never held for it. A red on any branch is reported to main the moment it is seen. The red watcher (.github/workflows/ci-red.yml, from master, every branch) posts each failed run with its branch, commit, red check and pushing author. Record: 7 October 2026, 11:26 to 12:47 UK, eight red runs on ca3-v4-node over three gate summaries carrying a 64-hex key, read by nobody; 31 runs queued on one runner at 13:15 UK, so igneum-build-2 joined the pool and docs-only pushes skip the compile jobs. - A code push (anything outside docs/, site/ and *.md) goes out only after that exact commit's crate suite is green on a build box (the suite's RESULT line in the push's own notes or the commit message); a test build the box never ran is not pushed. Record: 7 October 2026, 14:10 to 14:13 UK, four red runs in three minutes (a test-build compile error on ca3-v4-amend, the hidden-console check on miner-reliability, the installer's inputs pin on release-0.3.20), each a push ahead of its own box result, each emailed to Josh. +- Master takes only what CI has already passed (7 October 2026, 17:2x UK): a merge lands only when the branch's own `ci` run is green on the exact commit being merged, read from the runs API (`tools/ci/ci-state.mjs`), never the local stamp alone; `tools/ci/merge-to-master.sh` pushes the branch for a run when there is none, waits for a queued one printing the clock, refuses a red one and refuses any merge onto a red master except the declared fix (`--fixes-master`); the pre-push hook refuses a push to master whose commit (or merge parent) has no green run; a feature-branch push prints the branch's previous red first. GitHub's branch protection is unavailable on the free plan for a private repository (the API answers 403), so the two scripts are the enforcement. - Research and operations documents live outside the public export list: name them in `tools/ci/export-exclude.txt` (read by the identity check and by the mirror's sync.sh). What stays in the list is read by the public and must pass the identity grep. - A path Windows cannot hold (colon, trailing dot or space, reserved name, over 240 characters) never enters a commit: the pre-commit hook runs `tools/ci/windows-paths-check.sh --staged`. - Record: docs/analysis/ci-failures-2026-10-06.md (168 non-green runs in three days classified; 126 on master; all but two classes were tree checks that the gate now runs locally first). diff --git a/tools/ci/README.md b/tools/ci/README.md index 342f555ea..2edf6004b 100644 --- a/tools/ci/README.md +++ b/tools/ci/README.md @@ -4,4 +4,9 @@ |---|---|---| | no text overlaps (`overlap-check.mjs`) | A served page, or a miner or wallet screen (behind `IGNEUM_OVERLAP_APPS=1`), where a visible run of text is covered by another element (a pill over a caption, a label over a value, a card over its neighbour, text under the header at rest), clipped by an overflow-hidden ancestor, or past the viewport; a page that scrolls sideways. Five widths, light and dark, the home hero at rest and at each step. A fixture with one deliberate overlap of each kind must be flagged first (`--self-test`). Needs a headless Chromium: CI installs Playwright; the Mac ships the pages to build-2 (`infra/build-server/overlap-browser.sh`). | 7 October 2026: the hero's step pill sat on the caption's second line ("Two thirds of the weight sign. The checkpoint locks.") at every desktop width, found by Josh on the live site | +| master takes only CI-passed commits (`ci-state.mjs`, `merge-to-master.sh`, the hook's `master_ci_ok`) | A push to master whose commit, or whose merge's branch parent, has no green `ci` run on that exact sha (the runs API through gh: red, queued, none or gh unreachable all refuse); a merge onto a master whose last compiled run is red, unless declared the fix (`--fixes-master`). The merge tool pushes an unrun branch for a run and waits for a queued one with the clock. A feature-branch push prints the branch's previous red first (`--branch-red`). | 7 October 2026: era-vdf's tip 0e2d6b1c merged with no ci run; master's igneum-pow suite red from 16:31 UK under five docs-only green merges | +| every workflow job carries timeout-minutes (`workflow-timeouts-check.sh`) | A job in .github/workflows without `timeout-minutes`, or a budget off its measured line (site 15, changes 10, pow 60, sims 45). | 7 October 2026: three hosted site jobs on master hung over two hours each in the tree gate; the six-hour default was the only stop | +| a box or network check gets one retry (`retry-once.sh`) | Nothing by itself: wraps the box-locks check, the scene parity check and the live public API check so a first failure is printed and retried once; the second is the verdict. The checks keep their own skip line on a runner without the resource. | 7 October 2026 | +| the red watcher fires on cancelled and timed-out runs too (`ci-red.yml`, `red-watch.mjs`) | The watcher's `if` missing any of failure, cancelled, timed_out, or the conclusion not handed to the record step (the self-test reads the workflow file); the line names the kind: CI red, CI cancelled, CI timed out. | 7 October 2026 | + | kill by exact command or pid file (owed as a check) | 6 October 2026, 21:09Z: a Mac-side `pkill -f ` matched nothing (the log name was a redirect, not part of the command line), the roll-everything script lived on and wiped a box it had been told to hold. Rule: a job is stopped by its pid file (`tools/fleet/fleet-bg.sh start|stop `) or by a pattern anchored on its exact command line (`^python3 -u /root/fleet/in/box-prover.py`), never by a word that may or may not appear in it. The check that flags a `pkill -f`/`pgrep -f` whose literal is a path or a name that never starts a command line is owed to the CI lane | diff --git a/tools/ci/ci-state.mjs b/tools/ci/ci-state.mjs new file mode 100755 index 000000000..1e91b5811 --- /dev/null +++ b/tools/ci/ci-state.mjs @@ -0,0 +1,152 @@ +#!/usr/bin/env node +// What CI says about a commit or a branch, read from the runs API through gh (the Mac's gh login; the repository is +// igneum-network/igneum unless IGNEUM_REPO says otherwise). The merge tool and the pre-push hook read these lines, so a +// merge lands on master only when the branch's own ci run is green on the exact commit being merged (standing rule, +// 7 October 2026, 17:2x UK: era-vdf's tip 0e2d6b1c was merged with no ci run at all and master's igneum-pow suite went +// red for 40 minutes, hidden by docs-only merges whose runs skip the compile job). +// Node 22, standard library only; gh does the HTTP. +// +// node tools/ci/ci-state.mjs one line: " " +// state: success | failure | cancelled | timed_out | pending | none | unknown +// (the NEWEST ci run on that exact commit; a re-run replaces the first attempt) +// node tools/ci/ci-state.mjs --master-code the verdict of master's newest completed ci run whose compile job (igneum-pow) +// RAN: a docs-only push skips it and its green hides a red suite +// node tools/ci/ci-state.mjs --branch-red prints "previous CI red on : ..." when the branch's newest completed ci +// run is red and no newer run is green; prints nothing otherwise; exit 0 always +// node tools/ci/ci-state.mjs --self-test a fake gh on PATH answers every case: success, failure, pending, none, newest +// wins, the docs-only skip walked past, the branch line, a gh error = unknown +// +// Exit 0 for every answered query (callers read the first word); 2 on usage; the self-test exits 1 on a failure. +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +const REPO = process.env.IGNEUM_REPO || 'igneum-network/igneum'; +const FIELDS = 'databaseId,status,conclusion,headSha,url,createdAt,event'; +const RED = new Set(['failure', 'cancelled', 'timed_out', 'startup_failure', 'action_required']); + +function gh(args) { + const env = { ...process.env, PATH: `${process.env.PATH || ''}:/opt/homebrew/bin:/usr/local/bin` }; + const r = spawnSync('gh', args, { encoding: 'utf8', env, timeout: 60000 }); + if (r.error) throw new Error(`gh: ${r.error.message}`); + if (r.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')}: exit ${r.status}: ${(r.stderr || '').trim().slice(0, 200)}`); + return JSON.parse(r.stdout || 'null'); +} +function fullSha(sha) { + if (/^[0-9a-f]{40}$/.test(sha)) return sha; // gh's --commit filter matches the full sha only (an abbreviation answers nothing) + const r = spawnSync('git', ['rev-parse', '--verify', `${sha}^{commit}`], { encoding: 'utf8' }); + if (r.status !== 0) throw new Error(`${sha} is not a commit here`); + return r.stdout.trim(); +} +export const newest = (runs) => [...(runs || [])].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt))[0]; +export function verdictOf(run) { + if (!run) return 'none'; + if (run.status !== 'completed') return 'pending'; + return run.conclusion || 'pending'; +} +export function firstRed(jobs) { + for (const j of jobs || []) { + if (j.conclusion === 'success' || j.conclusion === 'skipped' || j.conclusion == null) continue; + const s = (j.steps || []).find((x) => x.conclusion && x.conclusion !== 'success' && x.conclusion !== 'skipped'); + return `${j.name.replace(/,.*$/, '')} at "${s ? s.name : '(no step)'}"`; + } + return ''; +} +const london = (iso) => new Intl.DateTimeFormat('en-GB', { timeZone: 'Europe/London', hour: '2-digit', minute: '2-digit', hour12: false }).format(new Date(iso)) + ' UK'; +const runsForSha = (sha) => gh(['run', 'list', '--repo', REPO, '--workflow', 'ci', '--commit', sha, '--limit', '10', '--json', FIELDS]) || []; +const runsForBranch = (branch, limit = 12) => gh(['run', 'list', '--repo', REPO, '--workflow', 'ci', '--branch', branch, '--limit', String(limit), '--json', FIELDS]) || []; +const jobsOf = (id) => (gh(['run', 'view', String(id), '--repo', REPO, '--json', 'jobs']) || {}).jobs || []; + +export function stateOfSha(sha) { + const run = newest(runsForSha(fullSha(sha))); + const state = verdictOf(run); + if (!run) return `none - - no ci run on ${sha.slice(0, 8)} yet`; + let detail = state === 'pending' ? `${run.status} since ${london(run.createdAt)}` : `${run.event} run at ${london(run.createdAt)}`; + if (RED.has(state)) { const red = firstRed(jobsOf(run.databaseId)); if (red) detail += `: ${red}`; } + return `${state} ${run.databaseId} ${run.url} ${detail}`; +} +export function masterCodeState() { + const runs = [...runsForBranch('master', 12)].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt)); + for (const run of runs) { + if (run.status !== 'completed') continue; + const jobs = jobsOf(run.databaseId); + const pow = jobs.find((j) => /^igneum-pow/.test(j.name)); + if (!pow || pow.conclusion === 'skipped') continue; // a docs-only push: its green says nothing about the suite + const red = RED.has(run.conclusion) ? firstRed(jobs) : ''; + return `${run.conclusion} ${run.databaseId} ${run.url} master @${run.headSha.slice(0, 8)} at ${london(run.createdAt)}, compile job ${pow.conclusion}${red ? `: ${red}` : ''}`; + } + return 'none - - no completed master ci run with the compile job among the last 12'; +} +export function branchRedLine(branch) { + const runs = [...runsForBranch(branch, 12)].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt)); + const done = runs.find((r) => r.status === 'completed'); + if (!done || !RED.has(done.conclusion)) return ''; + const red = firstRed(jobsOf(done.databaseId)); + return `previous CI red on ${branch}: @${done.headSha.slice(0, 7)} ${done.conclusion} at ${london(done.createdAt)}${red ? `: ${red}` : ''} ${done.url} (no newer green run on this branch; this push gets its own run, read it)`; +} + +async function selfTest() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-state-')); + const fake = path.join(dir, 'gh'); + // the fake gh answers `run list` from list-.json and `run view ` from view-.json; FAKE_GH_FAIL=1 fails + fs.writeFileSync(fake, `#!/usr/bin/env bash +[ "\${FAKE_GH_FAIL:-0}" = 1 ] && { echo "HTTP 502 from the fake" >&2; exit 1; } +key=""; prev="" +for a in "$@"; do case "$prev" in --commit|--branch) key="$a" ;; esac; prev="$a"; done +case "$1 $2" in + "run list") f="$FAKE_GH_DIR/list-$key.json"; [ -f "$f" ] && cat "$f" || echo '[]' ;; + "run view") f="$FAKE_GH_DIR/view-$3.json"; [ -f "$f" ] && cat "$f" || echo '{"jobs":[]}' ;; + *) echo "fake gh: unknown $*" >&2; exit 1 ;; +esac +`, { mode: 0o755 }); + const sha = (c) => c.repeat(40); + const run = (id, status, conclusion, created, headSha = sha('a')) => ({ databaseId: id, status, conclusion, headSha, url: `https://github.com/x/y/actions/runs/${id}`, createdAt: created, event: 'push' }); + const w = (name, obj) => fs.writeFileSync(path.join(dir, name), JSON.stringify(obj)); + w(`list-${sha('a')}.json`, [run(1, 'completed', 'success', '2026-10-07T15:00:00Z')]); + w(`list-${sha('b')}.json`, [run(2, 'completed', 'failure', '2026-10-07T15:00:00Z', sha('b'))]); + w('view-2.json', { jobs: [{ name: 'site build, link check', conclusion: 'success', steps: [] }, { name: 'igneum-pow tests, igneum-census build', conclusion: 'failure', steps: [{ name: 'toolchain', conclusion: 'success' }, { name: 'igneum-pow tests (release)', conclusion: 'failure' }] }] }); + w(`list-${sha('c')}.json`, [run(3, 'in_progress', null, '2026-10-07T15:00:00Z', sha('c'))]); + w(`list-${sha('d')}.json`, [run(4, 'completed', 'failure', '2026-10-07T14:00:00Z', sha('d')), run(5, 'completed', 'success', '2026-10-07T15:00:00Z', sha('d'))]); // the re-run wins + // master: the newest run is a docs-only green (compile job skipped); the one before it ran the compile job and is red + w('list-master.json', [run(10, 'completed', 'success', '2026-10-07T16:00:00Z', sha('1')), run(11, 'completed', 'failure', '2026-10-07T15:30:00Z', sha('2')), run(12, 'completed', 'success', '2026-10-07T15:00:00Z', sha('3'))]); + w('view-10.json', { jobs: [{ name: 'igneum-pow tests, igneum-census build', conclusion: 'skipped', steps: [] }, { name: 'site build', conclusion: 'success', steps: [] }] }); + w('view-11.json', { jobs: [{ name: 'igneum-pow tests, igneum-census build', conclusion: 'failure', steps: [{ name: 'igneum-pow tests (release)', conclusion: 'failure' }] }] }); + // branches: x red newest; y green newest; z pending newest over a red + w('list-x.json', [run(20, 'completed', 'failure', '2026-10-07T15:00:00Z', sha('e'))]); + w('view-20.json', { jobs: [{ name: 'site build, link check, identity grep', conclusion: 'failure', steps: [{ name: 'the tree gate', conclusion: 'failure' }] }] }); + w('list-y.json', [run(21, 'completed', 'success', '2026-10-07T15:00:00Z'), run(22, 'completed', 'failure', '2026-10-07T14:00:00Z')]); + w('list-z.json', [run(23, 'queued', null, '2026-10-07T15:10:00Z'), run(24, 'completed', 'cancelled', '2026-10-07T15:00:00Z', sha('f'))]); + const me = new URL(import.meta.url).pathname; + const ask = (args, extraEnv = {}) => { + const r = spawnSync(process.execPath, [me, ...args], { encoding: 'utf8', env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, FAKE_GH_DIR: dir, ...extraEnv } }); + return { out: (r.stdout || '').trim(), code: r.status, err: (r.stderr || '').trim() }; + }; + const fails = []; + const expect = (name, got, re) => { if (!re.test(got.out) || got.code !== 0) fails.push(`${name}: got exit ${got.code} "${got.out}" ${got.err}`); }; + expect('success', ask([sha('a')]), /^success 1 https:\/\/github.com\/x\/y\/actions\/runs\/1 push run at /); + expect('failure names the red step', ask([sha('b')]), /^failure 2 \S+ push run at \d\d:\d\d UK: igneum-pow tests at "igneum-pow tests \(release\)"$/); + expect('pending', ask([sha('c')]), /^pending 3 \S+ in_progress since /); + expect('newest wins', ask([sha('d')]), /^success 5 /); + expect('none', ask([sha('9')]), /^none - - no ci run on 99999999 yet$/); + expect('gh error is unknown', ask([sha('a')], { FAKE_GH_FAIL: '1' }), /^unknown - - gh run list: exit 1: HTTP 502 from the fake/); + expect('master-code walks past the docs-only green', ask(['--master-code']), /^failure 11 \S+ master @22222222 at \d\d:\d\d UK, compile job failure: igneum-pow tests at "igneum-pow tests \(release\)"$/); + expect('branch red line', ask(['--branch-red', 'x']), /^previous CI red on x: @eeeeeee failure at \d\d:\d\d UK: site build at "the tree gate" https:\/\/github.com\/x\/y\/actions\/runs\/20 \(no newer green/); + const y = ask(['--branch-red', 'y']); if (y.out !== '' || y.code !== 0) fails.push(`branch green prints nothing: "${y.out}" exit ${y.code}`); + expect('branch pending over a cancelled run still names the red', ask(['--branch-red', 'z']), /^previous CI red on z: @fffffff cancelled at /); + const noArg = ask([]); if (noArg.code !== 2) fails.push(`usage: exit ${noArg.code}`); + fs.rmSync(dir, { recursive: true, force: true }); + if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); } + console.log('self-test passed: a commit answers success, failure (with the red step), pending, none; the newest run wins; a gh error is unknown; --master-code walks past a docs-only green to the run that compiled; --branch-red names the newest completed red and stays silent on green'); +} + +const args = process.argv.slice(2); +try { + if (args[0] === '--self-test') await selfTest(); + else if (args[0] === '--master-code') console.log(masterCodeState()); + else if (args[0] === '--branch-red') { if (!args[1]) { console.error('usage: ci-state.mjs --branch-red '); process.exit(2); } console.log(branchRedLine(args[1])); } + else if (args[0] && !args[0].startsWith('-')) console.log(stateOfSha(args[0])); + else { console.error('usage: tools/ci/ci-state.mjs | --master-code | --branch-red | --self-test'); process.exit(2); } +} catch (e) { + console.log(`unknown - - ${e.message.replace(/\s+/g, ' ').slice(0, 300)}`); +} diff --git a/tools/ci/merge-to-master.sh b/tools/ci/merge-to-master.sh index 126545703..ada5a2c4f 100755 --- a/tools/ci/merge-to-master.sh +++ b/tools/ci/merge-to-master.sh @@ -5,34 +5,169 @@ # two-parent merge of the branch onto the exact remote tip, which the hook lets through on the light gate (20 s) while CI runs # the full gate on landing. Retries while master moves. Never force; never from a dirty branch. # -# tools/ci/merge-to-master.sh [] [--tries N] default: the current branch, 6 tries -# MERGE_REMOTE=box tools/ci/merge-to-master.sh ... land on another remote's master (the box mirror /srv/igneum.git while GitHub is -# suspended, 7 October 2026); the default stays origin +# The CI rule (standing rule, 7 October 2026, 17:2x UK): a merge lands only when the branch's OWN ci run is green on the exact +# commit being merged, read from the runs API (tools/ci/ci-state.mjs), not the local stamp alone. No run yet: the branch is pushed +# so CI runs it. A queued or running run: this tool waits, printing the UK clock, up to --ci-wait minutes (default 25). A red run: +# refused with the run's red check; fix the branch and push a new commit. And master itself must not be red: the newest master run +# that compiled (docs-only runs skip the compile job and their green says nothing) must be success, or the merge is refused unless +# it is the fix (--fixes-master). Record: era-vdf's tip 0e2d6b1c was merged with no ci run; master's igneum-pow suite was red from +# 16:31 UK and five docs-only merges landed green over it. The pre-push hook holds the same rule (pre-push.sh master_ci_ok). +# +# tools/ci/merge-to-master.sh [] [--tries N] [--ci-wait MIN] [--fixes-master] [--remote ] default: the current branch, 6 tries, origin +# MERGE_REMOTE=box tools/ci/merge-to-master.sh ... the same as --remote box (the box mirror /srv/igneum.git while GitHub is suspended, 7 October 2026) +# tools/ci/merge-to-master.sh --self-test the CI verdicts, with a fake ci-state: green goes, red refuses, master red refuses +# unless --fixes-master, none pushes the branch, pending waits then goes set -euo pipefail ROOT=$(git rev-parse --show-toplevel); cd "$ROOT" -BRANCH="$(git rev-parse --abbrev-ref HEAD)"; TRIES=6; R="${MERGE_REMOTE:-origin}" -while [ $# -gt 0 ]; do case "$1" in --tries) TRIES="$2"; shift 2 ;; -*) echo "unknown option $1" >&2; exit 2 ;; *) BRANCH="$1"; shift ;; esac; done +BRANCH="$(git rev-parse --abbrev-ref HEAD)"; TRIES=6; CI_WAIT_MIN="${CI_WAIT_MIN:-25}"; FIXES_MASTER=0; SELF_TEST=0; REMOTE="${MERGE_REMOTE:-origin}" +while [ $# -gt 0 ]; do case "$1" in --tries) TRIES="$2"; shift 2 ;; --ci-wait) CI_WAIT_MIN="$2"; shift 2 ;; --fixes-master) FIXES_MASTER=1; shift ;; --remote) REMOTE="$2"; shift 2 ;; --self-test) SELF_TEST=1; shift ;; -*) echo "unknown option $1" >&2; exit 2 ;; *) BRANCH="$1"; shift ;; esac; done +# --remote : land on another remote's master (a box mirror, build@:/srv/igneum.git, while GitHub is unreachable; main's +# ruling of 7 October 2026, 19:5x UK). CI runs on GitHub only, so the CI rule binds the origin remote; on a mirror the box gate +# stamp is the verdict and the merge message says so. IGNEUM_MASTER_EXCEPTION, when set, is printed by the hook with the push. +remote_is_github() { case "$(git remote get-url "$REMOTE" 2>/dev/null)" in *github.com*) return 0 ;; *) return 1 ;; esac; } +CI_STATE="${CI_STATE_CMD:-node tools/ci/ci-state.mjs}" # the self-test swaps in a fake +clock() { TZ=Europe/London date '+%H:%M %Z'; } + +# ci_gate : 0 when the branch's own ci run on is green; pushes the branch when there is no run; +# waits on a queued or running run; refuses (1) a red or unknown one. master_gate: refuses (1) when master's last compiled run is red. +ci_gate() { + local sha="$1" branch="$2"; shift 2; local line state deadline pushed=0 + deadline=$(( $(date +%s) + ${CI_WAIT_SECS:-$(( CI_WAIT_MIN * 60 ))} )) + while :; do + line=$($CI_STATE "$sha" 2>&1); state="${line%% *}" + case "$state" in + success) echo "merge-to-master: ci on ${sha:0:8} is green: $line"; return 0 ;; + none) if [ "$pushed" = 1 ]; then echo "merge-to-master: waiting for a ci run to appear on ${sha:0:8} ($(clock))"; else echo "merge-to-master: no ci run on ${sha:0:8} yet; pushing $branch so CI runs it ($(clock))"; "$@" || { echo "merge-to-master: the branch push failed; CI cannot run ${sha:0:8}" >&2; return 1; }; pushed=1; fi ;; + pending) echo "merge-to-master: ci on ${sha:0:8} is ${line#* * * }; waiting ($(clock))" ;; + unknown) echo "merge-to-master: ci on ${sha:0:8} could not be read (${line#* * * }); GitHub or gh wobbling, asking again ($(clock))" ;; # 7 Oct 2026 18:0x UK: 500s on pushes, a 404 on a live run id + *) echo "merge-to-master: REFUSED. ci on ${sha:0:8} is $line" >&2; echo " A branch whose own ci run is not green never merges (CI red is stop-the-line). Fix the branch, push a new commit, and run this again." >&2; return 1 ;; + esac + [ "$(date +%s)" -lt "$deadline" ] || { echo "merge-to-master: REFUSED. ci on ${sha:0:8} gave no verdict in $CI_WAIT_MIN minutes ($line); run it again when the queue drains (gh run list --branch $branch --limit 3)" >&2; return 1; } + sleep "${CI_POLL_SECS:-30}" + done +} +master_gate() { + local line state try + for try in 1 2 3; do # an unknown read (gh or GitHub wobbling) is asked again, three times over a minute, before it counts + line=$($CI_STATE --master-code 2>&1); state="${line%% *}" + [ "$state" = unknown ] && [ "$try" -lt 3 ] && { echo "merge-to-master: master's verdict could not be read (${line#* * * }); asking again ($(clock))"; sleep "${CI_POLL_SECS:-30}"; continue; } + break + done + case "$state" in + success|none|pending) echo "merge-to-master: master's last compiled run: $line"; return 0 ;; + *) if [ "$FIXES_MASTER" = 1 ]; then echo "merge-to-master: master's last compiled run is $state and this merge is declared the fix (--fixes-master): $line"; return 0; fi + echo "merge-to-master: REFUSED. master is red: $line" >&2; echo " CI red is stop-the-line: the owner fixes or reverts first; only the fix merges, with --fixes-master." >&2; return 1 ;; + esac +} + +# After a landing, master goes to the box mirrors too (7 October 2026, 18:25 UTC: build-2's /srv/igneum.git master sat at 15:27's +# a4bca198 while GitHub's carried cf7d6ccb, and three branches were cut from the stale tip; the mirrors only ever received what a +# build happened to push). One push per mirror in IGNEUM_MIRRORS (default: both box files), fast-forward only, best effort: a mirror +# that is down prints a line and never fails the landing. +mirror_master() { # + local sha="$1" m host key="${BS_KEY:-$HOME/.ssh/igneum_ed25519}" list="${IGNEUM_MIRRORS:-}" + if [ -z "$list" ]; then for f in "$HOME/.config/igneum/build-server" "$HOME/.config/igneum/build-server-2"; do [ -s "$f" ] && list="$list $(head -1 "$f" | tr -d '[:space:]'):/srv/igneum.git"; done; fi + for m in $list; do + if GIT_SSH_COMMAND="ssh -i $key -o BatchMode=yes -o ConnectTimeout=10" git push -q "$m" "$sha:refs/heads/master" 2>/dev/null; then echo "merge-to-master: mirror $m master -> ${sha:0:8}" + else echo "merge-to-master: mirror $m did not take master ${sha:0:8} (down, or not a fast-forward); the next landing tries again"; fi + done +} + +if [ "$SELF_TEST" = 1 ]; then + fails=0; d=$(mktemp -d); fake="$d/ci-state.sh" + # the fake answers from $d/answer- (one line per call, consumed top to bottom; the last line repeats) and $d/answer-master + cat > "$fake" <<'FAKE' +#!/usr/bin/env bash +d=$(dirname "$0"); key="$1"; [ "$key" = --master-code ] && key=master +f="$d/answer-$key"; [ -f "$f" ] || { echo "none - - no ci run on $key yet"; exit 0; } +n=$(wc -l < "$f" | tr -d ' '); i=$(cat "$d/i-$key" 2>/dev/null || echo 1); [ "$i" -gt "$n" ] && i=$n +sed -n "${i}p" "$f"; echo $((i + 1)) > "$d/i-$key" +FAKE + chmod +x "$fake"; export CI_STATE_CMD="$fake" CI_POLL_SECS=0 CI_WAIT_MIN=1 + CI_STATE="$fake" + printf 'success 1 u push run at 16:00 UK +' > "$d/answer-green" + ci_gate green b false >/dev/null || { echo "self-test failed: a green branch run was refused"; fails=1; } + printf 'failure 2 u push run: igneum-pow tests at "igneum-pow tests (release)" +' > "$d/answer-red" + ci_gate red b false >/dev/null 2>&1 && { echo "self-test failed: a red branch run merged"; fails=1; } + printf 'unknown - - gh: exit 1\n' > "$d/answer-unk" + CI_WAIT_SECS=1 ci_gate unk b false >/dev/null 2>&1 && { echo "self-test failed: a lasting unknown verdict merged"; fails=1; } + printf 'unknown - - gh: HTTP 404\nsuccess 9 u push run\n' > "$d/answer-blip" + ci_gate blip b false >/dev/null 2>&1 || { echo "self-test failed: a GitHub blip (unknown, then green) was refused instead of waited through"; fails=1; } + printf 'pending 3 u queued since 16:00 UK +pending 3 u in_progress since 16:00 UK +success 3 u push run at 16:05 UK +' > "$d/answer-wait" + out=$(ci_gate wait b false 2>&1) || { echo "self-test failed: a pending run that turned green was refused: $out"; fails=1; } + [ "$(printf '%s +' "$out" | grep -c 'waiting')" = 2 ] || { echo "self-test failed: the wait did not print the clock twice: $out"; fails=1; } + # no run: the branch is pushed (the push command runs once), then the run appears and goes green + printf 'none - - no ci run on none yet +success 4 u push run +' > "$d/answer-none"; : > "$d/pushes" + ci_gate none b bash -c "echo pushed >> '$d/pushes'" >/dev/null || { echo "self-test failed: an unrun branch was refused instead of pushed"; fails=1; } + [ "$(wc -l < "$d/pushes" | tr -d ' ')" = 1 ] || { echo "self-test failed: the branch was pushed $(wc -l < "$d/pushes") times"; fails=1; } + # master red: refused; with --fixes-master: goes; master green or unknown-none: goes + printf 'failure 5 u master @1210158d, compile job failure +' > "$d/answer-master" + FIXES_MASTER=0 master_gate >/dev/null 2>&1 && { echo "self-test failed: a merge onto a red master was let through"; fails=1; } + rm -f "$d/i-master"; FIXES_MASTER=1 master_gate >/dev/null || { echo "self-test failed: the declared fix was refused on a red master"; fails=1; } + printf 'success 6 u master @e5171a32, compile job success +' > "$d/answer-master"; rm -f "$d/i-master" + FIXES_MASTER=0 master_gate >/dev/null || { echo "self-test failed: a green master refused a merge"; fails=1; } + printf 'unknown - - gh: HTTP 404\nsuccess 7 u master, compile job success\n' > "$d/answer-master"; rm -f "$d/i-master" + FIXES_MASTER=0 master_gate >/dev/null || { echo "self-test failed: an unknown master read that turned green refused a merge"; fails=1; } + printf 'unknown - - gh: exit 1\n' > "$d/answer-master"; rm -f "$d/i-master" + FIXES_MASTER=0 master_gate >/dev/null 2>&1 && { echo "self-test failed: a lasting unknown master read let a merge through"; fails=1; } + # the mirror push: a bare repository behind master is fast-forwarded; one that is not a fast-forward is left alone with a line + mkdir -p "$d/src" && ( cd "$d/src" && git init -q -b master . && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m one && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m two ) 2>/dev/null + git clone -q --bare "$d/src" "$d/mirror.git" && ( cd "$d/mirror.git" && git update-ref refs/heads/master "$(git rev-parse master~1)" ) + tip=$(git -C "$d/src" rev-parse master) + out=$( cd "$d/src" && IGNEUM_MIRRORS="$d/mirror.git" mirror_master "$tip" ) + [ "$(git -C "$d/mirror.git" rev-parse master)" = "$tip" ] || { echo "self-test failed: the mirror was not fast-forwarded to the landed master: $out"; fails=1; } + ( cd "$d/src" && git checkout -q -b other master~1 && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m fork ); fork=$(git -C "$d/src" rev-parse other) + out=$( cd "$d/src" && IGNEUM_MIRRORS="$d/mirror.git" mirror_master "$fork" ) + [ "$(git -C "$d/mirror.git" rev-parse master)" = "$tip" ] || { echo "self-test failed: a non-fast-forward push moved the mirror's master"; fails=1; } + case "$out" in *"did not take master"*) ;; *) echo "self-test failed: the refused mirror push was not reported: $out"; fails=1 ;; esac + # the remote decides the rule: origin (GitHub) binds the CI gate; a mirror remote does not + REMOTE=origin; remote_is_github || { echo "self-test failed: origin was not read as GitHub (is origin's URL github.com?)"; fails=1; } + ( cd "$d/src" && git remote add mirror "$d/mirror.git" ) 2>/dev/null; REMOTE=mirror; ( cd "$d/src" && remote_is_github ) && { echo "self-test failed: a bare-path mirror remote was read as GitHub"; fails=1; }; REMOTE=origin + rm -rf "$d" + [ "$fails" = 0 ] && echo "self-test passed: the CI rule binds a GitHub remote only; a landed master is fast-forwarded to every mirror and a refused mirror push is a line, not a failure; a green branch run merges; a red one is refused; an unknown read is waited through and refused only at the deadline; a queued run is waited for with the clock; an unrun branch is pushed once and then waited for; a red master refuses every merge but the declared fix" + exit $fails +fi [ -z "$(git status --porcelain --untracked-files=no)" ] || { echo "merge-to-master: the tree has uncommitted tracked changes; commit first" >&2; exit 1; } SHA=$(git rev-parse "$BRANCH"); G=$(cd "$(git rev-parse --git-common-dir)" && pwd -P) if [ ! -f "$G/igneum-gate-green/$SHA" ]; then - echo "merge-to-master: no green stamp for ${SHA:0:8}; running the full gate on the branch first" + echo "merge-to-master: no green stamp for ${SHA:0:8}; running the full gate on the branch first (then CI's own verdict on it is read)" bash tools/ci/pre-push.sh || exit 1 [ -f "$G/igneum-gate-green/$SHA" ] || { echo "merge-to-master: the gate was green but no stamp was written (dirty tree?)" >&2; exit 1; } fi AUTHOR=(-c user.name=igneum-josh -c user.email=337424239+igneum-josh@users.noreply.github.com) +# the CI rule: the branch's own run on this exact commit must be green (pushed for a run when there is none, waited for when queued), and master must not be red +if remote_is_github; then + ci_gate "$SHA" "$BRANCH" git push -q "$REMOTE" "$SHA:refs/heads/$BRANCH" || exit 1 + master_gate || exit 1 + VERDICT="gate: green on ${SHA:0:8}, recorded by tools/ci/pre-push.sh; CI green on ${SHA:0:8}; the full gate runs in CI on this merge" +else + echo "merge-to-master: the remote $REMOTE is not GitHub (a mirror); the box gate stamp on ${SHA:0:8} is the verdict${IGNEUM_MASTER_EXCEPTION:+ (exception: $IGNEUM_MASTER_EXCEPTION)}" + VERDICT="gate: green on ${SHA:0:8}, recorded by tools/ci/pre-push.sh; landed on the $REMOTE mirror${IGNEUM_MASTER_EXCEPTION:+ under the exception declared by main: $IGNEUM_MASTER_EXCEPTION}" +fi for i in $(seq 1 "$TRIES"); do - git fetch -q "$R" master; TIP=$(git rev-parse "$R/master") - if git merge-base --is-ancestor "$SHA" "$TIP"; then echo "merge-to-master: ${SHA:0:8} is already on $R/master $(git log -1 --format=%h "$R/master")"; exit 0; fi + git fetch -q "$REMOTE" master; TIP=$(git rev-parse "$REMOTE/master") + if git merge-base --is-ancestor "$SHA" "$TIP"; then echo "merge-to-master: ${SHA:0:8} is already on $REMOTE/master $(git log -1 --format=%h "$REMOTE/master")"; exit 0; fi W=$(mktemp -d "${TMPDIR:-/tmp}/merge-to-master.XXXXXX"); rmdir "$W" git worktree add -q --detach "$W" "$TIP" - if ( cd "$W" && git "${AUTHOR[@]}" merge -q --no-ff -m "Merge $BRANCH ${SHA:0:8} into master (gate: green on ${SHA:0:8}, recorded by tools/ci/pre-push.sh; the full gate runs in CI on this merge)" "$SHA" ); then - if ( cd "$W" && git push -q "$R" HEAD:master ); then - git worktree remove --force "$W"; git fetch -q "$R" master - echo "merge-to-master: pushed on try $i: $R/master $(git log -1 --format='%h %ci' "$R/master") $(TZ=Europe/London date '+%H:%M %Z')"; exit 0 + if ( cd "$W" && git "${AUTHOR[@]}" merge -q --no-ff -m "Merge $BRANCH ${SHA:0:8} into master ($VERDICT)" "$SHA" ); then + if ( cd "$W" && git push -q "$REMOTE" HEAD:master ); then # on a GitHub remote the hook asks ci-state about ${SHA:0:8} once more + git worktree remove --force "$W"; git fetch -q "$REMOTE" master + echo "merge-to-master: pushed on try $i: $REMOTE/master $(git log -1 --format='%h %ci' "$REMOTE/master") $(TZ=Europe/London date '+%H:%M %Z')" + remote_is_github && mirror_master "$(git rev-parse "$REMOTE/master")"; exit 0 fi echo "merge-to-master: try $i: the push was rejected (master moved or the hook was red); again" else - echo "merge-to-master: the merge of $BRANCH onto ${TIP:0:8} does not apply cleanly; resolve on the branch (git merge $R/master) and retry" >&2 + echo "merge-to-master: the merge of $BRANCH onto ${TIP:0:8} does not apply cleanly; resolve on the branch (git merge origin/master) and retry" >&2 git worktree remove --force "$W"; exit 1 fi git worktree remove --force "$W" 2>/dev/null || true diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh index 270432454..96bcdbbe2 100755 --- a/tools/ci/pre-push.sh +++ b/tools/ci/pre-push.sh @@ -26,6 +26,7 @@ cd "$(git rev-parse --show-toplevel)" || exit 1 # and fire this hook again inside the fixture: the first master push through the gate died that way (6 October 2026). unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_PREFIX GIT_COMMON_DIR GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_QUARANTINE_PATH GIT_PUSH_OPTION_COUNT MODE="${1:-local}"; MODE="${MODE#--}" +GATE_ROOT="$(pwd -P)" # the readers below are called from fixture repositories in the self-test, so by absolute path RED=0; N=0; LOG="$(mktemp)"; trap 'rm -rf "$LOG" "${SITE_TMP:-}"' EXIT T0=$(date +%s) @@ -52,12 +53,19 @@ site_build() { (cd "$SITE_TMP/site" && SITE_DOWNLOADS_OFFLINE=1 node build.mjs) } +wall_clock() { # : under GNU timeout where it exists (the runners); the Mac has none, and the job's timeout-minutes is the stop there. + # No array here: an empty array expanded under set -u is "unbound variable" on the Mac's bash 3.2 and ended the gate with no RED line (17:3x UK, 7 October 2026). + local secs="$1"; shift + if command -v timeout >/dev/null 2>&1; then timeout "$secs" "$@"; else "$@"; fi +} overlap_sweep() { # tools/ci/overlap-check.mjs: the known-failed fixture first, then the built site (the gate's temporary copy locally, the tree in # CI). A browser is needed: CI installs Playwright in the workflow; a machine without one ships the pages to a build box # (nothing heavy on the Mac). IGNEUM_OVERLAP_APPS=1 adds the miner and wallet UIs through their mocks (slower, the box). local dir="site"; [ "$MODE" = ci ] || dir="$SITE_TMP/site" - if [ "${IGNEUM_OVERLAP_APPS:-0}" = 1 ]; then node tools/ci/overlap-check.mjs --self-test --site "$dir" --apps .; else node tools/ci/overlap-check.mjs --self-test --site "$dir"; fi + # a wall clock of 10 minutes where GNU timeout exists (the runners; the Mac ships the sweep to a box): the sweep took 192 s on a hosted + # runner on 7 October 2026 and three master jobs hung in this step for over two hours each the same afternoon + if [ "${IGNEUM_OVERLAP_APPS:-0}" = 1 ]; then wall_clock 600 node tools/ci/overlap-check.mjs --self-test --site "$dir" --apps .; else wall_clock 600 node tools/ci/overlap-check.mjs --self-test --site "$dir"; fi } structural_checks() { @@ -104,7 +112,7 @@ tree_checks() { run "long-running tools keep their body in one parsed block (the edited-while-running class)" bash -c 'bash tools/ci/whole-body-check.sh --self-test && bash tools/ci/whole-body-check.sh' run "build-remote without a priority flag bounds suites and benches (nice 10, 32 cores); a gate runs unbounded" bash tools/ci/build-kind-default-check.sh run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh - run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh)" bash tools/ci/box-locks-check.sh $( [ "$MODE" = ci ] && echo --ci ) + run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh; skipped on a runner with no box; one retry)" bash tools/ci/retry-once.sh box-locks bash tools/ci/box-locks-check.sh $( [ "$MODE" = ci ] && echo --ci ) run "the simulators job runs on master and release-* pushes and pull requests into them only" bash tools/ci/sims-branch-check.sh run "publish-jobs.sh never removes a running or installs-app job without --force (the PC 2 abort class)" bash tools/ci/publish-jobs-check.sh run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh' @@ -118,7 +126,7 @@ tree_checks() { run "chain scene: the site's and the app's copies are scene/ byte for byte, the palette tokens live once (self-test, then the tree)" bash -c 'node tools/scene/sync.mjs --self-test && node tools/scene/sync.mjs --check' run "chain scene: a push paints with the document hidden and no animation frame (the blank /live of 7 Oct 2026; known-failed first)" node tools/scene/paint-test.cjs run "chain scene: the live feed contract (the recorded reply validates; a rewritten miner, a float now, a stray key refused)" node --test tools/scene/feed-contract.test.mjs tools/scene/legend.test.mjs tools/scene/shard-words.test.mjs - run "chain scene parity: one recorded feed through the home fold, /live and the app's Inspect view on build-2, three frames each pixel-equal apart from the app's own-key overlay (a changed token fails first; skipped with no box and no Playwright)" bash tools/scene/parity-remote.sh + run "chain scene parity: one recorded feed through the home fold, /live and the app's Inspect view on build-2, three frames each pixel-equal apart from the app's own-key overlay (a changed token fails first; skipped with no box and no Playwright; one retry)" bash tools/ci/retry-once.sh scene-parity bash tools/scene/parity-remote.sh run "no text overlaps: every served page at 390 to 1600 px, light and dark, the hero at each step (self-test first; IGNEUM_OVERLAP_APPS=1 adds the miner and wallet UIs)" overlap_sweep run "explorer, emission and public stats unit tests" node --test site/lib/explorer.test.mjs site/lib/emission.test.mjs site/lib/money.test.mjs site/lib/leaderboard.test.mjs site/api/public-stats.test.mjs run "ship tool self-test" node tools/ship-app.mjs --self-test @@ -129,6 +137,9 @@ tree_checks() { run "hash-origin report: a known-finished day and a known-failed day" node --test tools/observer/hash-origin.test.mjs run "harness summaries never carry a raw 64-hex key (the writer's own redaction and check)" node infra/fast-time/lib/redact-keys.mjs --self-test run "docs-only pushes skip the compile-or-compute CI jobs (the changes job's classifier)" bash tools/ci/docs-only-check.sh --self-test + run "every workflow job carries timeout-minutes (site 15, changes 10, pow 60, sims 45; the hung-job class of 7 October 2026)" bash tools/ci/workflow-timeouts-check.sh --self-test + run "a box or network check gets one retry before it is red (retry-once self-test)" bash tools/ci/retry-once.sh --self-test + run "CI state reader: a commit's newest run, master's last compiled run, a branch's last red (fake gh; the merge rule's reader)" node tools/ci/ci-state.mjs --self-test } gated_refs() { @@ -164,6 +175,33 @@ deferred_merge() { # [repo dir] -> "defer /dev/null || stat -c %Y "$f") )); [ "$age" -le 43200 ] || { echo "full the stamp for ${p2:0:8} is $age s old"; return; } echo "defer $p2" } +# Master takes only what CI has already passed (standing rule, 7 October 2026, 17:2x UK: era-vdf's tip 0e2d6b1c was merged with +# no ci run at all and master's igneum-pow suite stayed red for 40 minutes under docs-only merges). For a push to master the hook +# asks tools/ci/ci-state.mjs: a two-parent merge needs a green run on its SECOND parent (the branch's own run on the exact +# commit), a plain commit needs a green run on itself (a fast-forward of a branch CI passed); anything else is refused with the +# run's state, and the lane uses tools/ci/merge-to-master.sh, which waits for a queued run. gh unreachable = refused (unknown). +# release-* branches keep the full local gate alone (the shipper's cuts carry their own box suite line). +master_ci_ok() { # -> 0 and a line, or 1 and the reason + local lsha="$1" rsha="$2" parents p2 want line state + parents=$(git rev-list --parents -n 1 "$lsha" 2>/dev/null | cut -d' ' -f2-); set -- $parents + if [ -n "${2:-}" ] && [ -z "${3:-}" ] && [ "$1" = "$rsha" ]; then want="$2"; else want="$lsha"; fi + line=$(node "$GATE_ROOT/tools/ci/ci-state.mjs" "$want" 2>&1); state="${line%% *}" + if [ "$state" = success ]; then echo " master takes ${want:0:8}: ci $line"; return 0; fi + echo "pre-push gate: REFUSED. master takes only a commit whose own ci run is green on that exact commit; ${want:0:8} is: $line" >&2 + echo " Use tools/ci/merge-to-master.sh (it pushes the branch for a run when there is none, waits for a queued run and refuses a red)." >&2 + return 1 +} +master_rule_binds() { # : 0 when the CI rule applies to this push (a GitHub remote, no declared exception), 1 with a printed line otherwise + local url="${1:-}" + if [ -n "${IGNEUM_MASTER_EXCEPTION:-}" ]; then echo " EXCEPTION to the CI rule for this push, declared by main: $IGNEUM_MASTER_EXCEPTION (the local gate is the verdict)"; return 1; fi + case "$url" in *github.com*) return 0 ;; esac + echo " the remote ${url:-?} is not GitHub (a mirror): the CI rule binds GitHub's master; the local gate is the verdict here"; return 1 +} +branch_red_line() { # : the branch's newest completed ci run, when red, printed before the light gate (nothing on green or no gh) + local line; line=$(node "$GATE_ROOT/tools/ci/ci-state.mjs" --branch-red "$1" 2>/dev/null) || return 0 + case "$line" in previous\ CI\ red*) echo " $line" ;; esac + return 0 +} finish() { local what="$1" secs=$(( $(date +%s) - T0 )) if [ "$RED" = 0 ]; then echo "pre-push gate ($what): GREEN, $N checks in ${secs}s"; [ "${STAMP:-0}" = 1 ] && stamp_green; exit 0; fi @@ -207,7 +245,36 @@ case "$MODE" in MODE=ci GITHUB_ACTIONS= site_in_place && { echo "self-test failed: --ci outside GitHub Actions chose the in-place build"; fails=1; } MODE=ci GITHUB_ACTIONS=true site_in_place || { echo "self-test failed: --ci inside GitHub Actions did not choose the in-place build"; fails=1; } declare -f tree_checks | grep -q 'never_push_checks' || { echo "self-test failed: the full gate does not run the never-push checks"; fails=1; } - [ "$fails" = 0 ] && echo "self-test passed: a failing check is RED and fails the gate, a passing one is ok; master and release-* select the full gate, other refs the light one; a merge of a green-stamped branch onto the remote tip defers to CI, every other shape takes the full gate; a --ci site build outside GitHub Actions leaves the tree unchanged (structural checks, no-secrets, identity grep)" + # the overlap sweep's wall clock runs the command with GNU timeout where it exists and plainly where it does not (bash 3.2 under set -u included) + [ "$(wall_clock 5 /bin/echo clocked 2>&1)" = clocked ] || { echo "self-test failed: wall_clock did not run its command"; fails=1; } + [ "$(PATH=/nonexistent wall_clock 5 /bin/echo plain 2>&1)" = plain ] || { echo "self-test failed: wall_clock without a timeout binary did not run its command plainly"; fails=1; } + grep -q 'wall_clock 600 node tools/ci/overlap-check.mjs' "$0" || { echo "self-test failed: the overlap sweep does not run under the wall clock"; fails=1; } + # master takes only CI-passed commits: a merge asks about its second parent, a plain commit about itself; red, pending, none and unknown refuse + grep -qE 'master_ci_ok "\$lsha" "\$rsha" \|\| exit 1' "$0" || { echo "self-test failed: the hook does not ask ci-state before a push to master"; fails=1; } + grep -qE 'branch_red_line "\$\{rref#refs/heads/\}"' "$0" || { echo "self-test failed: the feature-branch hook does not print the branch's previous red"; fails=1; } + fx=$(mktemp -d); ( cd "$fx" && git init -q -b master . && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m a ) 2>/dev/null + A=$(git -C "$fx" rev-parse HEAD); git -C "$fx" checkout -q -b b; git -C "$fx" -c user.name=t -c user.email=t@t commit -q --allow-empty -m b; B=$(git -C "$fx" rev-parse HEAD) + git -C "$fx" checkout -q master; git -C "$fx" -c user.name=t -c user.email=t@t merge -q --no-ff -m "merge b" b; M=$(git -C "$fx" rev-parse HEAD) + fakebin=$(mktemp -d) + cat > "$fakebin/gh" </dev/null 2>&1 ) || { echo "self-test failed: a merge whose branch parent has a green run was refused"; fails=1; } + ( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$B" "$A" >/dev/null 2>&1 ) || { echo "self-test failed: a plain commit with its own green run was refused"; fails=1; } + ( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$M" "$B" >/dev/null 2>&1 ) && { echo "self-test failed: a merge onto another tip (its own run queued) was let through"; fails=1; } + ( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$A" "$A" >/dev/null 2>&1 ) && { echo "self-test failed: a commit with no ci run was let through to master"; fails=1; } + rm -rf "$fx" "$fakebin" + # the CI rule binds a GitHub remote; a mirror remote and a declared exception take the local gate, each with a printed line + master_rule_binds https://github.com/igneum-network/igneum.git >/dev/null || { echo "self-test failed: the CI rule did not bind a GitHub remote"; fails=1; } + master_rule_binds build@188.40.146.49:/srv/igneum.git >/dev/null && { echo "self-test failed: the CI rule bound a box mirror remote"; fails=1; } + ( IGNEUM_MASTER_EXCEPTION="main, 7 Oct 2026 19:5x UK: GitHub suspended" master_rule_binds https://github.com/x/y.git >/dev/null ) && { echo "self-test failed: a declared exception did not lift the CI rule"; fails=1; } + out=$(IGNEUM_MASTER_EXCEPTION="ruling text" master_rule_binds https://github.com/x/y.git); case "$out" in *"EXCEPTION"*"ruling text"*) ;; *) echo "self-test failed: the exception was not printed with its ruling: $out"; fails=1 ;; esac + [ "$fails" = 0 ] && echo "self-test passed: a failing check is RED and fails the gate, a passing one is ok; master and release-* select the full gate, other refs the light one; a merge of a green-stamped branch onto the remote tip defers to CI, every other shape takes the full gate; master takes only a commit (or a merge's branch parent) whose own ci run is green, and refuses red, queued and unrun ones (GitHub remotes; a mirror remote or a declared exception takes the local gate, printed); a feature-branch push prints the branch's previous red first; a --ci site build outside GitHub Actions leaves the tree unchanged (structural checks, no-secrets, identity grep)" exit $fails ;; list) grep -E '^\s+run "' "$0" | sed -E 's/^\s+run "([^"]+)".*/\1/' ;; @@ -216,6 +283,16 @@ case "$MODE" in if [ "$which" = full ]; then # a merge of a green-stamped branch onto the exact remote tip goes through on the light gate (CI runs the full one) verdict=""; while read -r lref lsha rref rsha; do case "$rref" in refs/heads/master|refs/heads/release-*) verdict=$(deferred_merge "$lsha" "$rsha"); break ;; esac; done <<<"$REFS" + # a push to master: the pushed commit (or its branch parent) must already have a green ci run on that exact commit. CI runs on + # GitHub, so the rule binds a push whose remote is github.com; a push of master to a box mirror (build@:/srv/igneum.git) + # takes the local gate as before. IGNEUM_MASTER_EXCEPTION="" lifts the CI rule for one push and is printed with + # the push (7 October 2026, 19:5x UK: the GitHub account suspended, lanes landing on the box mirror's master by main's ruling, + # the box gate stamp as the verdict; GitHub gets the fast-forward when it answers again). + if master_rule_binds "${2:-}"; then + while read -r lref lsha rref rsha; do + if [ "$rref" = refs/heads/master ] && [ "$lsha" != 0000000000000000000000000000000000000000 ]; then master_ci_ok "$lsha" "$rsha" || exit 1; fi + done <<<"$REFS" + fi case "$verdict" in defer*) echo "pre-push gate: a merge of green-stamped ${verdict#defer } onto the remote tip: the light gate here, the full gate in CI on landing:" structural_checks; never_push_checks; finish "merge of a green branch (full gate deferred to CI)" ;; @@ -224,6 +301,7 @@ case "$MODE" in esac else echo "pre-push gate: a feature branch, the light gate (the two structural checks, the no-secrets check, the identity grep):" + while read -r lref lsha rref rsha; do case "$rref" in refs/heads/*) branch_red_line "${rref#refs/heads/}" ;; esac; done <<<"$REFS" structural_checks; never_push_checks; finish "feature branch" fi ;; ci|local) diff --git a/tools/ci/red-watch.mjs b/tools/ci/red-watch.mjs index 0269533d5..4d3d83e1f 100755 --- a/tools/ci/red-watch.mjs +++ b/tools/ci/red-watch.mjs @@ -78,6 +78,7 @@ export function runFromEnv(env = process.env) { if (!sha) throw new Error('record: neither RED_WATCH_SHA nor GITHUB_SHA is set'); return { run_id: id, attempt: Number(pick('RED_WATCH_ATTEMPT', 'GITHUB_RUN_ATTEMPT') || 1), workflow: pick('RED_WATCH_WORKFLOW', 'GITHUB_WORKFLOW'), + conclusion: env.RED_WATCH_CONCLUSION || 'failure', // failure, cancelled or timed_out (ci-red.yml fires on all three since 7 October 2026) branch: pick('RED_WATCH_BRANCH', 'GITHUB_REF_NAME'), sha: sha.slice(0, 7), event: pick('RED_WATCH_EVENT', 'GITHUB_EVENT_NAME'), actor: pick('RED_WATCH_ACTOR', 'GITHUB_ACTOR'), author: env.RED_WATCH_AUTHOR || '', // who pushed (the GitHub login), who the head commit names url: env.RED_WATCH_URL || `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${id}`, at: new Date().toISOString(), @@ -124,11 +125,15 @@ export async function record(file, env = process.env, fetchImpl = fetch, title = return { written: true, run: line }; } +// The kind of line: a failed run is "CI red"; a cancelled run "CI cancelled" (a hand on the run, or a job past its timeout-minutes +// under GitHub's older runner, which reports cancelled); a timed-out run "CI timed out". All three are read like a red. +export const KINDS = { failure: 'CI red', cancelled: 'CI cancelled', timed_out: 'CI timed out' }; export function formatLine(l) { const where = l.failed.length ? l.failed.map((f) => `${f.job.replace(/,.*$/, '')} at "${f.step}"`).join('; ') : (l.note || 'no step detail'); const title = l.title ? ` "${l.title}"` : ''; const who = l.actor ? ` pushed by ${l.actor}${l.author && l.author !== l.actor ? ` (commit by ${l.author})` : ''};` : ''; - return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}:${who} ${where} ${l.url}`; + const kind = KINDS[l.conclusion || 'failure'] || `CI ${l.conclusion}`; + return `${kind}: ${l.workflow} on ${l.branch} @${l.sha}${title}:${who} ${where} ${l.url}`; } function readCredentials(file) { @@ -259,6 +264,22 @@ async function selfTest() { if (!asked[0] || !asked[0].includes('/actions/runs/37620364667/jobs')) fails.push(`workflow_run shape: the jobs API was asked for ${asked[0]}, not the failed run`); const textRun = formatLine(lr); if (!/^CI red: ci on ca3-v4-node @26a4b0f "Counter ASIC 3.0 node plan 6.7": pushed by igneum-labs \(commit by igneum-josh\); site build at "identity grep of the public export list"; simulators at "\(job never started: runner or billing\)" https:\/\/github.com\/igneum-network\/igneum\/actions\/runs\/37620364667$/.test(textRun)) fails.push(`workflow_run line: ${textRun}`); + // a cancelled run and a timed-out run are recorded with their kind in the line (a hung job past its timeout-minutes, a hand on the run) + const cancelledJobs = { jobs: [{ name: 'site build, link check, identity grep', conclusion: 'cancelled', steps: [{ name: 'site build', conclusion: 'success' }, { name: 'the tree gate, tools/ci/pre-push.sh --ci', conclusion: 'cancelled' }] }] }; + const fileKinds = path.join(dir, 'kinds.jsonl'); + await record(fileKinds, { ...envRun, RED_WATCH_RUN_ID: '555', RED_WATCH_CONCLUSION: 'cancelled' }, async () => ({ ok: true, status: 200, json: async () => cancelledJobs })); + await record(fileKinds, { ...envRun, RED_WATCH_RUN_ID: '556', RED_WATCH_CONCLUSION: 'timed_out' }, async () => ({ ok: true, status: 200, json: async () => cancelledJobs })); + const [lc, lt] = readLines(fileKinds).map(formatLine); + if (!/^CI cancelled: ci on ca3-v4-node @26a4b0f .*site build at "the tree gate, tools\/ci\/pre-push.sh --ci"/.test(lc)) fails.push(`cancelled line: ${lc}`); + if (!/^CI timed out: ci on ca3-v4-node @26a4b0f /.test(lt)) fails.push(`timed-out line: ${lt}`); + if (readLines(fileKinds)[0].conclusion !== 'cancelled') fails.push('record: the conclusion was not kept in the line'); + // the watcher workflow fires on all three conclusions and hands the conclusion to the record step + const ymlPath = path.join(path.dirname(new URL(import.meta.url).pathname), '..', '..', '.github', 'workflows', 'ci-red.yml'); + if (fs.existsSync(ymlPath)) { + const yml = fs.readFileSync(ymlPath, 'utf8'); + for (const c of ['failure', 'cancelled', 'timed_out']) if (!yml.includes(`github.event.workflow_run.conclusion == '${c}'`)) fails.push(`ci-red.yml: the job's if does not fire on ${c}`); + if (!yml.includes('RED_WATCH_CONCLUSION: ${{ github.event.workflow_run.conclusion }}')) fails.push('ci-red.yml: RED_WATCH_CONCLUSION is not handed to the record step'); + } // post, dry run: prints, sends nothing, marks nothing let printed = []; const log = (s) => printed.push(s); const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; }; @@ -307,7 +328,7 @@ async function selfTest() { if (!d3.sent) fails.push('digest: not sent the next day'); fs.rmSync(dir, { recursive: true, force: true }); if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); } - console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author, the failed run and not the watcher\'s own under workflow_run; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00'); + console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author, the failed run and not the watcher\'s own under workflow_run; a cancelled and a timed-out run are recorded with their kind and ci-red.yml fires on all three; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00'); } const cmd = args[0]; diff --git a/tools/ci/retry-once.sh b/tools/ci/retry-once.sh new file mode 100755 index 000000000..411d58014 --- /dev/null +++ b/tools/ci/retry-once.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Run a check that can red on box state or the network twice before calling it red (standing rule, 7 October 2026, 17:2x UK: +# a check that talks to a build box or a live host gets one retry and, on a runner without the resource, a clear skip line +# from the check itself). The first failure is printed as a line, the command runs again after a pause, and only the second +# failure is the check's verdict. A check that passes first time runs once. +# +# tools/ci/retry-once.sh # exit = the last run's exit code; RETRY_ONCE_PAUSE seconds between (default 5) +# tools/ci/retry-once.sh --self-test # a command that fails once then passes is ok; one that fails twice is red; +# # a command that passes runs exactly once +set -uo pipefail +PAUSE="${RETRY_ONCE_PAUSE:-5}" +if [ "${1:-}" = --self-test ]; then + fails=0; d=$(mktemp -d) + # fails once then passes: the marker file is the memory + flaky="$d/flaky.sh"; printf '#!/usr/bin/env bash\nif [ -f "%s/seen" ]; then exit 0; fi; touch "%s/seen"; echo first-try-failed; exit 7\n' "$d" "$d" > "$flaky"; chmod +x "$flaky" + out=$(RETRY_ONCE_PAUSE=0 bash "$0" flaky "$flaky" 2>&1); rc=$? + [ "$rc" = 0 ] || { echo "self-test failed: a command that fails once then passes was red (exit $rc): $out"; fails=1; } + case "$out" in *"retry-once: flaky failed once (exit 7)"*) ;; *) echo "self-test failed: the first failure was not printed: $out"; fails=1 ;; esac + # fails twice: red with the second exit code + out=$(RETRY_ONCE_PAUSE=0 bash "$0" dead bash -c 'echo nope; exit 3' 2>&1); rc=$? + [ "$rc" = 3 ] || { echo "self-test failed: a command that fails twice was not red with its exit code (got $rc): $out"; fails=1; } + [ "$(printf '%s\n' "$out" | grep -c '^nope$')" = 2 ] || { echo "self-test failed: the command did not run exactly twice: $out"; fails=1; } + # passes: runs once + c="$d/count"; : > "$c" + RETRY_ONCE_PAUSE=0 bash "$0" fine bash -c "echo x >> '$c'" >/dev/null 2>&1 || { echo "self-test failed: a passing command was red"; fails=1; } + [ "$(wc -l < "$c" | tr -d ' ')" = 1 ] || { echo "self-test failed: a passing command ran $(wc -l < "$c") times"; fails=1; } + rm -rf "$d" + [ "$fails" = 0 ] && echo "self-test passed: one retry after a first failure, red only on the second, a passing command runs once" + exit $fails +fi +[ $# -ge 2 ] || { echo "usage: tools/ci/retry-once.sh | --self-test" >&2; exit 2; } +NAME="$1"; shift +"$@"; rc=$? +[ "$rc" = 0 ] && exit 0 +echo "retry-once: $NAME failed once (exit $rc); second try in ${PAUSE}s (a box or network check gets one retry before it is red)" +sleep "$PAUSE" +"$@" diff --git a/tools/ci/workflow-timeouts-check.sh b/tools/ci/workflow-timeouts-check.sh new file mode 100755 index 000000000..b4ab05404 --- /dev/null +++ b/tools/ci/workflow-timeouts-check.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Every job in .github/workflows carries timeout-minutes, and the named jobs carry their budgets (7 October 2026, 15:13 to +# 17:20 UK: three hosted `site` jobs on master hung in the tree gate for over two hours each, with GitHub's six-hour default +# as the only stop; a fourth would have been a failure email at 21:13 UK). The budgets, from the measured times on +# 7 October: the tree gate on a hosted runner 229 s plus a 40 s Playwright install, so `site` 15; the classifier `changes` +# a 7 s API call, so 10; the box's igneum-pow suite 45 s to 2 min 40 s, so `pow` 60; the two simulators 2 x 120 s, so `sims` 45. +# +# tools/ci/workflow-timeouts-check.sh # exit 1 naming each job without a timeout or with the wrong budget +# tools/ci/workflow-timeouts-check.sh --self-test # a fixture job without the key fails; a wrong budget fails; the tree passes +set -euo pipefail +cd "$(dirname "$0")/../.." + +# -> lines "job timeout" for every job (timeout "-" when missing). A job is a key at four spaces under `jobs:`; +# its timeout-minutes is the key at six spaces before the next job. +jobs_of() { + awk ' + /^jobs:/ { injobs = 1; next } + injobs && /^[^ ]/ { injobs = 0 } + injobs && /^ [A-Za-z0-9_-]+:/ { if (job != "") print job, (t == "" ? "-" : t); job = $1; sub(":", "", job); t = ""; next } + injobs && job != "" && /^ timeout-minutes:/ { t = $2 } + END { if (job != "") print job, (t == "" ? "-" : t) } + ' "$1" +} +check_tree() { # -> exit 1 with one line per wrong job + local f rc=0 job t want + for f in "$1"/*.yml "$1"/*.yaml; do + [ -f "$f" ] || continue + while read -r job t; do + if [ "$t" = "-" ]; then echo "workflow-timeouts: $(basename "$f") job \`$job\` has no timeout-minutes" >&2; rc=1; continue; fi + want="" + case "$(basename "$f"):$job" in + ci.yml:site) want=15 ;; ci.yml:changes) want=10 ;; ci.yml:pow) want=60 ;; ci.yml:sims) want=45 ;; + esac + if [ -n "$want" ] && [ "$t" != "$want" ]; then echo "workflow-timeouts: $(basename "$f") job \`$job\` has timeout-minutes $t, the budget is $want" >&2; rc=1; fi + done < <(jobs_of "$f") + done + return $rc +} +if [ "${1:-}" = --self-test ]; then + fails=0; d=$(mktemp -d); mkdir -p "$d/bad" "$d/budget" + printf 'name: x\non: push\njobs:\n a:\n runs-on: ubuntu-latest\n timeout-minutes: 5\n steps: []\n b:\n runs-on: ubuntu-latest\n steps: []\n' > "$d/bad/ci.yml" + out=$(check_tree "$d/bad" 2>&1) && { echo "self-test failed: a job without timeout-minutes passed"; fails=1; } + case "$out" in *'job `b` has no timeout-minutes'*) ;; *) echo "self-test failed: the job without a timeout was not named: $out"; fails=1 ;; esac + case "$out" in *'job `a`'*) echo "self-test failed: a job with a timeout was named: $out"; fails=1 ;; esac + printf 'name: ci\non: push\njobs:\n site:\n runs-on: ubuntu-latest\n timeout-minutes: 360\n steps: []\n' > "$d/budget/ci.yml" + out=$(check_tree "$d/budget" 2>&1) && { echo "self-test failed: site at 360 minutes passed"; fails=1; } + case "$out" in *'has timeout-minutes 360, the budget is 15'*) ;; *) echo "self-test failed: the wrong budget was not named: $out"; fails=1 ;; esac + rm -rf "$d" + check_tree .github/workflows || { echo "self-test failed: the tree's workflows do not pass"; fails=1; } + [ "$fails" = 0 ] && echo "self-test passed: a job without timeout-minutes fails, a wrong budget fails, the tree's workflows pass (site 15, changes 10, pow 60, sims 45)" + exit $fails +fi +check_tree .github/workflows && echo "workflow-timeouts: every job in .github/workflows carries timeout-minutes (site 15, changes 10, pow 60, sims 45)"