igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity

Merge remote-tracking branch 'build/master' into ca3-coord

Browse source
This commit is contained in:
igneum-josh 2026-10-07 20:15:19 +01:00
commit 4b1afd4e4c
11 changed files with 521 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.github/workflows/ci-red.yml vendored
View file

@ -3,7 +3,7 @@
# ci.yml never enters it (7 October 2026: the inline `red` job of ci.yml was conditioned on master and release-*, and
# a feature branch would have waited for a merge of master before its reds were posted at all).
#
# One line per failed run (tools/ci/red-watch.mjs record, idempotent per run attempt) to /srv/ci-red/red.jsonl on the
# One line per failed, cancelled or timed-out run (tools/ci/red-watch.mjs record, idempotent per run attempt) to /srv/ci-red/red.jsonl on the
# box; the box's igneum-ci-red.timer posts each new line once to the hidden updates channel, naming the branch, the
# commit, the red check and the pushing author. Runs on the box's own runner (not a GitHub-hosted machine: the billing
# block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told). Never blocks a
@ -16,7 +16,9 @@ on:
jobs:
red:
name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file)
if: ${{ github.event.workflow_run.conclusion == 'failure' }}
# failure, and since 7 October 2026 (17:2x UK) cancelled and timed_out too: a job that hangs into its timeout-minutes or a run
# someone cancels is a run that never answered, and a lane reads it like a red (tools/ci/red-watch.mjs names the kind)
if: ${{ github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'cancelled' || github.event.workflow_run.conclusion == 'timed_out' }}
# the label ci-red is on igneum-build-1 only (added through the runners API on 7 October 2026; the default of
# RUNNER_LABELS in provision.sh carries it): the record file and the poster (igneum-ci-red.timer, the webhook file)
# live on that box, and the pool label igneum-build-1 is shared with igneum-build-2 since the same day
@ -35,6 +37,7 @@ jobs:
RED_WATCH_RUN_ID: ${{ github.event.workflow_run.id }}
RED_WATCH_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
RED_WATCH_WORKFLOW: ${{ github.event.workflow_run.name }}
RED_WATCH_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
RED_WATCH_BRANCH: ${{ github.event.workflow_run.head_branch }}
RED_WATCH_SHA: ${{ github.event.workflow_run.head_sha }}
RED_WATCH_EVENT: ${{ github.event.workflow_run.event }}

9
.github/workflows/ci.yml vendored
View file

@ -36,6 +36,7 @@ jobs:
# code=true (no `before` to compare from), as does any error reading the compare API: when in doubt, run.
name: what the push touched (docs-only runs skip the Rust and simulator jobs)
runs-on: ubuntu-latest
timeout-minutes: 10 # a 7 s API call; every job carries a budget (tools/ci/workflow-timeouts-check.sh)
outputs:
code: ${{ steps.classify.outputs.code }}
steps:
@ -62,6 +63,7 @@ jobs:
needs: changes
if: ${{ needs.changes.outputs.code == 'true' }}
runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
timeout-minutes: 60 # the box's suite ran 45 s to 2 min 40 s on 7 October 2026; a hosted fallback compiles cold
steps:
- uses: actions/checkout@v4
- name: toolchain
@ -81,6 +83,7 @@ jobs:
# queue read 22); a feature-branch code push runs the igneum-pow tests alone. tools/ci/sims-branch-check.sh holds this rule.
if: ${{ needs.changes.outputs.code == 'true' && ((github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))) || (github.event_name == 'pull_request' && (github.base_ref == 'master' || startsWith(github.base_ref, 'release-')))) }}
runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
timeout-minutes: 45 # two simulators under 120 s each by their own timeout, plus a hosted fallback's pip install
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
@ -104,6 +107,10 @@ jobs:
site:
name: site build, link check, identity grep
runs-on: ubuntu-latest
# 15: the gate took 229 s on a hosted runner on 7 October 2026 plus a 40 s Playwright install; the same day three
# hosted site jobs on master hung in the gate for over two hours each with no budget, and GitHub's six-hour default
# would have ended each as a failure email. A hung job is a red the watcher posts (ci-red.yml fires on timed_out).
timeout-minutes: 15
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
@ -118,4 +125,4 @@ jobs:
run: bash tools/ci/pre-push.sh --ci
- name: public stats API answers with the documented fields (the live site; master only, the endpoints exist there after the merge)
if: github.ref == 'refs/heads/master'
run: node tools/ci/public-api-check.mjs https://igneum.network
run: bash tools/ci/retry-once.sh public-api node tools/ci/public-api-check.mjs https://igneum.network # a live host: one retry before red

8
.github/workflows/windows.yml vendored
View file

@ -216,8 +216,14 @@ jobs:
run: |
$payload = Resolve-Path 'packaging\windows\igneum-windows-app'
function Run-Capture([string]$exe, [string]$flag) {
# Start-Process -Wait on an exe that exits in milliseconds can throw "Cannot process request because the process has
# exited" before it attaches (release-0.3.21 run 37653903394, 7 October 2026, 17:40 UK): one retry before the verdict.
$out = Join-Path $env:RUNNER_TEMP ('smoke-' + [IO.Path]::GetRandomFileName() + '.txt')
$p = Start-Process -FilePath $exe -ArgumentList $flag -Wait -NoNewWindow -PassThru -RedirectStandardOutput $out
$p = $null
foreach ($try in 1, 2) {
try { $p = Start-Process -FilePath $exe -ArgumentList $flag -Wait -NoNewWindow -PassThru -RedirectStandardOutput $out; break }
catch { if ($try -eq 2) { throw }; Write-Host ("Start-Process on {0} {1} failed once ({2}); second try" -f (Split-Path -Leaf $exe), $flag, $_.Exception.Message); Start-Sleep -Milliseconds 500 }
}
$text = if (Test-Path $out) { (Get-Content $out -Raw) } else { '' }
Write-Host ("{0} {1} -> exit {2}: {3}" -f (Split-Path -Leaf $exe), $flag, $p.ExitCode, $text.Trim())
if ($p.ExitCode -ne 0) { throw "$exe $flag exited $($p.ExitCode)" }

1
CLAUDE.md
View file

@ -132,6 +132,7 @@ The Mac crashed and rebooted under agent load (about twenty lanes, local cargo t
- The pre-push gate is the same script CI runs: `tools/ci/pre-push.sh` (installed by `tools/ci/install-hooks.sh`; `--hook` before a push to master or release-*, `--ci` in the workflow). A check is added there, never only in ci.yml. On a feature branch the hook runs the light gate: the two structural checks plus the two never-push classes, the no-secrets check and the identity grep (7 October 2026).
- After every push, the pushing lane reads the conclusion when it lands (`gh run list --branch <branch> --limit 1`; `gh run view <id> --log-failed` on a red) and owns a red before the next push; pushes are never held for it. A red on any branch is reported to main the moment it is seen. The red watcher (.github/workflows/ci-red.yml, from master, every branch) posts each failed run with its branch, commit, red check and pushing author. Record: 7 October 2026, 11:26 to 12:47 UK, eight red runs on ca3-v4-node over three gate summaries carrying a 64-hex key, read by nobody; 31 runs queued on one runner at 13:15 UK, so igneum-build-2 joined the pool and docs-only pushes skip the compile jobs.
- A code push (anything outside docs/, site/ and *.md) goes out only after that exact commit's crate suite is green on a build box (the suite's RESULT line in the push's own notes or the commit message); a test build the box never ran is not pushed. Record: 7 October 2026, 14:10 to 14:13 UK, four red runs in three minutes (a test-build compile error on ca3-v4-amend, the hidden-console check on miner-reliability, the installer's inputs pin on release-0.3.20), each a push ahead of its own box result, each emailed to Josh.
- Master takes only what CI has already passed (7 October 2026, 17:2x UK): a merge lands only when the branch's own `ci` run is green on the exact commit being merged, read from the runs API (`tools/ci/ci-state.mjs`), never the local stamp alone; `tools/ci/merge-to-master.sh` pushes the branch for a run when there is none, waits for a queued one printing the clock, refuses a red one and refuses any merge onto a red master except the declared fix (`--fixes-master`); the pre-push hook refuses a push to master whose commit (or merge parent) has no green run; a feature-branch push prints the branch's previous red first. GitHub's branch protection is unavailable on the free plan for a private repository (the API answers 403), so the two scripts are the enforcement.
- Research and operations documents live outside the public export list: name them in `tools/ci/export-exclude.txt` (read by the identity check and by the mirror's sync.sh). What stays in the list is read by the public and must pass the identity grep.
- A path Windows cannot hold (colon, trailing dot or space, reserved name, over 240 characters) never enters a commit: the pre-commit hook runs `tools/ci/windows-paths-check.sh --staged`.
- Record: docs/analysis/ci-failures-2026-10-06.md (168 non-green runs in three days classified; 126 on master; all but two classes were tree checks that the gate now runs locally first).

5
tools/ci/README.md
View file

@ -4,4 +4,9 @@
|---|---|---|
| no text overlaps (`overlap-check.mjs`) | A served page, or a miner or wallet screen (behind `IGNEUM_OVERLAP_APPS=1`), where a visible run of text is covered by another element (a pill over a caption, a label over a value, a card over its neighbour, text under the header at rest), clipped by an overflow-hidden ancestor, or past the viewport; a page that scrolls sideways. Five widths, light and dark, the home hero at rest and at each step. A fixture with one deliberate overlap of each kind must be flagged first (`--self-test`). Needs a headless Chromium: CI installs Playwright; the Mac ships the pages to build-2 (`infra/build-server/overlap-browser.sh`). | 7 October 2026: the hero's step pill sat on the caption's second line ("Two thirds of the weight sign. The checkpoint locks.") at every desktop width, found by Josh on the live site |
| master takes only CI-passed commits (`ci-state.mjs`, `merge-to-master.sh`, the hook's `master_ci_ok`) | A push to master whose commit, or whose merge's branch parent, has no green `ci` run on that exact sha (the runs API through gh: red, queued, none or gh unreachable all refuse); a merge onto a master whose last compiled run is red, unless declared the fix (`--fixes-master`). The merge tool pushes an unrun branch for a run and waits for a queued one with the clock. A feature-branch push prints the branch's previous red first (`--branch-red`). | 7 October 2026: era-vdf's tip 0e2d6b1c merged with no ci run; master's igneum-pow suite red from 16:31 UK under five docs-only green merges |
| every workflow job carries timeout-minutes (`workflow-timeouts-check.sh`) | A job in .github/workflows without `timeout-minutes`, or a budget off its measured line (site 15, changes 10, pow 60, sims 45). | 7 October 2026: three hosted site jobs on master hung over two hours each in the tree gate; the six-hour default was the only stop |
| a box or network check gets one retry (`retry-once.sh`) | Nothing by itself: wraps the box-locks check, the scene parity check and the live public API check so a first failure is printed and retried once; the second is the verdict. The checks keep their own skip line on a runner without the resource. | 7 October 2026 |
| the red watcher fires on cancelled and timed-out runs too (`ci-red.yml`, `red-watch.mjs`) | The watcher's `if` missing any of failure, cancelled, timed_out, or the conclusion not handed to the record step (the self-test reads the workflow file); the line names the kind: CI red, CI cancelled, CI timed out. | 7 October 2026 |
| kill by exact command or pid file (owed as a check) | 6 October 2026, 21:09Z: a Mac-side `pkill -f <log file name>` matched nothing (the log name was a redirect, not part of the command line), the roll-everything script lived on and wiped a box it had been told to hold. Rule: a job is stopped by its pid file (`tools/fleet/fleet-bg.sh start|stop <name>`) or by a pattern anchored on its exact command line (`^python3 -u /root/fleet/in/box-prover.py`), never by a word that may or may not appear in it. The check that flags a `pkill -f`/`pgrep -f` whose literal is a path or a name that never starts a command line is owed to the CI lane |

152
tools/ci/ci-state.mjs Executable file
View file

@ -0,0 +1,152 @@
#!/usr/bin/env node
// What CI says about a commit or a branch, read from the runs API through gh (the Mac's gh login; the repository is
// igneum-network/igneum unless IGNEUM_REPO says otherwise). The merge tool and the pre-push hook read these lines, so a
// merge lands on master only when the branch's own ci run is green on the exact commit being merged (standing rule,
// 7 October 2026, 17:2x UK: era-vdf's tip 0e2d6b1c was merged with no ci run at all and master's igneum-pow suite went
// red for 40 minutes, hidden by docs-only merges whose runs skip the compile job).
// Node 22, standard library only; gh does the HTTP.
//
// node tools/ci/ci-state.mjs <sha> one line: "<state> <run id> <url> <detail>"
// state: success | failure | cancelled | timed_out | pending | none | unknown
// (the NEWEST ci run on that exact commit; a re-run replaces the first attempt)
// node tools/ci/ci-state.mjs --master-code the verdict of master's newest completed ci run whose compile job (igneum-pow)
// RAN: a docs-only push skips it and its green hides a red suite
// node tools/ci/ci-state.mjs --branch-red <branch> prints "previous CI red on <branch>: ..." when the branch's newest completed ci
// run is red and no newer run is green; prints nothing otherwise; exit 0 always
// node tools/ci/ci-state.mjs --self-test a fake gh on PATH answers every case: success, failure, pending, none, newest
// wins, the docs-only skip walked past, the branch line, a gh error = unknown
//
// Exit 0 for every answered query (callers read the first word); 2 on usage; the self-test exits 1 on a failure.
import { spawnSync } from 'node:child_process';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
const REPO = process.env.IGNEUM_REPO || 'igneum-network/igneum';
const FIELDS = 'databaseId,status,conclusion,headSha,url,createdAt,event';
const RED = new Set(['failure', 'cancelled', 'timed_out', 'startup_failure', 'action_required']);
function gh(args) {
const env = { ...process.env, PATH: `${process.env.PATH || ''}:/opt/homebrew/bin:/usr/local/bin` };
const r = spawnSync('gh', args, { encoding: 'utf8', env, timeout: 60000 });
if (r.error) throw new Error(`gh: ${r.error.message}`);
if (r.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')}: exit ${r.status}: ${(r.stderr || '').trim().slice(0, 200)}`);
return JSON.parse(r.stdout || 'null');
}
function fullSha(sha) {
if (/^[0-9a-f]{40}$/.test(sha)) return sha; // gh's --commit filter matches the full sha only (an abbreviation answers nothing)
const r = spawnSync('git', ['rev-parse', '--verify', `${sha}^{commit}`], { encoding: 'utf8' });
if (r.status !== 0) throw new Error(`${sha} is not a commit here`);
return r.stdout.trim();
}
export const newest = (runs) => [...(runs || [])].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt))[0];
export function verdictOf(run) {
if (!run) return 'none';
if (run.status !== 'completed') return 'pending';
return run.conclusion || 'pending';
}
export function firstRed(jobs) {
for (const j of jobs || []) {
if (j.conclusion === 'success' || j.conclusion === 'skipped' || j.conclusion == null) continue;
const s = (j.steps || []).find((x) => x.conclusion && x.conclusion !== 'success' && x.conclusion !== 'skipped');
return `${j.name.replace(/,.*$/, '')} at "${s ? s.name : '(no step)'}"`;
}
return '';
}
const london = (iso) => new Intl.DateTimeFormat('en-GB', { timeZone: 'Europe/London', hour: '2-digit', minute: '2-digit', hour12: false }).format(new Date(iso)) + ' UK';
const runsForSha = (sha) => gh(['run', 'list', '--repo', REPO, '--workflow', 'ci', '--commit', sha, '--limit', '10', '--json', FIELDS]) || [];
const runsForBranch = (branch, limit = 12) => gh(['run', 'list', '--repo', REPO, '--workflow', 'ci', '--branch', branch, '--limit', String(limit), '--json', FIELDS]) || [];
const jobsOf = (id) => (gh(['run', 'view', String(id), '--repo', REPO, '--json', 'jobs']) || {}).jobs || [];
export function stateOfSha(sha) {
const run = newest(runsForSha(fullSha(sha)));
const state = verdictOf(run);
if (!run) return `none - - no ci run on ${sha.slice(0, 8)} yet`;
let detail = state === 'pending' ? `${run.status} since ${london(run.createdAt)}` : `${run.event} run at ${london(run.createdAt)}`;
if (RED.has(state)) { const red = firstRed(jobsOf(run.databaseId)); if (red) detail += `: ${red}`; }
return `${state} ${run.databaseId} ${run.url} ${detail}`;
}
export function masterCodeState() {
const runs = [...runsForBranch('master', 12)].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt));
for (const run of runs) {
if (run.status !== 'completed') continue;
const jobs = jobsOf(run.databaseId);
const pow = jobs.find((j) => /^igneum-pow/.test(j.name));
if (!pow || pow.conclusion === 'skipped') continue; // a docs-only push: its green says nothing about the suite
const red = RED.has(run.conclusion) ? firstRed(jobs) : '';
return `${run.conclusion} ${run.databaseId} ${run.url} master @${run.headSha.slice(0, 8)} at ${london(run.createdAt)}, compile job ${pow.conclusion}${red ? `: ${red}` : ''}`;
}
return 'none - - no completed master ci run with the compile job among the last 12';
}
export function branchRedLine(branch) {
const runs = [...runsForBranch(branch, 12)].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt));
const done = runs.find((r) => r.status === 'completed');
if (!done || !RED.has(done.conclusion)) return '';
const red = firstRed(jobsOf(done.databaseId));
return `previous CI red on ${branch}: @${done.headSha.slice(0, 7)} ${done.conclusion} at ${london(done.createdAt)}${red ? `: ${red}` : ''} ${done.url} (no newer green run on this branch; this push gets its own run, read it)`;
}
async function selfTest() {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-state-'));
const fake = path.join(dir, 'gh');
// the fake gh answers `run list` from list-<commit or branch>.json and `run view <id>` from view-<id>.json; FAKE_GH_FAIL=1 fails
fs.writeFileSync(fake, `#!/usr/bin/env bash
[ "\${FAKE_GH_FAIL:-0}" = 1 ] && { echo "HTTP 502 from the fake" >&2; exit 1; }
key=""; prev=""
for a in "$@"; do case "$prev" in --commit|--branch) key="$a" ;; esac; prev="$a"; done
case "$1 $2" in
"run list") f="$FAKE_GH_DIR/list-$key.json"; [ -f "$f" ] && cat "$f" || echo '[]' ;;
"run view") f="$FAKE_GH_DIR/view-$3.json"; [ -f "$f" ] && cat "$f" || echo '{"jobs":[]}' ;;
*) echo "fake gh: unknown $*" >&2; exit 1 ;;
esac
`, { mode: 0o755 });
const sha = (c) => c.repeat(40);
const run = (id, status, conclusion, created, headSha = sha('a')) => ({ databaseId: id, status, conclusion, headSha, url: `https://github.com/x/y/actions/runs/${id}`, createdAt: created, event: 'push' });
const w = (name, obj) => fs.writeFileSync(path.join(dir, name), JSON.stringify(obj));
w(`list-${sha('a')}.json`, [run(1, 'completed', 'success', '2026-10-07T15:00:00Z')]);
w(`list-${sha('b')}.json`, [run(2, 'completed', 'failure', '2026-10-07T15:00:00Z', sha('b'))]);
w('view-2.json', { jobs: [{ name: 'site build, link check', conclusion: 'success', steps: [] }, { name: 'igneum-pow tests, igneum-census build', conclusion: 'failure', steps: [{ name: 'toolchain', conclusion: 'success' }, { name: 'igneum-pow tests (release)', conclusion: 'failure' }] }] });
w(`list-${sha('c')}.json`, [run(3, 'in_progress', null, '2026-10-07T15:00:00Z', sha('c'))]);
w(`list-${sha('d')}.json`, [run(4, 'completed', 'failure', '2026-10-07T14:00:00Z', sha('d')), run(5, 'completed', 'success', '2026-10-07T15:00:00Z', sha('d'))]); // the re-run wins
// master: the newest run is a docs-only green (compile job skipped); the one before it ran the compile job and is red
w('list-master.json', [run(10, 'completed', 'success', '2026-10-07T16:00:00Z', sha('1')), run(11, 'completed', 'failure', '2026-10-07T15:30:00Z', sha('2')), run(12, 'completed', 'success', '2026-10-07T15:00:00Z', sha('3'))]);
w('view-10.json', { jobs: [{ name: 'igneum-pow tests, igneum-census build', conclusion: 'skipped', steps: [] }, { name: 'site build', conclusion: 'success', steps: [] }] });
w('view-11.json', { jobs: [{ name: 'igneum-pow tests, igneum-census build', conclusion: 'failure', steps: [{ name: 'igneum-pow tests (release)', conclusion: 'failure' }] }] });
// branches: x red newest; y green newest; z pending newest over a red
w('list-x.json', [run(20, 'completed', 'failure', '2026-10-07T15:00:00Z', sha('e'))]);
w('view-20.json', { jobs: [{ name: 'site build, link check, identity grep', conclusion: 'failure', steps: [{ name: 'the tree gate', conclusion: 'failure' }] }] });
w('list-y.json', [run(21, 'completed', 'success', '2026-10-07T15:00:00Z'), run(22, 'completed', 'failure', '2026-10-07T14:00:00Z')]);
w('list-z.json', [run(23, 'queued', null, '2026-10-07T15:10:00Z'), run(24, 'completed', 'cancelled', '2026-10-07T15:00:00Z', sha('f'))]);
const me = new URL(import.meta.url).pathname;
const ask = (args, extraEnv = {}) => {
const r = spawnSync(process.execPath, [me, ...args], { encoding: 'utf8', env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, FAKE_GH_DIR: dir, ...extraEnv } });
return { out: (r.stdout || '').trim(), code: r.status, err: (r.stderr || '').trim() };
};
const fails = [];
const expect = (name, got, re) => { if (!re.test(got.out) || got.code !== 0) fails.push(`${name}: got exit ${got.code} "${got.out}" ${got.err}`); };
expect('success', ask([sha('a')]), /^success 1 https:\/\/github.com\/x\/y\/actions\/runs\/1 push run at /);
expect('failure names the red step', ask([sha('b')]), /^failure 2 \S+ push run at \d\d:\d\d UK: igneum-pow tests at "igneum-pow tests \(release\)"$/);
expect('pending', ask([sha('c')]), /^pending 3 \S+ in_progress since /);
expect('newest wins', ask([sha('d')]), /^success 5 /);
expect('none', ask([sha('9')]), /^none - - no ci run on 99999999 yet$/);
expect('gh error is unknown', ask([sha('a')], { FAKE_GH_FAIL: '1' }), /^unknown - - gh run list: exit 1: HTTP 502 from the fake/);
expect('master-code walks past the docs-only green', ask(['--master-code']), /^failure 11 \S+ master @22222222 at \d\d:\d\d UK, compile job failure: igneum-pow tests at "igneum-pow tests \(release\)"$/);
expect('branch red line', ask(['--branch-red', 'x']), /^previous CI red on x: @eeeeeee failure at \d\d:\d\d UK: site build at "the tree gate" https:\/\/github.com\/x\/y\/actions\/runs\/20 \(no newer green/);
const y = ask(['--branch-red', 'y']); if (y.out !== '' || y.code !== 0) fails.push(`branch green prints nothing: "${y.out}" exit ${y.code}`);
expect('branch pending over a cancelled run still names the red', ask(['--branch-red', 'z']), /^previous CI red on z: @fffffff cancelled at /);
const noArg = ask([]); if (noArg.code !== 2) fails.push(`usage: exit ${noArg.code}`);
fs.rmSync(dir, { recursive: true, force: true });
if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); }
console.log('self-test passed: a commit answers success, failure (with the red step), pending, none; the newest run wins; a gh error is unknown; --master-code walks past a docs-only green to the run that compiled; --branch-red names the newest completed red and stays silent on green');
}
const args = process.argv.slice(2);
try {
if (args[0] === '--self-test') await selfTest();
else if (args[0] === '--master-code') console.log(masterCodeState());
else if (args[0] === '--branch-red') { if (!args[1]) { console.error('usage: ci-state.mjs --branch-red <branch>'); process.exit(2); } console.log(branchRedLine(args[1])); }
else if (args[0] && !args[0].startsWith('-')) console.log(stateOfSha(args[0]));
else { console.error('usage: tools/ci/ci-state.mjs <sha> | --master-code | --branch-red <branch> | --self-test'); process.exit(2); }
} catch (e) {
console.log(`unknown - - ${e.message.replace(/\s+/g, ' ').slice(0, 300)}`);
}

161
tools/ci/merge-to-master.sh
View file

@ -5,34 +5,169 @@
# two-parent merge of the branch onto the exact remote tip, which the hook lets through on the light gate (20 s) while CI runs
# the full gate on landing. Retries while master moves. Never force; never from a dirty branch.
#
# tools/ci/merge-to-master.sh [<branch>] [--tries N] default: the current branch, 6 tries
# MERGE_REMOTE=box tools/ci/merge-to-master.sh ... land on another remote's master (the box mirror /srv/igneum.git while GitHub is
# suspended, 7 October 2026); the default stays origin
# The CI rule (standing rule, 7 October 2026, 17:2x UK): a merge lands only when the branch's OWN ci run is green on the exact
# commit being merged, read from the runs API (tools/ci/ci-state.mjs), not the local stamp alone. No run yet: the branch is pushed
# so CI runs it. A queued or running run: this tool waits, printing the UK clock, up to --ci-wait minutes (default 25). A red run:
# refused with the run's red check; fix the branch and push a new commit. And master itself must not be red: the newest master run
# that compiled (docs-only runs skip the compile job and their green says nothing) must be success, or the merge is refused unless
# it is the fix (--fixes-master). Record: era-vdf's tip 0e2d6b1c was merged with no ci run; master's igneum-pow suite was red from
# 16:31 UK and five docs-only merges landed green over it. The pre-push hook holds the same rule (pre-push.sh master_ci_ok).
#
# tools/ci/merge-to-master.sh [<branch>] [--tries N] [--ci-wait MIN] [--fixes-master] [--remote <name>] default: the current branch, 6 tries, origin
# MERGE_REMOTE=box tools/ci/merge-to-master.sh ... the same as --remote box (the box mirror /srv/igneum.git while GitHub is suspended, 7 October 2026)
# tools/ci/merge-to-master.sh --self-test the CI verdicts, with a fake ci-state: green goes, red refuses, master red refuses
# unless --fixes-master, none pushes the branch, pending waits then goes
set -euo pipefail
ROOT=$(git rev-parse --show-toplevel); cd "$ROOT"
BRANCH="$(git rev-parse --abbrev-ref HEAD)"; TRIES=6; R="${MERGE_REMOTE:-origin}"
while [ $# -gt 0 ]; do case "$1" in --tries) TRIES="$2"; shift 2 ;; -*) echo "unknown option $1" >&2; exit 2 ;; *) BRANCH="$1"; shift ;; esac; done
BRANCH="$(git rev-parse --abbrev-ref HEAD)"; TRIES=6; CI_WAIT_MIN="${CI_WAIT_MIN:-25}"; FIXES_MASTER=0; SELF_TEST=0; REMOTE="${MERGE_REMOTE:-origin}"
while [ $# -gt 0 ]; do case "$1" in --tries) TRIES="$2"; shift 2 ;; --ci-wait) CI_WAIT_MIN="$2"; shift 2 ;; --fixes-master) FIXES_MASTER=1; shift ;; --remote) REMOTE="$2"; shift 2 ;; --self-test) SELF_TEST=1; shift ;; -*) echo "unknown option $1" >&2; exit 2 ;; *) BRANCH="$1"; shift ;; esac; done
# --remote <name>: land on another remote's master (a box mirror, build@<box>:/srv/igneum.git, while GitHub is unreachable; main's
# ruling of 7 October 2026, 19:5x UK). CI runs on GitHub only, so the CI rule binds the origin remote; on a mirror the box gate
# stamp is the verdict and the merge message says so. IGNEUM_MASTER_EXCEPTION, when set, is printed by the hook with the push.
remote_is_github() { case "$(git remote get-url "$REMOTE" 2>/dev/null)" in *github.com*) return 0 ;; *) return 1 ;; esac; }
CI_STATE="${CI_STATE_CMD:-node tools/ci/ci-state.mjs}" # the self-test swaps in a fake
clock() { TZ=Europe/London date '+%H:%M %Z'; }
# ci_gate <sha> <branch> <push-cmd...>: 0 when the branch's own ci run on <sha> is green; pushes the branch when there is no run;
# waits on a queued or running run; refuses (1) a red or unknown one. master_gate: refuses (1) when master's last compiled run is red.
ci_gate() {
local sha="$1" branch="$2"; shift 2; local line state deadline pushed=0
deadline=$(( $(date +%s) + ${CI_WAIT_SECS:-$(( CI_WAIT_MIN * 60 ))} ))
while :; do
line=$($CI_STATE "$sha" 2>&1); state="${line%% *}"
case "$state" in
success) echo "merge-to-master: ci on ${sha:0:8} is green: $line"; return 0 ;;
none) if [ "$pushed" = 1 ]; then echo "merge-to-master: waiting for a ci run to appear on ${sha:0:8} ($(clock))"; else echo "merge-to-master: no ci run on ${sha:0:8} yet; pushing $branch so CI runs it ($(clock))"; "$@" || { echo "merge-to-master: the branch push failed; CI cannot run ${sha:0:8}" >&2; return 1; }; pushed=1; fi ;;
pending) echo "merge-to-master: ci on ${sha:0:8} is ${line#* * * }; waiting ($(clock))" ;;
unknown) echo "merge-to-master: ci on ${sha:0:8} could not be read (${line#* * * }); GitHub or gh wobbling, asking again ($(clock))" ;; # 7 Oct 2026 18:0x UK: 500s on pushes, a 404 on a live run id
*) echo "merge-to-master: REFUSED. ci on ${sha:0:8} is $line" >&2; echo " A branch whose own ci run is not green never merges (CI red is stop-the-line). Fix the branch, push a new commit, and run this again." >&2; return 1 ;;
esac
[ "$(date +%s)" -lt "$deadline" ] || { echo "merge-to-master: REFUSED. ci on ${sha:0:8} gave no verdict in $CI_WAIT_MIN minutes ($line); run it again when the queue drains (gh run list --branch $branch --limit 3)" >&2; return 1; }
sleep "${CI_POLL_SECS:-30}"
done
}
master_gate() {
local line state try
for try in 1 2 3; do # an unknown read (gh or GitHub wobbling) is asked again, three times over a minute, before it counts
line=$($CI_STATE --master-code 2>&1); state="${line%% *}"
[ "$state" = unknown ] && [ "$try" -lt 3 ] && { echo "merge-to-master: master's verdict could not be read (${line#* * * }); asking again ($(clock))"; sleep "${CI_POLL_SECS:-30}"; continue; }
break
done
case "$state" in
success|none|pending) echo "merge-to-master: master's last compiled run: $line"; return 0 ;;
*) if [ "$FIXES_MASTER" = 1 ]; then echo "merge-to-master: master's last compiled run is $state and this merge is declared the fix (--fixes-master): $line"; return 0; fi
echo "merge-to-master: REFUSED. master is red: $line" >&2; echo " CI red is stop-the-line: the owner fixes or reverts first; only the fix merges, with --fixes-master." >&2; return 1 ;;
esac
}
# After a landing, master goes to the box mirrors too (7 October 2026, 18:25 UTC: build-2's /srv/igneum.git master sat at 15:27's
# a4bca198 while GitHub's carried cf7d6ccb, and three branches were cut from the stale tip; the mirrors only ever received what a
# build happened to push). One push per mirror in IGNEUM_MIRRORS (default: both box files), fast-forward only, best effort: a mirror
# that is down prints a line and never fails the landing.
mirror_master() { # <sha>
local sha="$1" m host key="${BS_KEY:-$HOME/.ssh/igneum_ed25519}" list="${IGNEUM_MIRRORS:-}"
if [ -z "$list" ]; then for f in "$HOME/.config/igneum/build-server" "$HOME/.config/igneum/build-server-2"; do [ -s "$f" ] && list="$list $(head -1 "$f" | tr -d '[:space:]'):/srv/igneum.git"; done; fi
for m in $list; do
if GIT_SSH_COMMAND="ssh -i $key -o BatchMode=yes -o ConnectTimeout=10" git push -q "$m" "$sha:refs/heads/master" 2>/dev/null; then echo "merge-to-master: mirror $m master -> ${sha:0:8}"
else echo "merge-to-master: mirror $m did not take master ${sha:0:8} (down, or not a fast-forward); the next landing tries again"; fi
done
}
if [ "$SELF_TEST" = 1 ]; then
fails=0; d=$(mktemp -d); fake="$d/ci-state.sh"
# the fake answers from $d/answer-<sha> (one line per call, consumed top to bottom; the last line repeats) and $d/answer-master
cat > "$fake" <<'FAKE'
#!/usr/bin/env bash
d=$(dirname "$0"); key="$1"; [ "$key" = --master-code ] && key=master
f="$d/answer-$key"; [ -f "$f" ] || { echo "none - - no ci run on $key yet"; exit 0; }
n=$(wc -l < "$f" | tr -d ' '); i=$(cat "$d/i-$key" 2>/dev/null || echo 1); [ "$i" -gt "$n" ] && i=$n
sed -n "${i}p" "$f"; echo $((i + 1)) > "$d/i-$key"
FAKE
chmod +x "$fake"; export CI_STATE_CMD="$fake" CI_POLL_SECS=0 CI_WAIT_MIN=1
CI_STATE="$fake"
printf 'success 1 u push run at 16:00 UK
' > "$d/answer-green"
ci_gate green b false >/dev/null || { echo "self-test failed: a green branch run was refused"; fails=1; }
printf 'failure 2 u push run: igneum-pow tests at "igneum-pow tests (release)"
' > "$d/answer-red"
ci_gate red b false >/dev/null 2>&1 && { echo "self-test failed: a red branch run merged"; fails=1; }
printf 'unknown - - gh: exit 1\n' > "$d/answer-unk"
CI_WAIT_SECS=1 ci_gate unk b false >/dev/null 2>&1 && { echo "self-test failed: a lasting unknown verdict merged"; fails=1; }
printf 'unknown - - gh: HTTP 404\nsuccess 9 u push run\n' > "$d/answer-blip"
ci_gate blip b false >/dev/null 2>&1 || { echo "self-test failed: a GitHub blip (unknown, then green) was refused instead of waited through"; fails=1; }
printf 'pending 3 u queued since 16:00 UK
pending 3 u in_progress since 16:00 UK
success 3 u push run at 16:05 UK
' > "$d/answer-wait"
out=$(ci_gate wait b false 2>&1) || { echo "self-test failed: a pending run that turned green was refused: $out"; fails=1; }
[ "$(printf '%s
' "$out" | grep -c 'waiting')" = 2 ] || { echo "self-test failed: the wait did not print the clock twice: $out"; fails=1; }
# no run: the branch is pushed (the push command runs once), then the run appears and goes green
printf 'none - - no ci run on none yet
success 4 u push run
' > "$d/answer-none"; : > "$d/pushes"
ci_gate none b bash -c "echo pushed >> '$d/pushes'" >/dev/null || { echo "self-test failed: an unrun branch was refused instead of pushed"; fails=1; }
[ "$(wc -l < "$d/pushes" | tr -d ' ')" = 1 ] || { echo "self-test failed: the branch was pushed $(wc -l < "$d/pushes") times"; fails=1; }
# master red: refused; with --fixes-master: goes; master green or unknown-none: goes
printf 'failure 5 u master @1210158d, compile job failure
' > "$d/answer-master"
FIXES_MASTER=0 master_gate >/dev/null 2>&1 && { echo "self-test failed: a merge onto a red master was let through"; fails=1; }
rm -f "$d/i-master"; FIXES_MASTER=1 master_gate >/dev/null || { echo "self-test failed: the declared fix was refused on a red master"; fails=1; }
printf 'success 6 u master @e5171a32, compile job success
' > "$d/answer-master"; rm -f "$d/i-master"
FIXES_MASTER=0 master_gate >/dev/null || { echo "self-test failed: a green master refused a merge"; fails=1; }
printf 'unknown - - gh: HTTP 404\nsuccess 7 u master, compile job success\n' > "$d/answer-master"; rm -f "$d/i-master"
FIXES_MASTER=0 master_gate >/dev/null || { echo "self-test failed: an unknown master read that turned green refused a merge"; fails=1; }
printf 'unknown - - gh: exit 1\n' > "$d/answer-master"; rm -f "$d/i-master"
FIXES_MASTER=0 master_gate >/dev/null 2>&1 && { echo "self-test failed: a lasting unknown master read let a merge through"; fails=1; }
# the mirror push: a bare repository behind master is fast-forwarded; one that is not a fast-forward is left alone with a line
mkdir -p "$d/src" && ( cd "$d/src" && git init -q -b master . && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m one && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m two ) 2>/dev/null
git clone -q --bare "$d/src" "$d/mirror.git" && ( cd "$d/mirror.git" && git update-ref refs/heads/master "$(git rev-parse master~1)" )
tip=$(git -C "$d/src" rev-parse master)
out=$( cd "$d/src" && IGNEUM_MIRRORS="$d/mirror.git" mirror_master "$tip" )
[ "$(git -C "$d/mirror.git" rev-parse master)" = "$tip" ] || { echo "self-test failed: the mirror was not fast-forwarded to the landed master: $out"; fails=1; }
( cd "$d/src" && git checkout -q -b other master~1 && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m fork ); fork=$(git -C "$d/src" rev-parse other)
out=$( cd "$d/src" && IGNEUM_MIRRORS="$d/mirror.git" mirror_master "$fork" )
[ "$(git -C "$d/mirror.git" rev-parse master)" = "$tip" ] || { echo "self-test failed: a non-fast-forward push moved the mirror's master"; fails=1; }
case "$out" in *"did not take master"*) ;; *) echo "self-test failed: the refused mirror push was not reported: $out"; fails=1 ;; esac
# the remote decides the rule: origin (GitHub) binds the CI gate; a mirror remote does not
REMOTE=origin; remote_is_github || { echo "self-test failed: origin was not read as GitHub (is origin's URL github.com?)"; fails=1; }
( cd "$d/src" && git remote add mirror "$d/mirror.git" ) 2>/dev/null; REMOTE=mirror; ( cd "$d/src" && remote_is_github ) && { echo "self-test failed: a bare-path mirror remote was read as GitHub"; fails=1; }; REMOTE=origin
rm -rf "$d"
[ "$fails" = 0 ] && echo "self-test passed: the CI rule binds a GitHub remote only; a landed master is fast-forwarded to every mirror and a refused mirror push is a line, not a failure; a green branch run merges; a red one is refused; an unknown read is waited through and refused only at the deadline; a queued run is waited for with the clock; an unrun branch is pushed once and then waited for; a red master refuses every merge but the declared fix"
exit $fails
fi
[ -z "$(git status --porcelain --untracked-files=no)" ] || { echo "merge-to-master: the tree has uncommitted tracked changes; commit first" >&2; exit 1; }
SHA=$(git rev-parse "$BRANCH"); G=$(cd "$(git rev-parse --git-common-dir)" && pwd -P)
if [ ! -f "$G/igneum-gate-green/$SHA" ]; then
echo "merge-to-master: no green stamp for ${SHA:0:8}; running the full gate on the branch first"
echo "merge-to-master: no green stamp for ${SHA:0:8}; running the full gate on the branch first (then CI's own verdict on it is read)"
bash tools/ci/pre-push.sh || exit 1
[ -f "$G/igneum-gate-green/$SHA" ] || { echo "merge-to-master: the gate was green but no stamp was written (dirty tree?)" >&2; exit 1; }
fi
AUTHOR=(-c user.name=igneum-josh -c user.email=337424239+igneum-josh@users.noreply.github.com)
# the CI rule: the branch's own run on this exact commit must be green (pushed for a run when there is none, waited for when queued), and master must not be red
if remote_is_github; then
ci_gate "$SHA" "$BRANCH" git push -q "$REMOTE" "$SHA:refs/heads/$BRANCH" || exit 1
master_gate || exit 1
VERDICT="gate: green on ${SHA:0:8}, recorded by tools/ci/pre-push.sh; CI green on ${SHA:0:8}; the full gate runs in CI on this merge"
else
echo "merge-to-master: the remote $REMOTE is not GitHub (a mirror); the box gate stamp on ${SHA:0:8} is the verdict${IGNEUM_MASTER_EXCEPTION:+ (exception: $IGNEUM_MASTER_EXCEPTION)}"
VERDICT="gate: green on ${SHA:0:8}, recorded by tools/ci/pre-push.sh; landed on the $REMOTE mirror${IGNEUM_MASTER_EXCEPTION:+ under the exception declared by main: $IGNEUM_MASTER_EXCEPTION}"
fi
for i in $(seq 1 "$TRIES"); do
git fetch -q "$R" master; TIP=$(git rev-parse "$R/master")
if git merge-base --is-ancestor "$SHA" "$TIP"; then echo "merge-to-master: ${SHA:0:8} is already on $R/master $(git log -1 --format=%h "$R/master")"; exit 0; fi
git fetch -q "$REMOTE" master; TIP=$(git rev-parse "$REMOTE/master")
if git merge-base --is-ancestor "$SHA" "$TIP"; then echo "merge-to-master: ${SHA:0:8} is already on $REMOTE/master $(git log -1 --format=%h "$REMOTE/master")"; exit 0; fi
W=$(mktemp -d "${TMPDIR:-/tmp}/merge-to-master.XXXXXX"); rmdir "$W"
git worktree add -q --detach "$W" "$TIP"
if ( cd "$W" && git "${AUTHOR[@]}" merge -q --no-ff -m "Merge $BRANCH ${SHA:0:8} into master (gate: green on ${SHA:0:8}, recorded by tools/ci/pre-push.sh; the full gate runs in CI on this merge)" "$SHA" ); then
if ( cd "$W" && git push -q "$R" HEAD:master ); then
git worktree remove --force "$W"; git fetch -q "$R" master
echo "merge-to-master: pushed on try $i: $R/master $(git log -1 --format='%h %ci' "$R/master") $(TZ=Europe/London date '+%H:%M %Z')"; exit 0
if ( cd "$W" && git "${AUTHOR[@]}" merge -q --no-ff -m "Merge $BRANCH ${SHA:0:8} into master ($VERDICT)" "$SHA" ); then
if ( cd "$W" && git push -q "$REMOTE" HEAD:master ); then # on a GitHub remote the hook asks ci-state about ${SHA:0:8} once more
git worktree remove --force "$W"; git fetch -q "$REMOTE" master
echo "merge-to-master: pushed on try $i: $REMOTE/master $(git log -1 --format='%h %ci' "$REMOTE/master") $(TZ=Europe/London date '+%H:%M %Z')"
remote_is_github && mirror_master "$(git rev-parse "$REMOTE/master")"; exit 0
fi
echo "merge-to-master: try $i: the push was rejected (master moved or the hook was red); again"
else
echo "merge-to-master: the merge of $BRANCH onto ${TIP:0:8} does not apply cleanly; resolve on the branch (git merge $R/master) and retry" >&2
echo "merge-to-master: the merge of $BRANCH onto ${TIP:0:8} does not apply cleanly; resolve on the branch (git merge origin/master) and retry" >&2
git worktree remove --force "$W"; exit 1
fi
git worktree remove --force "$W" 2>/dev/null || true

86
tools/ci/pre-push.sh
View file

@ -26,6 +26,7 @@ cd "$(git rev-parse --show-toplevel)" || exit 1
# and fire this hook again inside the fixture: the first master push through the gate died that way (6 October 2026).
unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_PREFIX GIT_COMMON_DIR GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_QUARANTINE_PATH GIT_PUSH_OPTION_COUNT
MODE="${1:-local}"; MODE="${MODE#--}"
GATE_ROOT="$(pwd -P)" # the readers below are called from fixture repositories in the self-test, so by absolute path
RED=0; N=0; LOG="$(mktemp)"; trap 'rm -rf "$LOG" "${SITE_TMP:-}"' EXIT
T0=$(date +%s)
@ -52,12 +53,19 @@ site_build() {
(cd "$SITE_TMP/site" && SITE_DOWNLOADS_OFFLINE=1 node build.mjs)
}
wall_clock() { # <secs> <command...>: under GNU timeout where it exists (the runners); the Mac has none, and the job's timeout-minutes is the stop there.
# No array here: an empty array expanded under set -u is "unbound variable" on the Mac's bash 3.2 and ended the gate with no RED line (17:3x UK, 7 October 2026).
local secs="$1"; shift
if command -v timeout >/dev/null 2>&1; then timeout "$secs" "$@"; else "$@"; fi
}
overlap_sweep() {
# tools/ci/overlap-check.mjs: the known-failed fixture first, then the built site (the gate's temporary copy locally, the tree in
# CI). A browser is needed: CI installs Playwright in the workflow; a machine without one ships the pages to a build box
# (nothing heavy on the Mac). IGNEUM_OVERLAP_APPS=1 adds the miner and wallet UIs through their mocks (slower, the box).
local dir="site"; [ "$MODE" = ci ] || dir="$SITE_TMP/site"
if [ "${IGNEUM_OVERLAP_APPS:-0}" = 1 ]; then node tools/ci/overlap-check.mjs --self-test --site "$dir" --apps .; else node tools/ci/overlap-check.mjs --self-test --site "$dir"; fi
# a wall clock of 10 minutes where GNU timeout exists (the runners; the Mac ships the sweep to a box): the sweep took 192 s on a hosted
# runner on 7 October 2026 and three master jobs hung in this step for over two hours each the same afternoon
if [ "${IGNEUM_OVERLAP_APPS:-0}" = 1 ]; then wall_clock 600 node tools/ci/overlap-check.mjs --self-test --site "$dir" --apps .; else wall_clock 600 node tools/ci/overlap-check.mjs --self-test --site "$dir"; fi
}
structural_checks() {
@ -104,7 +112,7 @@ tree_checks() {
run "long-running tools keep their body in one parsed block (the edited-while-running class)" bash -c 'bash tools/ci/whole-body-check.sh --self-test && bash tools/ci/whole-body-check.sh'
run "build-remote without a priority flag bounds suites and benches (nice 10, 32 cores); a gate runs unbounded" bash tools/ci/build-kind-default-check.sh
run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh
run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh)" bash tools/ci/box-locks-check.sh $( [ "$MODE" = ci ] && echo --ci )
run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh; skipped on a runner with no box; one retry)" bash tools/ci/retry-once.sh box-locks bash tools/ci/box-locks-check.sh $( [ "$MODE" = ci ] && echo --ci )
run "the simulators job runs on master and release-* pushes and pull requests into them only" bash tools/ci/sims-branch-check.sh
run "publish-jobs.sh never removes a running or installs-app job without --force (the PC 2 abort class)" bash tools/ci/publish-jobs-check.sh
run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh'
@ -118,7 +126,7 @@ tree_checks() {
run "chain scene: the site's and the app's copies are scene/ byte for byte, the palette tokens live once (self-test, then the tree)" bash -c 'node tools/scene/sync.mjs --self-test && node tools/scene/sync.mjs --check'
run "chain scene: a push paints with the document hidden and no animation frame (the blank /live of 7 Oct 2026; known-failed first)" node tools/scene/paint-test.cjs
run "chain scene: the live feed contract (the recorded reply validates; a rewritten miner, a float now, a stray key refused)" node --test tools/scene/feed-contract.test.mjs tools/scene/legend.test.mjs tools/scene/shard-words.test.mjs
run "chain scene parity: one recorded feed through the home fold, /live and the app's Inspect view on build-2, three frames each pixel-equal apart from the app's own-key overlay (a changed token fails first; skipped with no box and no Playwright)" bash tools/scene/parity-remote.sh
run "chain scene parity: one recorded feed through the home fold, /live and the app's Inspect view on build-2, three frames each pixel-equal apart from the app's own-key overlay (a changed token fails first; skipped with no box and no Playwright; one retry)" bash tools/ci/retry-once.sh scene-parity bash tools/scene/parity-remote.sh
run "no text overlaps: every served page at 390 to 1600 px, light and dark, the hero at each step (self-test first; IGNEUM_OVERLAP_APPS=1 adds the miner and wallet UIs)" overlap_sweep
run "explorer, emission and public stats unit tests" node --test site/lib/explorer.test.mjs site/lib/emission.test.mjs site/lib/money.test.mjs site/lib/leaderboard.test.mjs site/api/public-stats.test.mjs
run "ship tool self-test" node tools/ship-app.mjs --self-test
@ -129,6 +137,9 @@ tree_checks() {
run "hash-origin report: a known-finished day and a known-failed day" node --test tools/observer/hash-origin.test.mjs
run "harness summaries never carry a raw 64-hex key (the writer's own redaction and check)" node infra/fast-time/lib/redact-keys.mjs --self-test
run "docs-only pushes skip the compile-or-compute CI jobs (the changes job's classifier)" bash tools/ci/docs-only-check.sh --self-test
run "every workflow job carries timeout-minutes (site 15, changes 10, pow 60, sims 45; the hung-job class of 7 October 2026)" bash tools/ci/workflow-timeouts-check.sh --self-test
run "a box or network check gets one retry before it is red (retry-once self-test)" bash tools/ci/retry-once.sh --self-test
run "CI state reader: a commit's newest run, master's last compiled run, a branch's last red (fake gh; the merge rule's reader)" node tools/ci/ci-state.mjs --self-test
}
gated_refs() {
@ -164,6 +175,33 @@ deferred_merge() { # <local sha> <remote sha> [repo dir] -> "defer <branch sha
age=$(( $(date +%s) - $(stat -f %m "$f" 2>/dev/null || stat -c %Y "$f") )); [ "$age" -le 43200 ] || { echo "full the stamp for ${p2:0:8} is $age s old"; return; }
echo "defer $p2"
}
# Master takes only what CI has already passed (standing rule, 7 October 2026, 17:2x UK: era-vdf's tip 0e2d6b1c was merged with
# no ci run at all and master's igneum-pow suite stayed red for 40 minutes under docs-only merges). For a push to master the hook
# asks tools/ci/ci-state.mjs: a two-parent merge needs a green run on its SECOND parent (the branch's own run on the exact
# commit), a plain commit needs a green run on itself (a fast-forward of a branch CI passed); anything else is refused with the
# run's state, and the lane uses tools/ci/merge-to-master.sh, which waits for a queued run. gh unreachable = refused (unknown).
# release-* branches keep the full local gate alone (the shipper's cuts carry their own box suite line).
master_ci_ok() { # <local sha> <remote sha> -> 0 and a line, or 1 and the reason
local lsha="$1" rsha="$2" parents p2 want line state
parents=$(git rev-list --parents -n 1 "$lsha" 2>/dev/null | cut -d' ' -f2-); set -- $parents
if [ -n "${2:-}" ] && [ -z "${3:-}" ] && [ "$1" = "$rsha" ]; then want="$2"; else want="$lsha"; fi
line=$(node "$GATE_ROOT/tools/ci/ci-state.mjs" "$want" 2>&1); state="${line%% *}"
if [ "$state" = success ]; then echo " master takes ${want:0:8}: ci $line"; return 0; fi
echo "pre-push gate: REFUSED. master takes only a commit whose own ci run is green on that exact commit; ${want:0:8} is: $line" >&2
echo " Use tools/ci/merge-to-master.sh (it pushes the branch for a run when there is none, waits for a queued run and refuses a red)." >&2
return 1
}
master_rule_binds() { # <remote url>: 0 when the CI rule applies to this push (a GitHub remote, no declared exception), 1 with a printed line otherwise
local url="${1:-}"
if [ -n "${IGNEUM_MASTER_EXCEPTION:-}" ]; then echo " EXCEPTION to the CI rule for this push, declared by main: $IGNEUM_MASTER_EXCEPTION (the local gate is the verdict)"; return 1; fi
case "$url" in *github.com*) return 0 ;; esac
echo " the remote ${url:-?} is not GitHub (a mirror): the CI rule binds GitHub's master; the local gate is the verdict here"; return 1
}
branch_red_line() { # <branch>: the branch's newest completed ci run, when red, printed before the light gate (nothing on green or no gh)
local line; line=$(node "$GATE_ROOT/tools/ci/ci-state.mjs" --branch-red "$1" 2>/dev/null) || return 0
case "$line" in previous\ CI\ red*) echo " $line" ;; esac
return 0
}
finish() {
local what="$1" secs=$(( $(date +%s) - T0 ))
if [ "$RED" = 0 ]; then echo "pre-push gate ($what): GREEN, $N checks in ${secs}s"; [ "${STAMP:-0}" = 1 ] && stamp_green; exit 0; fi
@ -207,7 +245,36 @@ case "$MODE" in
MODE=ci GITHUB_ACTIONS= site_in_place && { echo "self-test failed: --ci outside GitHub Actions chose the in-place build"; fails=1; }
MODE=ci GITHUB_ACTIONS=true site_in_place || { echo "self-test failed: --ci inside GitHub Actions did not choose the in-place build"; fails=1; }
declare -f tree_checks | grep -q 'never_push_checks' || { echo "self-test failed: the full gate does not run the never-push checks"; fails=1; }
[ "$fails" = 0 ] && echo "self-test passed: a failing check is RED and fails the gate, a passing one is ok; master and release-* select the full gate, other refs the light one; a merge of a green-stamped branch onto the remote tip defers to CI, every other shape takes the full gate; a --ci site build outside GitHub Actions leaves the tree unchanged (structural checks, no-secrets, identity grep)"
# the overlap sweep's wall clock runs the command with GNU timeout where it exists and plainly where it does not (bash 3.2 under set -u included)
[ "$(wall_clock 5 /bin/echo clocked 2>&1)" = clocked ] || { echo "self-test failed: wall_clock did not run its command"; fails=1; }
[ "$(PATH=/nonexistent wall_clock 5 /bin/echo plain 2>&1)" = plain ] || { echo "self-test failed: wall_clock without a timeout binary did not run its command plainly"; fails=1; }
grep -q 'wall_clock 600 node tools/ci/overlap-check.mjs' "$0" || { echo "self-test failed: the overlap sweep does not run under the wall clock"; fails=1; }
# master takes only CI-passed commits: a merge asks about its second parent, a plain commit about itself; red, pending, none and unknown refuse
grep -qE 'master_ci_ok "\$lsha" "\$rsha" \|\| exit 1' "$0" || { echo "self-test failed: the hook does not ask ci-state before a push to master"; fails=1; }
grep -qE 'branch_red_line "\$\{rref#refs/heads/\}"' "$0" || { echo "self-test failed: the feature-branch hook does not print the branch's previous red"; fails=1; }
fx=$(mktemp -d); ( cd "$fx" && git init -q -b master . && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m a ) 2>/dev/null
A=$(git -C "$fx" rev-parse HEAD); git -C "$fx" checkout -q -b b; git -C "$fx" -c user.name=t -c user.email=t@t commit -q --allow-empty -m b; B=$(git -C "$fx" rev-parse HEAD)
git -C "$fx" checkout -q master; git -C "$fx" -c user.name=t -c user.email=t@t merge -q --no-ff -m "merge b" b; M=$(git -C "$fx" rev-parse HEAD)
fakebin=$(mktemp -d)
cat > "$fakebin/gh" <<FAKEGH
#!/usr/bin/env bash
# the fake gh of the gate's self-test: a green run on the branch commit, a queued run on the merge commit, nothing elsewhere
prev=""; key=""; for a in "\$@"; do [ "\$prev" = --commit ] && key="\$a"; prev="\$a"; done
row() { printf '[{"databaseId":%s,"status":"%s","conclusion":%s,"headSha":"%s","url":"u","createdAt":"2026-10-07T15:00:00Z","event":"push"}]\\n' "\$1" "\$2" "\$3" "\$key"; }
case "\$key" in $B) row 1 completed '"success"' ;; $M) row 2 queued null ;; *) echo "[]" ;; esac
FAKEGH
chmod +x "$fakebin/gh"
( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$M" "$A" >/dev/null 2>&1 ) || { echo "self-test failed: a merge whose branch parent has a green run was refused"; fails=1; }
( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$B" "$A" >/dev/null 2>&1 ) || { echo "self-test failed: a plain commit with its own green run was refused"; fails=1; }
( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$M" "$B" >/dev/null 2>&1 ) && { echo "self-test failed: a merge onto another tip (its own run queued) was let through"; fails=1; }
( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$A" "$A" >/dev/null 2>&1 ) && { echo "self-test failed: a commit with no ci run was let through to master"; fails=1; }
rm -rf "$fx" "$fakebin"
# the CI rule binds a GitHub remote; a mirror remote and a declared exception take the local gate, each with a printed line
master_rule_binds https://github.com/igneum-network/igneum.git >/dev/null || { echo "self-test failed: the CI rule did not bind a GitHub remote"; fails=1; }
master_rule_binds build@188.40.146.49:/srv/igneum.git >/dev/null && { echo "self-test failed: the CI rule bound a box mirror remote"; fails=1; }
( IGNEUM_MASTER_EXCEPTION="main, 7 Oct 2026 19:5x UK: GitHub suspended" master_rule_binds https://github.com/x/y.git >/dev/null ) && { echo "self-test failed: a declared exception did not lift the CI rule"; fails=1; }
out=$(IGNEUM_MASTER_EXCEPTION="ruling text" master_rule_binds https://github.com/x/y.git); case "$out" in *"EXCEPTION"*"ruling text"*) ;; *) echo "self-test failed: the exception was not printed with its ruling: $out"; fails=1 ;; esac
[ "$fails" = 0 ] && echo "self-test passed: a failing check is RED and fails the gate, a passing one is ok; master and release-* select the full gate, other refs the light one; a merge of a green-stamped branch onto the remote tip defers to CI, every other shape takes the full gate; master takes only a commit (or a merge's branch parent) whose own ci run is green, and refuses red, queued and unrun ones (GitHub remotes; a mirror remote or a declared exception takes the local gate, printed); a feature-branch push prints the branch's previous red first; a --ci site build outside GitHub Actions leaves the tree unchanged (structural checks, no-secrets, identity grep)"
exit $fails ;;
list)
grep -E '^\s+run "' "$0" | sed -E 's/^\s+run "([^"]+)".*/\1/' ;;
@ -216,6 +283,16 @@ case "$MODE" in
if [ "$which" = full ]; then
# a merge of a green-stamped branch onto the exact remote tip goes through on the light gate (CI runs the full one)
verdict=""; while read -r lref lsha rref rsha; do case "$rref" in refs/heads/master|refs/heads/release-*) verdict=$(deferred_merge "$lsha" "$rsha"); break ;; esac; done <<<"$REFS"
# a push to master: the pushed commit (or its branch parent) must already have a green ci run on that exact commit. CI runs on
# GitHub, so the rule binds a push whose remote is github.com; a push of master to a box mirror (build@<box>:/srv/igneum.git)
# takes the local gate as before. IGNEUM_MASTER_EXCEPTION="<main's ruling>" lifts the CI rule for one push and is printed with
# the push (7 October 2026, 19:5x UK: the GitHub account suspended, lanes landing on the box mirror's master by main's ruling,
# the box gate stamp as the verdict; GitHub gets the fast-forward when it answers again).
if master_rule_binds "${2:-}"; then
while read -r lref lsha rref rsha; do
if [ "$rref" = refs/heads/master ] && [ "$lsha" != 0000000000000000000000000000000000000000 ]; then master_ci_ok "$lsha" "$rsha" || exit 1; fi
done <<<"$REFS"
fi
case "$verdict" in
defer*) echo "pre-push gate: a merge of green-stamped ${verdict#defer } onto the remote tip: the light gate here, the full gate in CI on landing:"
structural_checks; never_push_checks; finish "merge of a green branch (full gate deferred to CI)" ;;
@ -224,6 +301,7 @@ case "$MODE" in
esac
else
echo "pre-push gate: a feature branch, the light gate (the two structural checks, the no-secrets check, the identity grep):"
while read -r lref lsha rref rsha; do case "$rref" in refs/heads/*) branch_red_line "${rref#refs/heads/}" ;; esac; done <<<"$REFS"
structural_checks; never_push_checks; finish "feature branch"
fi ;;
ci|local)

25
tools/ci/red-watch.mjs
View file

@ -78,6 +78,7 @@ export function runFromEnv(env = process.env) {
if (!sha) throw new Error('record: neither RED_WATCH_SHA nor GITHUB_SHA is set');
return {
run_id: id, attempt: Number(pick('RED_WATCH_ATTEMPT', 'GITHUB_RUN_ATTEMPT') || 1), workflow: pick('RED_WATCH_WORKFLOW', 'GITHUB_WORKFLOW'),
conclusion: env.RED_WATCH_CONCLUSION || 'failure', // failure, cancelled or timed_out (ci-red.yml fires on all three since 7 October 2026)
branch: pick('RED_WATCH_BRANCH', 'GITHUB_REF_NAME'), sha: sha.slice(0, 7), event: pick('RED_WATCH_EVENT', 'GITHUB_EVENT_NAME'),
actor: pick('RED_WATCH_ACTOR', 'GITHUB_ACTOR'), author: env.RED_WATCH_AUTHOR || '', // who pushed (the GitHub login), who the head commit names
url: env.RED_WATCH_URL || `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${id}`, at: new Date().toISOString(),
@ -124,11 +125,15 @@ export async function record(file, env = process.env, fetchImpl = fetch, title =
return { written: true, run: line };
}
// The kind of line: a failed run is "CI red"; a cancelled run "CI cancelled" (a hand on the run, or a job past its timeout-minutes
// under GitHub's older runner, which reports cancelled); a timed-out run "CI timed out". All three are read like a red.
export const KINDS = { failure: 'CI red', cancelled: 'CI cancelled', timed_out: 'CI timed out' };
export function formatLine(l) {
const where = l.failed.length ? l.failed.map((f) => `${f.job.replace(/,.*$/, '')} at "${f.step}"`).join('; ') : (l.note || 'no step detail');
const title = l.title ? ` "${l.title}"` : '';
const who = l.actor ? ` pushed by ${l.actor}${l.author && l.author !== l.actor ? ` (commit by ${l.author})` : ''};` : '';
return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}:${who} ${where} ${l.url}`;
const kind = KINDS[l.conclusion || 'failure'] || `CI ${l.conclusion}`;
return `${kind}: ${l.workflow} on ${l.branch} @${l.sha}${title}:${who} ${where} ${l.url}`;
}
function readCredentials(file) {
@ -259,6 +264,22 @@ async function selfTest() {
if (!asked[0] || !asked[0].includes('/actions/runs/37620364667/jobs')) fails.push(`workflow_run shape: the jobs API was asked for ${asked[0]}, not the failed run`);
const textRun = formatLine(lr);
if (!/^CI red: ci on ca3-v4-node @26a4b0f "Counter ASIC 3.0 node plan 6.7": pushed by igneum-labs \(commit by igneum-josh\); site build at "identity grep of the public export list"; simulators at "\(job never started: runner or billing\)" https:\/\/github.com\/igneum-network\/igneum\/actions\/runs\/37620364667$/.test(textRun)) fails.push(`workflow_run line: ${textRun}`);
// a cancelled run and a timed-out run are recorded with their kind in the line (a hung job past its timeout-minutes, a hand on the run)
const cancelledJobs = { jobs: [{ name: 'site build, link check, identity grep', conclusion: 'cancelled', steps: [{ name: 'site build', conclusion: 'success' }, { name: 'the tree gate, tools/ci/pre-push.sh --ci', conclusion: 'cancelled' }] }] };
const fileKinds = path.join(dir, 'kinds.jsonl');
await record(fileKinds, { ...envRun, RED_WATCH_RUN_ID: '555', RED_WATCH_CONCLUSION: 'cancelled' }, async () => ({ ok: true, status: 200, json: async () => cancelledJobs }));
await record(fileKinds, { ...envRun, RED_WATCH_RUN_ID: '556', RED_WATCH_CONCLUSION: 'timed_out' }, async () => ({ ok: true, status: 200, json: async () => cancelledJobs }));
const [lc, lt] = readLines(fileKinds).map(formatLine);
if (!/^CI cancelled: ci on ca3-v4-node @26a4b0f .*site build at "the tree gate, tools\/ci\/pre-push.sh --ci"/.test(lc)) fails.push(`cancelled line: ${lc}`);
if (!/^CI timed out: ci on ca3-v4-node @26a4b0f /.test(lt)) fails.push(`timed-out line: ${lt}`);
if (readLines(fileKinds)[0].conclusion !== 'cancelled') fails.push('record: the conclusion was not kept in the line');
// the watcher workflow fires on all three conclusions and hands the conclusion to the record step
const ymlPath = path.join(path.dirname(new URL(import.meta.url).pathname), '..', '..', '.github', 'workflows', 'ci-red.yml');
if (fs.existsSync(ymlPath)) {
const yml = fs.readFileSync(ymlPath, 'utf8');
for (const c of ['failure', 'cancelled', 'timed_out']) if (!yml.includes(`github.event.workflow_run.conclusion == '${c}'`)) fails.push(`ci-red.yml: the job's if does not fire on ${c}`);
if (!yml.includes('RED_WATCH_CONCLUSION: ${{ github.event.workflow_run.conclusion }}')) fails.push('ci-red.yml: RED_WATCH_CONCLUSION is not handed to the record step');
}
// post, dry run: prints, sends nothing, marks nothing
let printed = []; const log = (s) => printed.push(s);
const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; };
@ -307,7 +328,7 @@ async function selfTest() {
if (!d3.sent) fails.push('digest: not sent the next day');
fs.rmSync(dir, { recursive: true, force: true });
if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); }
console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author, the failed run and not the watcher\'s own under workflow_run; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author, the failed run and not the watcher\'s own under workflow_run; a cancelled and a timed-out run are recorded with their kind and ci-red.yml fires on all three; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
}
const cmd = args[0];

37
tools/ci/retry-once.sh Executable file
View file

@ -0,0 +1,37 @@
#!/usr/bin/env bash
# Run a check that can red on box state or the network twice before calling it red (standing rule, 7 October 2026, 17:2x UK:
# a check that talks to a build box or a live host gets one retry and, on a runner without the resource, a clear skip line
# from the check itself). The first failure is printed as a line, the command runs again after a pause, and only the second
# failure is the check's verdict. A check that passes first time runs once.
#
# tools/ci/retry-once.sh <name> <command...> # exit = the last run's exit code; RETRY_ONCE_PAUSE seconds between (default 5)
# tools/ci/retry-once.sh --self-test # a command that fails once then passes is ok; one that fails twice is red;
# # a command that passes runs exactly once
set -uo pipefail
PAUSE="${RETRY_ONCE_PAUSE:-5}"
if [ "${1:-}" = --self-test ]; then
fails=0; d=$(mktemp -d)
# fails once then passes: the marker file is the memory
flaky="$d/flaky.sh"; printf '#!/usr/bin/env bash\nif [ -f "%s/seen" ]; then exit 0; fi; touch "%s/seen"; echo first-try-failed; exit 7\n' "$d" "$d" > "$flaky"; chmod +x "$flaky"
out=$(RETRY_ONCE_PAUSE=0 bash "$0" flaky "$flaky" 2>&1); rc=$?
[ "$rc" = 0 ] || { echo "self-test failed: a command that fails once then passes was red (exit $rc): $out"; fails=1; }
case "$out" in *"retry-once: flaky failed once (exit 7)"*) ;; *) echo "self-test failed: the first failure was not printed: $out"; fails=1 ;; esac
# fails twice: red with the second exit code
out=$(RETRY_ONCE_PAUSE=0 bash "$0" dead bash -c 'echo nope; exit 3' 2>&1); rc=$?
[ "$rc" = 3 ] || { echo "self-test failed: a command that fails twice was not red with its exit code (got $rc): $out"; fails=1; }
[ "$(printf '%s\n' "$out" | grep -c '^nope$')" = 2 ] || { echo "self-test failed: the command did not run exactly twice: $out"; fails=1; }
# passes: runs once
c="$d/count"; : > "$c"
RETRY_ONCE_PAUSE=0 bash "$0" fine bash -c "echo x >> '$c'" >/dev/null 2>&1 || { echo "self-test failed: a passing command was red"; fails=1; }
[ "$(wc -l < "$c" | tr -d ' ')" = 1 ] || { echo "self-test failed: a passing command ran $(wc -l < "$c") times"; fails=1; }
rm -rf "$d"
[ "$fails" = 0 ] && echo "self-test passed: one retry after a first failure, red only on the second, a passing command runs once"
exit $fails
fi
[ $# -ge 2 ] || { echo "usage: tools/ci/retry-once.sh <name> <command...> | --self-test" >&2; exit 2; }
NAME="$1"; shift
"$@"; rc=$?
[ "$rc" = 0 ] && exit 0
echo "retry-once: $NAME failed once (exit $rc); second try in ${PAUSE}s (a box or network check gets one retry before it is red)"
sleep "$PAUSE"
"$@"

53
tools/ci/workflow-timeouts-check.sh Executable file
View file

@ -0,0 +1,53 @@
#!/usr/bin/env bash
# Every job in .github/workflows carries timeout-minutes, and the named jobs carry their budgets (7 October 2026, 15:13 to
# 17:20 UK: three hosted `site` jobs on master hung in the tree gate for over two hours each, with GitHub's six-hour default
# as the only stop; a fourth would have been a failure email at 21:13 UK). The budgets, from the measured times on
# 7 October: the tree gate on a hosted runner 229 s plus a 40 s Playwright install, so `site` 15; the classifier `changes`
# a 7 s API call, so 10; the box's igneum-pow suite 45 s to 2 min 40 s, so `pow` 60; the two simulators 2 x 120 s, so `sims` 45.
#
# tools/ci/workflow-timeouts-check.sh # exit 1 naming each job without a timeout or with the wrong budget
# tools/ci/workflow-timeouts-check.sh --self-test # a fixture job without the key fails; a wrong budget fails; the tree passes
set -euo pipefail
cd "$(dirname "$0")/../.."
# <file> -> lines "job timeout" for every job (timeout "-" when missing). A job is a key at four spaces under `jobs:`;
# its timeout-minutes is the key at six spaces before the next job.
jobs_of() {
awk '
/^jobs:/ { injobs = 1; next }
injobs && /^[^ ]/ { injobs = 0 }
injobs && /^ [A-Za-z0-9_-]+:/ { if (job != "") print job, (t == "" ? "-" : t); job = $1; sub(":", "", job); t = ""; next }
injobs && job != "" && /^ timeout-minutes:/ { t = $2 }
END { if (job != "") print job, (t == "" ? "-" : t) }
' "$1"
}
check_tree() { # <dir with workflows> -> exit 1 with one line per wrong job
local f rc=0 job t want
for f in "$1"/*.yml "$1"/*.yaml; do
[ -f "$f" ] || continue
while read -r job t; do
if [ "$t" = "-" ]; then echo "workflow-timeouts: $(basename "$f") job \`$job\` has no timeout-minutes" >&2; rc=1; continue; fi
want=""
case "$(basename "$f"):$job" in
ci.yml:site) want=15 ;; ci.yml:changes) want=10 ;; ci.yml:pow) want=60 ;; ci.yml:sims) want=45 ;;
esac
if [ -n "$want" ] && [ "$t" != "$want" ]; then echo "workflow-timeouts: $(basename "$f") job \`$job\` has timeout-minutes $t, the budget is $want" >&2; rc=1; fi
done < <(jobs_of "$f")
done
return $rc
}
if [ "${1:-}" = --self-test ]; then
fails=0; d=$(mktemp -d); mkdir -p "$d/bad" "$d/budget"
printf 'name: x\non: push\njobs:\n a:\n runs-on: ubuntu-latest\n timeout-minutes: 5\n steps: []\n b:\n runs-on: ubuntu-latest\n steps: []\n' > "$d/bad/ci.yml"
out=$(check_tree "$d/bad" 2>&1) && { echo "self-test failed: a job without timeout-minutes passed"; fails=1; }
case "$out" in *'job `b` has no timeout-minutes'*) ;; *) echo "self-test failed: the job without a timeout was not named: $out"; fails=1 ;; esac
case "$out" in *'job `a`'*) echo "self-test failed: a job with a timeout was named: $out"; fails=1 ;; esac
printf 'name: ci\non: push\njobs:\n site:\n runs-on: ubuntu-latest\n timeout-minutes: 360\n steps: []\n' > "$d/budget/ci.yml"
out=$(check_tree "$d/budget" 2>&1) && { echo "self-test failed: site at 360 minutes passed"; fails=1; }
case "$out" in *'has timeout-minutes 360, the budget is 15'*) ;; *) echo "self-test failed: the wrong budget was not named: $out"; fails=1 ;; esac
rm -rf "$d"
check_tree .github/workflows || { echo "self-test failed: the tree's workflows do not pass"; fails=1; }
[ "$fails" = 0 ] && echo "self-test passed: a job without timeout-minutes fails, a wrong budget fails, the tree's workflows pass (site 15, changes 10, pow 60, sims 45)"
exit $fails
fi
check_tree .github/workflows && echo "workflow-timeouts: every job in .github/workflows carries timeout-minutes (site 15, changes 10, pow 60, sims 45)"