Merge branch 'ci-steward' into ci-steward-2

.github/workflows/ci-red.yml

@@ -3,7 +3,7 @@
 # ci.yml never enters it (7 October 2026: the inline `red` job of ci.yml was conditioned on master and release-*, and
 # a feature branch would have waited for a merge of master before its reds were posted at all).
 #
-# One line per failed run (tools/ci/red-watch.mjs record, idempotent per run attempt) to /srv/ci-red/red.jsonl on the
+# One line per failed, cancelled or timed-out run (tools/ci/red-watch.mjs record, idempotent per run attempt) to /srv/ci-red/red.jsonl on the
 # box; the box's igneum-ci-red.timer posts each new line once to the hidden updates channel, naming the branch, the
 # commit, the red check and the pushing author. Runs on the box's own runner (not a GitHub-hosted machine: the billing
 # block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told). Never blocks a
@@ -16,7 +16,9 @@ on:
 jobs:
   red:
     name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file)
-    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
+    # failure, and since 7 October 2026 (17:2x UK) cancelled and timed_out too: a job that hangs into its timeout-minutes or a run
+    # someone cancels is a run that never answered, and a lane reads it like a red (tools/ci/red-watch.mjs names the kind)
+    if: ${{ github.event.workflow_run.conclusion == 'failure' || github.event.workflow_run.conclusion == 'cancelled' || github.event.workflow_run.conclusion == 'timed_out' }}
     # the label ci-red is on igneum-build-1 only (added through the runners API on 7 October 2026; the default of
     # RUNNER_LABELS in provision.sh carries it): the record file and the poster (igneum-ci-red.timer, the webhook file)
     # live on that box, and the pool label igneum-build-1 is shared with igneum-build-2 since the same day
@@ -35,6 +37,7 @@ jobs:
           RED_WATCH_RUN_ID: ${{ github.event.workflow_run.id }}
           RED_WATCH_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
           RED_WATCH_WORKFLOW: ${{ github.event.workflow_run.name }}
+          RED_WATCH_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
           RED_WATCH_BRANCH: ${{ github.event.workflow_run.head_branch }}
           RED_WATCH_SHA: ${{ github.event.workflow_run.head_sha }}
           RED_WATCH_EVENT: ${{ github.event.workflow_run.event }}

.github/workflows/ci.yml

@@ -36,6 +36,7 @@ jobs:
     # code=true (no `before` to compare from), as does any error reading the compare API: when in doubt, run.
     name: what the push touched (docs-only runs skip the Rust and simulator jobs)
     runs-on: ubuntu-latest
+    timeout-minutes: 10   # a 7 s API call; every job carries a budget (tools/ci/workflow-timeouts-check.sh)
     outputs:
       code: ${{ steps.classify.outputs.code }}
     steps:
@@ -62,6 +63,7 @@ jobs:
     needs: changes
     if: ${{ needs.changes.outputs.code == 'true' }}
     runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
+    timeout-minutes: 60   # the box's suite ran 45 s to 2 min 40 s on 7 October 2026; a hosted fallback compiles cold
     steps:
       - uses: actions/checkout@v4
       - name: toolchain
@@ -81,6 +83,7 @@ jobs:
     # queue read 22); a feature-branch code push runs the igneum-pow tests alone. tools/ci/sims-branch-check.sh holds this rule.
     if: ${{ needs.changes.outputs.code == 'true' && ((github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))) || (github.event_name == 'pull_request' && (github.base_ref == 'master' || startsWith(github.base_ref, 'release-')))) }}
     runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
+    timeout-minutes: 45   # two simulators under 120 s each by their own timeout, plus a hosted fallback's pip install
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -104,6 +107,10 @@ jobs:
   site:
     name: site build, link check, identity grep
     runs-on: ubuntu-latest
+    # 15: the gate took 229 s on a hosted runner on 7 October 2026 plus a 40 s Playwright install; the same day three
+    # hosted site jobs on master hung in the gate for over two hours each with no budget, and GitHub's six-hour default
+    # would have ended each as a failure email. A hung job is a red the watcher posts (ci-red.yml fires on timed_out).
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -118,4 +125,4 @@ jobs:
         run: bash tools/ci/pre-push.sh --ci
       - name: public stats API answers with the documented fields (the live site; master only, the endpoints exist there after the merge)
         if: github.ref == 'refs/heads/master'
-        run: node tools/ci/public-api-check.mjs https://igneum.network
+        run: bash tools/ci/retry-once.sh public-api node tools/ci/public-api-check.mjs https://igneum.network   # a live host: one retry before red

tools/ci/README.md

@@ -4,4 +4,9 @@
 |---|---|---|
 | no text overlaps (`overlap-check.mjs`) | A served page, or a miner or wallet screen (behind `IGNEUM_OVERLAP_APPS=1`), where a visible run of text is covered by another element (a pill over a caption, a label over a value, a card over its neighbour, text under the header at rest), clipped by an overflow-hidden ancestor, or past the viewport; a page that scrolls sideways. Five widths, light and dark, the home hero at rest and at each step. A fixture with one deliberate overlap of each kind must be flagged first (`--self-test`). Needs a headless Chromium: CI installs Playwright; the Mac ships the pages to build-2 (`infra/build-server/overlap-browser.sh`). | 7 October 2026: the hero's step pill sat on the caption's second line ("Two thirds of the weight sign. The checkpoint locks.") at every desktop width, found by the project lead on the live site |
 
+| master takes only CI-passed commits (`ci-state.mjs`, `merge-to-master.sh`, the hook's `master_ci_ok`) | A push to master whose commit, or whose merge's branch parent, has no green `ci` run on that exact sha (the runs API through gh: red, queued, none or gh unreachable all refuse); a merge onto a master whose last compiled run is red, unless declared the fix (`--fixes-master`). The merge tool pushes an unrun branch for a run and waits for a queued one with the clock. A feature-branch push prints the branch's previous red first (`--branch-red`). | 7 October 2026: era-vdf's tip 0e2d6b1c merged with no ci run; master's igneum-pow suite red from 16:31 UK under five docs-only green merges |
+| every workflow job carries timeout-minutes (`workflow-timeouts-check.sh`) | A job in .github/workflows without `timeout-minutes`, or a budget off its measured line (site 15, changes 10, pow 60, sims 45). | 7 October 2026: three hosted site jobs on master hung over two hours each in the tree gate; the six-hour default was the only stop |
+| a box or network check gets one retry (`retry-once.sh`) | Nothing by itself: wraps the box-locks check, the scene parity check and the live public API check so a first failure is printed and retried once; the second is the verdict. The checks keep their own skip line on a runner without the resource. | 7 October 2026 |
+| the red watcher fires on cancelled and timed-out runs too (`ci-red.yml`, `red-watch.mjs`) | The watcher's `if` missing any of failure, cancelled, timed_out, or the conclusion not handed to the record step (the self-test reads the workflow file); the line names the kind: CI red, CI cancelled, CI timed out. | 7 October 2026 |
+
 | kill by exact command or pid file (owed as a check) | 6 October 2026, 21:09Z: a Mac-side `pkill -f <log file name>` matched nothing (the log name was a redirect, not part of the command line), the roll-everything script lived on and wiped a box it had been told to hold. Rule: a job is stopped by its pid file (`tools/fleet/fleet-bg.sh start|stop <name>`) or by a pattern anchored on its exact command line (`^python3 -u /root/fleet/in/box-prover.py`), never by a word that may or may not appear in it. The check that flags a `pkill -f`/`pgrep -f` whose literal is a path or a name that never starts a command line is owed to the CI lane |

tools/ci/ci-state.mjs

@@ -0,0 +1,152 @@
+#!/usr/bin/env node
+// What CI says about a commit or a branch, read from the runs API through gh (the Mac's gh login; the repository is
+// igneum-network/igneum unless IGNEUM_REPO says otherwise). The merge tool and the pre-push hook read these lines, so a
+// merge lands on master only when the branch's own ci run is green on the exact commit being merged (standing rule,
+// 7 October 2026, 17:2x UK: era-vdf's tip 0e2d6b1c was merged with no ci run at all and master's igneum-pow suite went
+// red for 40 minutes, hidden by docs-only merges whose runs skip the compile job).
+// Node 22, standard library only; gh does the HTTP.
+//
+//   node tools/ci/ci-state.mjs <sha>                  one line: "<state> <run id> <url> <detail>"
+//                                                      state: success | failure | cancelled | timed_out | pending | none | unknown
+//                                                      (the NEWEST ci run on that exact commit; a re-run replaces the first attempt)
+//   node tools/ci/ci-state.mjs --master-code          the verdict of master's newest completed ci run whose compile job (igneum-pow)
+//                                                      RAN: a docs-only push skips it and its green hides a red suite
+//   node tools/ci/ci-state.mjs --branch-red <branch>  prints "previous CI red on <branch>: ..." when the branch's newest completed ci
+//                                                      run is red and no newer run is green; prints nothing otherwise; exit 0 always
+//   node tools/ci/ci-state.mjs --self-test            a fake gh on PATH answers every case: success, failure, pending, none, newest
+//                                                      wins, the docs-only skip walked past, the branch line, a gh error = unknown
+//
+// Exit 0 for every answered query (callers read the first word); 2 on usage; the self-test exits 1 on a failure.
+import { spawnSync } from 'node:child_process';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+
+const REPO = process.env.IGNEUM_REPO || 'igneum-network/igneum';
+const FIELDS = 'databaseId,status,conclusion,headSha,url,createdAt,event';
+const RED = new Set(['failure', 'cancelled', 'timed_out', 'startup_failure', 'action_required']);
+
+function gh(args) {
+  const env = { ...process.env, PATH: `${process.env.PATH || ''}:/opt/homebrew/bin:/usr/local/bin` };
+  const r = spawnSync('gh', args, { encoding: 'utf8', env, timeout: 60000 });
+  if (r.error) throw new Error(`gh: ${r.error.message}`);
+  if (r.status !== 0) throw new Error(`gh ${args.slice(0, 2).join(' ')}: exit ${r.status}: ${(r.stderr || '').trim().slice(0, 200)}`);
+  return JSON.parse(r.stdout || 'null');
+}
+function fullSha(sha) {
+  if (/^[0-9a-f]{40}$/.test(sha)) return sha;   // gh's --commit filter matches the full sha only (an abbreviation answers nothing)
+  const r = spawnSync('git', ['rev-parse', '--verify', `${sha}^{commit}`], { encoding: 'utf8' });
+  if (r.status !== 0) throw new Error(`${sha} is not a commit here`);
+  return r.stdout.trim();
+}
+export const newest = (runs) => [...(runs || [])].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt))[0];
+export function verdictOf(run) {
+  if (!run) return 'none';
+  if (run.status !== 'completed') return 'pending';
+  return run.conclusion || 'pending';
+}
+export function firstRed(jobs) {
+  for (const j of jobs || []) {
+    if (j.conclusion === 'success' || j.conclusion === 'skipped' || j.conclusion == null) continue;
+    const s = (j.steps || []).find((x) => x.conclusion && x.conclusion !== 'success' && x.conclusion !== 'skipped');
+    return `${j.name.replace(/,.*$/, '')} at "${s ? s.name : '(no step)'}"`;
+  }
+  return '';
+}
+const london = (iso) => new Intl.DateTimeFormat('en-GB', { timeZone: 'Europe/London', hour: '2-digit', minute: '2-digit', hour12: false }).format(new Date(iso)) + ' UK';
+const runsForSha = (sha) => gh(['run', 'list', '--repo', REPO, '--workflow', 'ci', '--commit', sha, '--limit', '10', '--json', FIELDS]) || [];
+const runsForBranch = (branch, limit = 12) => gh(['run', 'list', '--repo', REPO, '--workflow', 'ci', '--branch', branch, '--limit', String(limit), '--json', FIELDS]) || [];
+const jobsOf = (id) => (gh(['run', 'view', String(id), '--repo', REPO, '--json', 'jobs']) || {}).jobs || [];
+
+export function stateOfSha(sha) {
+  const run = newest(runsForSha(fullSha(sha)));
+  const state = verdictOf(run);
+  if (!run) return `none - - no ci run on ${sha.slice(0, 8)} yet`;
+  let detail = state === 'pending' ? `${run.status} since ${london(run.createdAt)}` : `${run.event} run at ${london(run.createdAt)}`;
+  if (RED.has(state)) { const red = firstRed(jobsOf(run.databaseId)); if (red) detail += `: ${red}`; }
+  return `${state} ${run.databaseId} ${run.url} ${detail}`;
+}
+export function masterCodeState() {
+  const runs = [...runsForBranch('master', 12)].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt));
+  for (const run of runs) {
+    if (run.status !== 'completed') continue;
+    const jobs = jobsOf(run.databaseId);
+    const pow = jobs.find((j) => /^igneum-pow/.test(j.name));
+    if (!pow || pow.conclusion === 'skipped') continue;   // a docs-only push: its green says nothing about the suite
+    const red = RED.has(run.conclusion) ? firstRed(jobs) : '';
+    return `${run.conclusion} ${run.databaseId} ${run.url} master @${run.headSha.slice(0, 8)} at ${london(run.createdAt)}, compile job ${pow.conclusion}${red ? `: ${red}` : ''}`;
+  }
+  return 'none - - no completed master ci run with the compile job among the last 12';
+}
+export function branchRedLine(branch) {
+  const runs = [...runsForBranch(branch, 12)].sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt));
+  const done = runs.find((r) => r.status === 'completed');
+  if (!done || !RED.has(done.conclusion)) return '';
+  const red = firstRed(jobsOf(done.databaseId));
+  return `previous CI red on ${branch}: @${done.headSha.slice(0, 7)} ${done.conclusion} at ${london(done.createdAt)}${red ? `: ${red}` : ''} ${done.url} (no newer green run on this branch; this push gets its own run, read it)`;
+}
+
+async function selfTest() {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-state-'));
+  const fake = path.join(dir, 'gh');
+  // the fake gh answers `run list` from list-<commit or branch>.json and `run view <id>` from view-<id>.json; FAKE_GH_FAIL=1 fails
+  fs.writeFileSync(fake, `#!/usr/bin/env bash
+[ "\${FAKE_GH_FAIL:-0}" = 1 ] && { echo "HTTP 502 from the fake" >&2; exit 1; }
+key=""; prev=""
+for a in "$@"; do case "$prev" in --commit|--branch) key="$a" ;; esac; prev="$a"; done
+case "$1 $2" in
+  "run list") f="$FAKE_GH_DIR/list-$key.json"; [ -f "$f" ] && cat "$f" || echo '[]' ;;
+  "run view") f="$FAKE_GH_DIR/view-$3.json"; [ -f "$f" ] && cat "$f" || echo '{"jobs":[]}' ;;
+  *) echo "fake gh: unknown $*" >&2; exit 1 ;;
+esac
+`, { mode: 0o755 });
+  const sha = (c) => c.repeat(40);
+  const run = (id, status, conclusion, created, headSha = sha('a')) => ({ databaseId: id, status, conclusion, headSha, url: `https://github.com/x/y/actions/runs/${id}`, createdAt: created, event: 'push' });
+  const w = (name, obj) => fs.writeFileSync(path.join(dir, name), JSON.stringify(obj));
+  w(`list-${sha('a')}.json`, [run(1, 'completed', 'success', '2026-10-07T15:00:00Z')]);
+  w(`list-${sha('b')}.json`, [run(2, 'completed', 'failure', '2026-10-07T15:00:00Z', sha('b'))]);
+  w('view-2.json', { jobs: [{ name: 'site build, link check', conclusion: 'success', steps: [] }, { name: 'igneum-pow tests, igneum-census build', conclusion: 'failure', steps: [{ name: 'toolchain', conclusion: 'success' }, { name: 'igneum-pow tests (release)', conclusion: 'failure' }] }] });
+  w(`list-${sha('c')}.json`, [run(3, 'in_progress', null, '2026-10-07T15:00:00Z', sha('c'))]);
+  w(`list-${sha('d')}.json`, [run(4, 'completed', 'failure', '2026-10-07T14:00:00Z', sha('d')), run(5, 'completed', 'success', '2026-10-07T15:00:00Z', sha('d'))]);   // the re-run wins
+  // master: the newest run is a docs-only green (compile job skipped); the one before it ran the compile job and is red
+  w('list-master.json', [run(10, 'completed', 'success', '2026-10-07T16:00:00Z', sha('1')), run(11, 'completed', 'failure', '2026-10-07T15:30:00Z', sha('2')), run(12, 'completed', 'success', '2026-10-07T15:00:00Z', sha('3'))]);
+  w('view-10.json', { jobs: [{ name: 'igneum-pow tests, igneum-census build', conclusion: 'skipped', steps: [] }, { name: 'site build', conclusion: 'success', steps: [] }] });
+  w('view-11.json', { jobs: [{ name: 'igneum-pow tests, igneum-census build', conclusion: 'failure', steps: [{ name: 'igneum-pow tests (release)', conclusion: 'failure' }] }] });
+  // branches: x red newest; y green newest; z pending newest over a red
+  w('list-x.json', [run(20, 'completed', 'failure', '2026-10-07T15:00:00Z', sha('e'))]);
+  w('view-20.json', { jobs: [{ name: 'site build, link check, identity grep', conclusion: 'failure', steps: [{ name: 'the tree gate', conclusion: 'failure' }] }] });
+  w('list-y.json', [run(21, 'completed', 'success', '2026-10-07T15:00:00Z'), run(22, 'completed', 'failure', '2026-10-07T14:00:00Z')]);
+  w('list-z.json', [run(23, 'queued', null, '2026-10-07T15:10:00Z'), run(24, 'completed', 'cancelled', '2026-10-07T15:00:00Z', sha('f'))]);
+  const me = new URL(import.meta.url).pathname;
+  const ask = (args, extraEnv = {}) => {
+    const r = spawnSync(process.execPath, [me, ...args], { encoding: 'utf8', env: { ...process.env, PATH: `${dir}:${process.env.PATH}`, FAKE_GH_DIR: dir, ...extraEnv } });
+    return { out: (r.stdout || '').trim(), code: r.status, err: (r.stderr || '').trim() };
+  };
+  const fails = [];
+  const expect = (name, got, re) => { if (!re.test(got.out) || got.code !== 0) fails.push(`${name}: got exit ${got.code} "${got.out}" ${got.err}`); };
+  expect('success', ask([sha('a')]), /^success 1 https:\/\/github.com\/x\/y\/actions\/runs\/1 push run at /);
+  expect('failure names the red step', ask([sha('b')]), /^failure 2 \S+ push run at \d\d:\d\d UK: igneum-pow tests at "igneum-pow tests \(release\)"$/);
+  expect('pending', ask([sha('c')]), /^pending 3 \S+ in_progress since /);
+  expect('newest wins', ask([sha('d')]), /^success 5 /);
+  expect('none', ask([sha('9')]), /^none - - no ci run on 99999999 yet$/);
+  expect('gh error is unknown', ask([sha('a')], { FAKE_GH_FAIL: '1' }), /^unknown - - gh run list: exit 1: HTTP 502 from the fake/);
+  expect('master-code walks past the docs-only green', ask(['--master-code']), /^failure 11 \S+ master @22222222 at \d\d:\d\d UK, compile job failure: igneum-pow tests at "igneum-pow tests \(release\)"$/);
+  expect('branch red line', ask(['--branch-red', 'x']), /^previous CI red on x: @eeeeeee failure at \d\d:\d\d UK: site build at "the tree gate" https:\/\/github.com\/x\/y\/actions\/runs\/20 \(no newer green/);
+  const y = ask(['--branch-red', 'y']); if (y.out !== '' || y.code !== 0) fails.push(`branch green prints nothing: "${y.out}" exit ${y.code}`);
+  expect('branch pending over a cancelled run still names the red', ask(['--branch-red', 'z']), /^previous CI red on z: @fffffff cancelled at /);
+  const noArg = ask([]); if (noArg.code !== 2) fails.push(`usage: exit ${noArg.code}`);
+  fs.rmSync(dir, { recursive: true, force: true });
+  if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); }
+  console.log('self-test passed: a commit answers success, failure (with the red step), pending, none; the newest run wins; a gh error is unknown; --master-code walks past a docs-only green to the run that compiled; --branch-red names the newest completed red and stays silent on green');
+}
+
+const args = process.argv.slice(2);
+try {
+  if (args[0] === '--self-test') await selfTest();
+  else if (args[0] === '--master-code') console.log(masterCodeState());
+  else if (args[0] === '--branch-red') { if (!args[1]) { console.error('usage: ci-state.mjs --branch-red <branch>'); process.exit(2); } console.log(branchRedLine(args[1])); }
+  else if (args[0] && !args[0].startsWith('-')) console.log(stateOfSha(args[0]));
+  else { console.error('usage: tools/ci/ci-state.mjs <sha> | --master-code | --branch-red <branch> | --self-test'); process.exit(2); }
+} catch (e) {
+  console.log(`unknown - - ${e.message.replace(/\s+/g, ' ').slice(0, 300)}`);
+}

tools/ci/merge-to-master.sh

@@ -5,26 +5,114 @@
 # two-parent merge of the branch onto the exact remote tip, which the hook lets through on the light gate (20 s) while CI runs
 # the full gate on landing. Retries while master moves. Never force; never from a dirty branch.
 #
-#   tools/ci/merge-to-master.sh [<branch>] [--tries N]     default: the current branch, 6 tries
+# The CI rule (standing rule, 7 October 2026, 17:2x UK): a merge lands only when the branch's OWN ci run is green on the exact
+# commit being merged, read from the runs API (tools/ci/ci-state.mjs), not the local stamp alone. No run yet: the branch is pushed
+# so CI runs it. A queued or running run: this tool waits, printing the UK clock, up to --ci-wait minutes (default 25). A red run:
+# refused with the run's red check; fix the branch and push a new commit. And master itself must not be red: the newest master run
+# that compiled (docs-only runs skip the compile job and their green says nothing) must be success, or the merge is refused unless
+# it is the fix (--fixes-master). Record: era-vdf's tip 0e2d6b1c was merged with no ci run; master's igneum-pow suite was red from
+# 16:31 UK and five docs-only merges landed green over it. The pre-push hook holds the same rule (pre-push.sh master_ci_ok).
+#
+#   tools/ci/merge-to-master.sh [<branch>] [--tries N] [--ci-wait MIN] [--fixes-master]     default: the current branch, 6 tries
+#   tools/ci/merge-to-master.sh --self-test     the CI verdicts, with a fake ci-state: green goes, red refuses, master red refuses
+#                                               unless --fixes-master, none pushes the branch, pending waits then goes
 set -euo pipefail
 ROOT=$(git rev-parse --show-toplevel); cd "$ROOT"
-BRANCH="$(git rev-parse --abbrev-ref HEAD)"; TRIES=6
+BRANCH="$(git rev-parse --abbrev-ref HEAD)"; TRIES=6; CI_WAIT_MIN="${CI_WAIT_MIN:-25}"; FIXES_MASTER=0; SELF_TEST=0
-while [ $# -gt 0 ]; do case "$1" in --tries) TRIES="$2"; shift 2 ;; -*) echo "unknown option $1" >&2; exit 2 ;; *) BRANCH="$1"; shift ;; esac; done
+while [ $# -gt 0 ]; do case "$1" in --tries) TRIES="$2"; shift 2 ;; --ci-wait) CI_WAIT_MIN="$2"; shift 2 ;; --fixes-master) FIXES_MASTER=1; shift ;; --self-test) SELF_TEST=1; shift ;; -*) echo "unknown option $1" >&2; exit 2 ;; *) BRANCH="$1"; shift ;; esac; done
+CI_STATE="${CI_STATE_CMD:-node tools/ci/ci-state.mjs}"   # the self-test swaps in a fake
+clock() { TZ=Europe/London date '+%H:%M %Z'; }
+
+# ci_gate <sha> <branch> <push-cmd...>: 0 when the branch's own ci run on <sha> is green; pushes the branch when there is no run;
+# waits on a queued or running run; refuses (1) a red or unknown one. master_gate: refuses (1) when master's last compiled run is red.
+ci_gate() {
+  local sha="$1" branch="$2"; shift 2; local line state deadline pushed=0
+  deadline=$(( $(date +%s) + CI_WAIT_MIN * 60 ))
+  while :; do
+    line=$($CI_STATE "$sha" 2>&1); state="${line%% *}"
+    case "$state" in
+      success) echo "merge-to-master: ci on ${sha:0:8} is green: $line"; return 0 ;;
+      none) if [ "$pushed" = 1 ]; then echo "merge-to-master: waiting for a ci run to appear on ${sha:0:8} ($(clock))"; else echo "merge-to-master: no ci run on ${sha:0:8} yet; pushing $branch so CI runs it ($(clock))"; "$@" || { echo "merge-to-master: the branch push failed; CI cannot run ${sha:0:8}" >&2; return 1; }; pushed=1; fi ;;
+      pending) echo "merge-to-master: ci on ${sha:0:8} is ${line#* * * }; waiting ($(clock))" ;;
+      *) echo "merge-to-master: REFUSED. ci on ${sha:0:8} is $line" >&2; echo "  A branch whose own ci run is not green never merges (CI red is stop-the-line). Fix the branch, push a new commit, and run this again." >&2; return 1 ;;
+    esac
+    [ "$(date +%s)" -lt "$deadline" ] || { echo "merge-to-master: REFUSED. ci on ${sha:0:8} gave no verdict in $CI_WAIT_MIN minutes ($line); run it again when the queue drains (gh run list --branch $branch --limit 3)" >&2; return 1; }
+    sleep "${CI_POLL_SECS:-30}"
+  done
+}
+master_gate() {
+  local line state; line=$($CI_STATE --master-code 2>&1); state="${line%% *}"
+  case "$state" in
+    success|none|pending) echo "merge-to-master: master's last compiled run: $line"; return 0 ;;
+    *) if [ "$FIXES_MASTER" = 1 ]; then echo "merge-to-master: master's last compiled run is $state and this merge is declared the fix (--fixes-master): $line"; return 0; fi
+       echo "merge-to-master: REFUSED. master is red: $line" >&2; echo "  CI red is stop-the-line: the owner fixes or reverts first; only the fix merges, with --fixes-master." >&2; return 1 ;;
+  esac
+}
+
+if [ "$SELF_TEST" = 1 ]; then
+  fails=0; d=$(mktemp -d); fake="$d/ci-state.sh"
+  # the fake answers from $d/answer-<sha> (one line per call, consumed top to bottom; the last line repeats) and $d/answer-master
+  cat > "$fake" <<'FAKE'
+#!/usr/bin/env bash
+d=$(dirname "$0"); key="$1"; [ "$key" = --master-code ] && key=master
+f="$d/answer-$key"; [ -f "$f" ] || { echo "none - - no ci run on $key yet"; exit 0; }
+n=$(wc -l < "$f" | tr -d ' '); i=$(cat "$d/i-$key" 2>/dev/null || echo 1); [ "$i" -gt "$n" ] && i=$n
+sed -n "${i}p" "$f"; echo $((i + 1)) > "$d/i-$key"
+FAKE
+  chmod +x "$fake"; export CI_STATE_CMD="$fake" CI_POLL_SECS=0 CI_WAIT_MIN=1
+  CI_STATE="$fake"
+  printf 'success 1 u push run at 16:00 UK
+' > "$d/answer-green"
+  ci_gate green b false >/dev/null || { echo "self-test failed: a green branch run was refused"; fails=1; }
+  printf 'failure 2 u push run: igneum-pow tests at "igneum-pow tests (release)"
+' > "$d/answer-red"
+  ci_gate red b false >/dev/null 2>&1 && { echo "self-test failed: a red branch run merged"; fails=1; }
+  printf 'unknown - - gh: exit 1
+' > "$d/answer-unk"
+  ci_gate unk b false >/dev/null 2>&1 && { echo "self-test failed: an unknown verdict merged"; fails=1; }
+  printf 'pending 3 u queued since 16:00 UK
+pending 3 u in_progress since 16:00 UK
+success 3 u push run at 16:05 UK
+' > "$d/answer-wait"
+  out=$(ci_gate wait b false 2>&1) || { echo "self-test failed: a pending run that turned green was refused: $out"; fails=1; }
+  [ "$(printf '%s
+' "$out" | grep -c 'waiting')" = 2 ] || { echo "self-test failed: the wait did not print the clock twice: $out"; fails=1; }
+  # no run: the branch is pushed (the push command runs once), then the run appears and goes green
+  printf 'none - - no ci run on none yet
+success 4 u push run
+' > "$d/answer-none"; : > "$d/pushes"
+  ci_gate none b bash -c "echo pushed >> '$d/pushes'" >/dev/null || { echo "self-test failed: an unrun branch was refused instead of pushed"; fails=1; }
+  [ "$(wc -l < "$d/pushes" | tr -d ' ')" = 1 ] || { echo "self-test failed: the branch was pushed $(wc -l < "$d/pushes") times"; fails=1; }
+  # master red: refused; with --fixes-master: goes; master green or unknown-none: goes
+  printf 'failure 5 u master @1210158d, compile job failure
+' > "$d/answer-master"
+  FIXES_MASTER=0 master_gate >/dev/null 2>&1 && { echo "self-test failed: a merge onto a red master was let through"; fails=1; }
+  rm -f "$d/i-master"; FIXES_MASTER=1 master_gate >/dev/null || { echo "self-test failed: the declared fix was refused on a red master"; fails=1; }
+  printf 'success 6 u master @e5171a32, compile job success
+' > "$d/answer-master"; rm -f "$d/i-master"
+  FIXES_MASTER=0 master_gate >/dev/null || { echo "self-test failed: a green master refused a merge"; fails=1; }
+  rm -rf "$d"
+  [ "$fails" = 0 ] && echo "self-test passed: a green branch run merges; a red or unknown one is refused; a queued run is waited for with the clock; an unrun branch is pushed once and then waited for; a red master refuses every merge but the declared fix"
+  exit $fails
+fi
 [ -z "$(git status --porcelain --untracked-files=no)" ] || { echo "merge-to-master: the tree has uncommitted tracked changes; commit first" >&2; exit 1; }
 SHA=$(git rev-parse "$BRANCH"); G=$(cd "$(git rev-parse --git-common-dir)" && pwd -P)
 if [ ! -f "$G/igneum-gate-green/$SHA" ]; then
-  echo "merge-to-master: no green stamp for ${SHA:0:8}; running the full gate on the branch first"
+  echo "merge-to-master: no green stamp for ${SHA:0:8}; running the full gate on the branch first (then CI's own verdict on it is read)"
   bash tools/ci/pre-push.sh || exit 1
   [ -f "$G/igneum-gate-green/$SHA" ] || { echo "merge-to-master: the gate was green but no stamp was written (dirty tree?)" >&2; exit 1; }
 fi
 AUTHOR=(-c user.name=igneum-labs -c user.email=337424239+[removed])
+# the CI rule: the branch's own run on this exact commit must be green (pushed for a run when there is none, waited for when queued), and master must not be red
+ci_gate "$SHA" "$BRANCH" git push -q origin "$SHA:refs/heads/$BRANCH" || exit 1
+master_gate || exit 1
 for i in $(seq 1 "$TRIES"); do
   git fetch -q origin master; TIP=$(git rev-parse origin/master)
   if git merge-base --is-ancestor "$SHA" "$TIP"; then echo "merge-to-master: ${SHA:0:8} is already on origin/master $(git log -1 --format=%h origin/master)"; exit 0; fi
   W=$(mktemp -d "${TMPDIR:-/tmp}/merge-to-master.XXXXXX"); rmdir "$W"
   git worktree add -q --detach "$W" "$TIP"
   if ( cd "$W" && git "${AUTHOR[@]}" merge -q --no-ff -m "Merge $BRANCH ${SHA:0:8} into master (gate: green on ${SHA:0:8}, recorded by tools/ci/pre-push.sh; the full gate runs in CI on this merge)" "$SHA" ); then
-    if ( cd "$W" && git push -q origin HEAD:master ); then
+    if ( cd "$W" && git push -q origin HEAD:master ); then   # the hook asks ci-state about ${SHA:0:8} once more on this push
       git worktree remove --force "$W"; git fetch -q origin master
       echo "merge-to-master: pushed on try $i: origin/master $(git log -1 --format='%h %ci' origin/master) $(TZ=Europe/London date '+%H:%M %Z')"; exit 0
     fi

tools/ci/pre-push.sh

@@ -26,6 +26,7 @@ cd "$(git rev-parse --show-toplevel)" || exit 1
 # and fire this hook again inside the fixture: the first master push through the gate died that way (6 October 2026).
 unset GIT_DIR GIT_WORK_TREE GIT_INDEX_FILE GIT_PREFIX GIT_COMMON_DIR GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_QUARANTINE_PATH GIT_PUSH_OPTION_COUNT
 MODE="${1:-local}"; MODE="${MODE#--}"
+GATE_ROOT="$(pwd -P)"   # the readers below are called from fixture repositories in the self-test, so by absolute path
 RED=0; N=0; LOG="$(mktemp)"; trap 'rm -rf "$LOG" "${SITE_TMP:-}"' EXIT
 T0=$(date +%s)
 
@@ -52,12 +53,19 @@ site_build() {
   (cd "$SITE_TMP/site" && SITE_DOWNLOADS_OFFLINE=1 node build.mjs)
 }
 
+wall_clock() {   # <secs> <command...>: under GNU timeout where it exists (the runners); the Mac has none, and the job's timeout-minutes is the stop there.
+  # No array here: an empty array expanded under set -u is "unbound variable" on the Mac's bash 3.2 and ended the gate with no RED line (17:3x UK, 7 October 2026).
+  local secs="$1"; shift
+  if command -v timeout >/dev/null 2>&1; then timeout "$secs" "$@"; else "$@"; fi
+}
 overlap_sweep() {
   # tools/ci/overlap-check.mjs: the known-failed fixture first, then the built site (the gate's temporary copy locally, the tree in
   # CI). A browser is needed: CI installs Playwright in the workflow; a machine without one ships the pages to a build box
   # (nothing heavy on the Mac). IGNEUM_OVERLAP_APPS=1 adds the miner and wallet UIs through their mocks (slower, the box).
   local dir="site"; [ "$MODE" = ci ] || dir="$SITE_TMP/site"
-  if [ "${IGNEUM_OVERLAP_APPS:-0}" = 1 ]; then node tools/ci/overlap-check.mjs --self-test --site "$dir" --apps .; else node tools/ci/overlap-check.mjs --self-test --site "$dir"; fi
+  # a wall clock of 10 minutes where GNU timeout exists (the runners; the Mac ships the sweep to a box): the sweep took 192 s on a hosted
+  # runner on 7 October 2026 and three master jobs hung in this step for over two hours each the same afternoon
+  if [ "${IGNEUM_OVERLAP_APPS:-0}" = 1 ]; then wall_clock 600 node tools/ci/overlap-check.mjs --self-test --site "$dir" --apps .; else wall_clock 600 node tools/ci/overlap-check.mjs --self-test --site "$dir"; fi
 }
 
 structural_checks() {
@@ -103,7 +111,7 @@ tree_checks() {
   run "long-running tools keep their body in one parsed block (the edited-while-running class)" bash -c 'bash tools/ci/whole-body-check.sh --self-test && bash tools/ci/whole-body-check.sh'
   run "build-remote without a priority flag bounds suites and benches (nice 10, 32 cores); a gate runs unbounded" bash tools/ci/build-kind-default-check.sh
   run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh
-  run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh)" bash tools/ci/box-locks-check.sh $( [ "$MODE" = ci ] && echo --ci )
+  run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh; skipped on a runner with no box; one retry)" bash tools/ci/retry-once.sh box-locks bash tools/ci/box-locks-check.sh $( [ "$MODE" = ci ] && echo --ci )
   run "the simulators job runs on master and release-* pushes and pull requests into them only" bash tools/ci/sims-branch-check.sh
   run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh'
   run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh'
@@ -116,7 +124,7 @@ tree_checks() {
   run "chain scene: the site's and the app's copies are scene/ byte for byte, the palette tokens live once (self-test, then the tree)" bash -c 'node tools/scene/sync.mjs --self-test && node tools/scene/sync.mjs --check'
   run "chain scene: a push paints with the document hidden and no animation frame (the blank /live of 7 Oct 2026; known-failed first)" node tools/scene/paint-test.cjs
   run "chain scene: the live feed contract (the recorded reply validates; a rewritten miner, a float now, a stray key refused)" node --test tools/scene/feed-contract.test.mjs
-  run "chain scene parity: one recorded feed through the home fold, /live and the app's Inspect view on build-2, three frames each pixel-equal apart from the app's own-key overlay (a changed token fails first; skipped with no box and no Playwright)" bash tools/scene/parity-remote.sh
+  run "chain scene parity: one recorded feed through the home fold, /live and the app's Inspect view on build-2, three frames each pixel-equal apart from the app's own-key overlay (a changed token fails first; skipped with no box and no Playwright; one retry)" bash tools/ci/retry-once.sh scene-parity bash tools/scene/parity-remote.sh
   run "no text overlaps: every served page at 390 to 1600 px, light and dark, the hero at each step (self-test first; IGNEUM_OVERLAP_APPS=1 adds the miner and wallet UIs)" overlap_sweep
   run "explorer, emission and public stats unit tests"                         node --test site/lib/explorer.test.mjs site/lib/emission.test.mjs site/lib/money.test.mjs site/api/public-stats.test.mjs
   run "ship tool self-test"                                                    node tools/ship-app.mjs --self-test
@@ -127,6 +135,9 @@ tree_checks() {
   run "hash-origin report: a known-finished day and a known-failed day"         node --test tools/observer/hash-origin.test.mjs
   run "harness summaries never carry a raw 64-hex key (the writer's own redaction and check)" node infra/fast-time/lib/redact-keys.mjs --self-test
   run "docs-only pushes skip the compile-or-compute CI jobs (the changes job's classifier)" bash tools/ci/docs-only-check.sh --self-test
+  run "every workflow job carries timeout-minutes (site 15, changes 10, pow 60, sims 45; the hung-job class of 7 October 2026)" bash tools/ci/workflow-timeouts-check.sh --self-test
+  run "a box or network check gets one retry before it is red (retry-once self-test)" bash tools/ci/retry-once.sh --self-test
+  run "CI state reader: a commit's newest run, master's last compiled run, a branch's last red (fake gh; the merge rule's reader)" node tools/ci/ci-state.mjs --self-test
 }
 
 gated_refs() {
@@ -162,6 +173,27 @@ deferred_merge() {   # <local sha> <remote sha> [repo dir] -> "defer <branch sha
   age=$(( $(date +%s) - $(stat -f %m "$f" 2>/dev/null || stat -c %Y "$f") )); [ "$age" -le 43200 ] || { echo "full the stamp for ${p2:0:8} is $age s old"; return; }
   echo "defer $p2"
 }
+# Master takes only what CI has already passed (standing rule, 7 October 2026, 17:2x UK: era-vdf's tip 0e2d6b1c was merged with
+# no ci run at all and master's igneum-pow suite stayed red for 40 minutes under docs-only merges). For a push to master the hook
+# asks tools/ci/ci-state.mjs: a two-parent merge needs a green run on its SECOND parent (the branch's own run on the exact
+# commit), a plain commit needs a green run on itself (a fast-forward of a branch CI passed); anything else is refused with the
+# run's state, and the lane uses tools/ci/merge-to-master.sh, which waits for a queued run. gh unreachable = refused (unknown).
+# release-* branches keep the full local gate alone (the shipper's cuts carry their own box suite line).
+master_ci_ok() {   # <local sha> <remote sha> -> 0 and a line, or 1 and the reason
+  local lsha="$1" rsha="$2" parents p2 want line state
+  parents=$(git rev-list --parents -n 1 "$lsha" 2>/dev/null | cut -d' ' -f2-); set -- $parents
+  if [ -n "${2:-}" ] && [ -z "${3:-}" ] && [ "$1" = "$rsha" ]; then want="$2"; else want="$lsha"; fi
+  line=$(node "$GATE_ROOT/tools/ci/ci-state.mjs" "$want" 2>&1); state="${line%% *}"
+  if [ "$state" = success ]; then echo "  master takes ${want:0:8}: ci $line"; return 0; fi
+  echo "pre-push gate: REFUSED. master takes only a commit whose own ci run is green on that exact commit; ${want:0:8} is: $line" >&2
+  echo "  Use tools/ci/merge-to-master.sh (it pushes the branch for a run when there is none, waits for a queued run and refuses a red)." >&2
+  return 1
+}
+branch_red_line() {   # <branch>: the branch's newest completed ci run, when red, printed before the light gate (nothing on green or no gh)
+  local line; line=$(node "$GATE_ROOT/tools/ci/ci-state.mjs" --branch-red "$1" 2>/dev/null) || return 0
+  case "$line" in previous\ CI\ red*) echo "  $line" ;; esac
+  return 0
+}
 finish() {
   local what="$1" secs=$(( $(date +%s) - T0 ))
   if [ "$RED" = 0 ]; then echo "pre-push gate ($what): GREEN, $N checks in ${secs}s"; [ "${STAMP:-0}" = 1 ] && stamp_green; exit 0; fi
@@ -205,7 +237,31 @@ case "$MODE" in
     MODE=ci GITHUB_ACTIONS= site_in_place && { echo "self-test failed: --ci outside GitHub Actions chose the in-place build"; fails=1; }
     MODE=ci GITHUB_ACTIONS=true site_in_place || { echo "self-test failed: --ci inside GitHub Actions did not choose the in-place build"; fails=1; }
     declare -f tree_checks | grep -q 'never_push_checks' || { echo "self-test failed: the full gate does not run the never-push checks"; fails=1; }
-    [ "$fails" = 0 ] && echo "self-test passed: a failing check is RED and fails the gate, a passing one is ok; master and release-* select the full gate, other refs the light one; a merge of a green-stamped branch onto the remote tip defers to CI, every other shape takes the full gate; a --ci site build outside GitHub Actions leaves the tree unchanged (structural checks, no-secrets, identity grep)"
+    # the overlap sweep's wall clock runs the command with GNU timeout where it exists and plainly where it does not (bash 3.2 under set -u included)
+    [ "$(wall_clock 5 /bin/echo clocked 2>&1)" = clocked ] || { echo "self-test failed: wall_clock did not run its command"; fails=1; }
+    [ "$(PATH=/nonexistent wall_clock 5 /bin/echo plain 2>&1)" = plain ] || { echo "self-test failed: wall_clock without a timeout binary did not run its command plainly"; fails=1; }
+    grep -q 'wall_clock 600 node tools/ci/overlap-check.mjs' "$0" || { echo "self-test failed: the overlap sweep does not run under the wall clock"; fails=1; }
+    # master takes only CI-passed commits: a merge asks about its second parent, a plain commit about itself; red, pending, none and unknown refuse
+    grep -qE 'master_ci_ok "\$lsha" "\$rsha" \|\| exit 1' "$0" || { echo "self-test failed: the hook does not ask ci-state before a push to master"; fails=1; }
+    grep -qE 'branch_red_line "\$\{rref#refs/heads/\}"' "$0" || { echo "self-test failed: the feature-branch hook does not print the branch's previous red"; fails=1; }
+    fx=$(mktemp -d); ( cd "$fx" && git init -q -b master . && git -c user.name=t -c user.email=t@t commit -q --allow-empty -m a ) 2>/dev/null
+    A=$(git -C "$fx" rev-parse HEAD); git -C "$fx" checkout -q -b b; git -C "$fx" -c user.name=t -c user.email=t@t commit -q --allow-empty -m b; B=$(git -C "$fx" rev-parse HEAD)
+    git -C "$fx" checkout -q master; git -C "$fx" -c user.name=t -c user.email=t@t merge -q --no-ff -m "merge b" b; M=$(git -C "$fx" rev-parse HEAD)
+    fakebin=$(mktemp -d)
+    cat > "$fakebin/gh" <<FAKEGH
+#!/usr/bin/env bash
+# the fake gh of the gate's self-test: a green run on the branch commit, a queued run on the merge commit, nothing elsewhere
+prev=""; key=""; for a in "\$@"; do [ "\$prev" = --commit ] && key="\$a"; prev="\$a"; done
+row() { printf '[{"databaseId":%s,"status":"%s","conclusion":%s,"headSha":"%s","url":"u","createdAt":"2026-10-07T15:00:00Z","event":"push"}]\\n' "\$1" "\$2" "\$3" "\$key"; }
+case "\$key" in $B) row 1 completed '"success"' ;; $M) row 2 queued null ;; *) echo "[]" ;; esac
+FAKEGH
+    chmod +x "$fakebin/gh"
+    ( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$M" "$A" >/dev/null 2>&1 ) || { echo "self-test failed: a merge whose branch parent has a green run was refused"; fails=1; }
+    ( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$B" "$A" >/dev/null 2>&1 ) || { echo "self-test failed: a plain commit with its own green run was refused"; fails=1; }
+    ( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$M" "$B" >/dev/null 2>&1 ) && { echo "self-test failed: a merge onto another tip (its own run queued) was let through"; fails=1; }
+    ( cd "$fx" && PATH="$fakebin:$PATH" master_ci_ok "$A" "$A" >/dev/null 2>&1 ) && { echo "self-test failed: a commit with no ci run was let through to master"; fails=1; }
+    rm -rf "$fx" "$fakebin"
+    [ "$fails" = 0 ] && echo "self-test passed: a failing check is RED and fails the gate, a passing one is ok; master and release-* select the full gate, other refs the light one; a merge of a green-stamped branch onto the remote tip defers to CI, every other shape takes the full gate; master takes only a commit (or a merge's branch parent) whose own ci run is green, and refuses red, queued and unrun ones; a feature-branch push prints the branch's previous red first; a --ci site build outside GitHub Actions leaves the tree unchanged (structural checks, no-secrets, identity grep)"
     exit $fails ;;
   list)
     grep -E '^\s+run "' "$0" | sed -E 's/^\s+run "([^"]+)".*/\1/' ;;
@@ -214,6 +270,10 @@ case "$MODE" in
     if [ "$which" = full ]; then
       # a merge of a green-stamped branch onto the exact remote tip goes through on the light gate (CI runs the full one)
       verdict=""; while read -r lref lsha rref rsha; do case "$rref" in refs/heads/master|refs/heads/release-*) verdict=$(deferred_merge "$lsha" "$rsha"); break ;; esac; done <<<"$REFS"
+      # a push to master: the pushed commit (or its branch parent) must already have a green ci run on that exact commit
+      while read -r lref lsha rref rsha; do
+        if [ "$rref" = refs/heads/master ] && [ "$lsha" != 0000000000000000000000000000000000000000 ]; then master_ci_ok "$lsha" "$rsha" || exit 1; fi
+      done <<<"$REFS"
       case "$verdict" in
         defer*) echo "pre-push gate: a merge of green-stamped ${verdict#defer } onto the remote tip: the light gate here, the full gate in CI on landing:"
                 structural_checks; never_push_checks; finish "merge of a green branch (full gate deferred to CI)" ;;
@@ -222,6 +282,7 @@ case "$MODE" in
       esac
     else
       echo "pre-push gate: a feature branch, the light gate (the two structural checks, the no-secrets check, the identity grep):"
+      while read -r lref lsha rref rsha; do case "$rref" in refs/heads/*) branch_red_line "${rref#refs/heads/}" ;; esac; done <<<"$REFS"
       structural_checks; never_push_checks; finish "feature branch"
     fi ;;
   ci|local)

tools/ci/red-watch.mjs

@@ -78,6 +78,7 @@ export function runFromEnv(env = process.env) {
   if (!sha) throw new Error('record: neither RED_WATCH_SHA nor GITHUB_SHA is set');
   return {
     run_id: id, attempt: Number(pick('RED_WATCH_ATTEMPT', 'GITHUB_RUN_ATTEMPT') || 1), workflow: pick('RED_WATCH_WORKFLOW', 'GITHUB_WORKFLOW'),
+    conclusion: env.RED_WATCH_CONCLUSION || 'failure',   // failure, cancelled or timed_out (ci-red.yml fires on all three since 7 October 2026)
     branch: pick('RED_WATCH_BRANCH', 'GITHUB_REF_NAME'), sha: sha.slice(0, 7), event: pick('RED_WATCH_EVENT', 'GITHUB_EVENT_NAME'),
     actor: pick('RED_WATCH_ACTOR', 'GITHUB_ACTOR'), author: env.RED_WATCH_AUTHOR || '',   // who pushed (the GitHub login), who the head commit names
     url: env.RED_WATCH_URL || `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${id}`, at: new Date().toISOString(),
@@ -124,11 +125,15 @@ export async function record(file, env = process.env, fetchImpl = fetch, title =
   return { written: true, run: line };
 }
 
+// The kind of line: a failed run is "CI red"; a cancelled run "CI cancelled" (a hand on the run, or a job past its timeout-minutes
+// under GitHub's older runner, which reports cancelled); a timed-out run "CI timed out". All three are read like a red.
+export const KINDS = { failure: 'CI red', cancelled: 'CI cancelled', timed_out: 'CI timed out' };
 export function formatLine(l) {
   const where = l.failed.length ? l.failed.map((f) => `${f.job.replace(/,.*$/, '')} at "${f.step}"`).join('; ') : (l.note || 'no step detail');
   const title = l.title ? ` "${l.title}"` : '';
   const who = l.actor ? ` pushed by ${l.actor}${l.author && l.author !== l.actor ? ` (commit by ${l.author})` : ''};` : '';
-  return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}:${who} ${where} ${l.url}`;
+  const kind = KINDS[l.conclusion || 'failure'] || `CI ${l.conclusion}`;
+  return `${kind}: ${l.workflow} on ${l.branch} @${l.sha}${title}:${who} ${where} ${l.url}`;
 }
 
 function readCredentials(file) {
@@ -259,6 +264,22 @@ async function selfTest() {
   if (!asked[0] || !asked[0].includes('/actions/runs/37620364667/jobs')) fails.push(`workflow_run shape: the jobs API was asked for ${asked[0]}, not the failed run`);
   const textRun = formatLine(lr);
   if (!/^CI red: ci on ca3-v4-node @26a4b0f "Counter ASIC 3.0 node plan 6.7": pushed by igneum-labs \(commit by igneum-labs\); site build at "identity grep of the public export list"; simulators at "\(job never started: runner or billing\)" https:\/\/github.com\/igneum-network\/igneum\/actions\/runs\/37620364667$/.test(textRun)) fails.push(`workflow_run line: ${textRun}`);
+  // a cancelled run and a timed-out run are recorded with their kind in the line (a hung job past its timeout-minutes, a hand on the run)
+  const cancelledJobs = { jobs: [{ name: 'site build, link check, identity grep', conclusion: 'cancelled', steps: [{ name: 'site build', conclusion: 'success' }, { name: 'the tree gate, tools/ci/pre-push.sh --ci', conclusion: 'cancelled' }] }] };
+  const fileKinds = path.join(dir, 'kinds.jsonl');
+  await record(fileKinds, { ...envRun, RED_WATCH_RUN_ID: '555', RED_WATCH_CONCLUSION: 'cancelled' }, async () => ({ ok: true, status: 200, json: async () => cancelledJobs }));
+  await record(fileKinds, { ...envRun, RED_WATCH_RUN_ID: '556', RED_WATCH_CONCLUSION: 'timed_out' }, async () => ({ ok: true, status: 200, json: async () => cancelledJobs }));
+  const [lc, lt] = readLines(fileKinds).map(formatLine);
+  if (!/^CI cancelled: ci on ca3-v4-node @26a4b0f .*site build at "the tree gate, tools\/ci\/pre-push.sh --ci"/.test(lc)) fails.push(`cancelled line: ${lc}`);
+  if (!/^CI timed out: ci on ca3-v4-node @26a4b0f /.test(lt)) fails.push(`timed-out line: ${lt}`);
+  if (readLines(fileKinds)[0].conclusion !== 'cancelled') fails.push('record: the conclusion was not kept in the line');
+  // the watcher workflow fires on all three conclusions and hands the conclusion to the record step
+  const ymlPath = path.join(path.dirname(new URL(import.meta.url).pathname), '..', '..', '.github', 'workflows', 'ci-red.yml');
+  if (fs.existsSync(ymlPath)) {
+    const yml = fs.readFileSync(ymlPath, 'utf8');
+    for (const c of ['failure', 'cancelled', 'timed_out']) if (!yml.includes(`github.event.workflow_run.conclusion == '${c}'`)) fails.push(`ci-red.yml: the job's if does not fire on ${c}`);
+    if (!yml.includes('RED_WATCH_CONCLUSION: ${{ github.event.workflow_run.conclusion }}')) fails.push('ci-red.yml: RED_WATCH_CONCLUSION is not handed to the record step');
+  }
   // post, dry run: prints, sends nothing, marks nothing
   let printed = []; const log = (s) => printed.push(s);
   const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; };
@@ -307,7 +328,7 @@ async function selfTest() {
   if (!d3.sent) fails.push('digest: not sent the next day');
   fs.rmSync(dir, { recursive: true, force: true });
   if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); }
-  console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author, the failed run and not the watcher\'s own under workflow_run; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
+  console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author, the failed run and not the watcher\'s own under workflow_run; a cancelled and a timed-out run are recorded with their kind and ci-red.yml fires on all three; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
 }
 
 const cmd = args[0];

tools/ci/retry-once.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Run a check that can red on box state or the network twice before calling it red (standing rule, 7 October 2026, 17:2x UK:
+# a check that talks to a build box or a live host gets one retry and, on a runner without the resource, a clear skip line
+# from the check itself). The first failure is printed as a line, the command runs again after a pause, and only the second
+# failure is the check's verdict. A check that passes first time runs once.
+#
+#   tools/ci/retry-once.sh <name> <command...>      # exit = the last run's exit code; RETRY_ONCE_PAUSE seconds between (default 5)
+#   tools/ci/retry-once.sh --self-test              # a command that fails once then passes is ok; one that fails twice is red;
+#                                                   # a command that passes runs exactly once
+set -uo pipefail
+PAUSE="${RETRY_ONCE_PAUSE:-5}"
+if [ "${1:-}" = --self-test ]; then
+  fails=0; d=$(mktemp -d)
+  # fails once then passes: the marker file is the memory
+  flaky="$d/flaky.sh"; printf '#!/usr/bin/env bash\nif [ -f "%s/seen" ]; then exit 0; fi; touch "%s/seen"; echo first-try-failed; exit 7\n' "$d" "$d" > "$flaky"; chmod +x "$flaky"
+  out=$(RETRY_ONCE_PAUSE=0 bash "$0" flaky "$flaky" 2>&1); rc=$?
+  [ "$rc" = 0 ] || { echo "self-test failed: a command that fails once then passes was red (exit $rc): $out"; fails=1; }
+  case "$out" in *"retry-once: flaky failed once (exit 7)"*) ;; *) echo "self-test failed: the first failure was not printed: $out"; fails=1 ;; esac
+  # fails twice: red with the second exit code
+  out=$(RETRY_ONCE_PAUSE=0 bash "$0" dead bash -c 'echo nope; exit 3' 2>&1); rc=$?
+  [ "$rc" = 3 ] || { echo "self-test failed: a command that fails twice was not red with its exit code (got $rc): $out"; fails=1; }
+  [ "$(printf '%s\n' "$out" | grep -c '^nope$')" = 2 ] || { echo "self-test failed: the command did not run exactly twice: $out"; fails=1; }
+  # passes: runs once
+  c="$d/count"; : > "$c"
+  RETRY_ONCE_PAUSE=0 bash "$0" fine bash -c "echo x >> '$c'" >/dev/null 2>&1 || { echo "self-test failed: a passing command was red"; fails=1; }
+  [ "$(wc -l < "$c" | tr -d ' ')" = 1 ] || { echo "self-test failed: a passing command ran $(wc -l < "$c") times"; fails=1; }
+  rm -rf "$d"
+  [ "$fails" = 0 ] && echo "self-test passed: one retry after a first failure, red only on the second, a passing command runs once"
+  exit $fails
+fi
+[ $# -ge 2 ] || { echo "usage: tools/ci/retry-once.sh <name> <command...> | --self-test" >&2; exit 2; }
+NAME="$1"; shift
+"$@"; rc=$?
+[ "$rc" = 0 ] && exit 0
+echo "retry-once: $NAME failed once (exit $rc); second try in ${PAUSE}s (a box or network check gets one retry before it is red)"
+sleep "$PAUSE"
+"$@"

tools/ci/workflow-timeouts-check.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+# Every job in .github/workflows carries timeout-minutes, and the named jobs carry their budgets (7 October 2026, 15:13 to
+# 17:20 UK: three hosted `site` jobs on master hung in the tree gate for over two hours each, with GitHub's six-hour default
+# as the only stop; a fourth would have been a failure email at 21:13 UK). The budgets, from the measured times on
+# 7 October: the tree gate on a hosted runner 229 s plus a 40 s Playwright install, so `site` 15; the classifier `changes`
+# a 7 s API call, so 10; the box's igneum-pow suite 45 s to 2 min 40 s, so `pow` 60; the two simulators 2 x 120 s, so `sims` 45.
+#
+#   tools/ci/workflow-timeouts-check.sh              # exit 1 naming each job without a timeout or with the wrong budget
+#   tools/ci/workflow-timeouts-check.sh --self-test  # a fixture job without the key fails; a wrong budget fails; the tree passes
+set -euo pipefail
+cd "$(dirname "$0")/../.."
+
+# <file> -> lines "job timeout" for every job (timeout "-" when missing). A job is a key at four spaces under `jobs:`;
+# its timeout-minutes is the key at six spaces before the next job.
+jobs_of() {
+  awk '
+    /^jobs:/ { injobs = 1; next }
+    injobs && /^[^ ]/ { injobs = 0 }
+    injobs && /^  [A-Za-z0-9_-]+:/ { if (job != "") print job, (t == "" ? "-" : t); job = $1; sub(":", "", job); t = ""; next }
+    injobs && job != "" && /^    timeout-minutes:/ { t = $2 }
+    END { if (job != "") print job, (t == "" ? "-" : t) }
+  ' "$1"
+}
+check_tree() {   # <dir with workflows> -> exit 1 with one line per wrong job
+  local f rc=0 job t want
+  for f in "$1"/*.yml "$1"/*.yaml; do
+    [ -f "$f" ] || continue
+    while read -r job t; do
+      if [ "$t" = "-" ]; then echo "workflow-timeouts: $(basename "$f") job \`$job\` has no timeout-minutes" >&2; rc=1; continue; fi
+      want=""
+      case "$(basename "$f"):$job" in
+        ci.yml:site) want=15 ;; ci.yml:changes) want=10 ;; ci.yml:pow) want=60 ;; ci.yml:sims) want=45 ;;
+      esac
+      if [ -n "$want" ] && [ "$t" != "$want" ]; then echo "workflow-timeouts: $(basename "$f") job \`$job\` has timeout-minutes $t, the budget is $want" >&2; rc=1; fi
+    done < <(jobs_of "$f")
+  done
+  return $rc
+}
+if [ "${1:-}" = --self-test ]; then
+  fails=0; d=$(mktemp -d); mkdir -p "$d/bad" "$d/budget"
+  printf 'name: x\non: push\njobs:\n  a:\n    runs-on: ubuntu-latest\n    timeout-minutes: 5\n    steps: []\n  b:\n    runs-on: ubuntu-latest\n    steps: []\n' > "$d/bad/ci.yml"
+  out=$(check_tree "$d/bad" 2>&1) && { echo "self-test failed: a job without timeout-minutes passed"; fails=1; }
+  case "$out" in *'job `b` has no timeout-minutes'*) ;; *) echo "self-test failed: the job without a timeout was not named: $out"; fails=1 ;; esac
+  case "$out" in *'job `a`'*) echo "self-test failed: a job with a timeout was named: $out"; fails=1 ;; esac
+  printf 'name: ci\non: push\njobs:\n  site:\n    runs-on: ubuntu-latest\n    timeout-minutes: 360\n    steps: []\n' > "$d/budget/ci.yml"
+  out=$(check_tree "$d/budget" 2>&1) && { echo "self-test failed: site at 360 minutes passed"; fails=1; }
+  case "$out" in *'has timeout-minutes 360, the budget is 15'*) ;; *) echo "self-test failed: the wrong budget was not named: $out"; fails=1 ;; esac
+  rm -rf "$d"
+  check_tree .github/workflows || { echo "self-test failed: the tree's workflows do not pass"; fails=1; }
+  [ "$fails" = 0 ] && echo "self-test passed: a job without timeout-minutes fails, a wrong budget fails, the tree's workflows pass (site 15, changes 10, pow 60, sims 45)"
+  exit $fails
+fi
+check_tree .github/workflows && echo "workflow-timeouts: every job in .github/workflows carries timeout-minutes (site 15, changes 10, pow 60, sims 45)"