Merge remote-tracking branch 'origin/master' into build-server

This commit is contained in:
igneum-labs 2026-10-07 13:17:19 +00:00
commit 721e0f3ab9
6 changed files with 262 additions and 51 deletions

42
.github/workflows/ci-red.yml vendored Normal file
View file

@ -0,0 +1,42 @@
# The red watcher as its own workflow, on workflow_run, so the copy on master watches EVERY branch's ci run whatever
# ci.yml that branch carries: GitHub runs a workflow_run workflow from the default branch only, and the branch's own
# ci.yml never enters it (7 October 2026: the inline `red` job of ci.yml was conditioned on master and release-*, and
# a feature branch would have waited for a merge of master before its reds were posted at all).
#
# One line per failed run (tools/ci/red-watch.mjs record, idempotent per run attempt) to /srv/ci-red/red.jsonl on the
# box; the box's igneum-ci-red.timer posts each new line once to the hidden updates channel, naming the branch, the
# commit, the red check and the pushing author. Runs on the box's own runner (not a GitHub-hosted machine: the billing
# block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told). Never blocks a
# release: it reads the run, writes one line, and ends.
name: ci-red
on:
workflow_run:
workflows: [ci]
types: [completed]
jobs:
red:
name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file)
if: ${{ github.event.workflow_run.conclusion == 'failure' }}
runs-on: [self-hosted, linux, x64, igneum-build-1]
timeout-minutes: 5
permissions:
actions: read # the failed run's jobs API (the first real red run, 21:19Z on 6 October: the default token answered 403 and the line carried no step)
contents: read
steps:
- uses: actions/checkout@v4
with:
sparse-checkout: tools/ci
- name: record the failed run (one line, the branch, the commit, the failed jobs and their first failed step from the run's own API, the pushing author)
env:
GITHUB_TOKEN: ${{ github.token }}
RED_WATCH_RUN_ID: ${{ github.event.workflow_run.id }}
RED_WATCH_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
RED_WATCH_WORKFLOW: ${{ github.event.workflow_run.name }}
RED_WATCH_BRANCH: ${{ github.event.workflow_run.head_branch }}
RED_WATCH_SHA: ${{ github.event.workflow_run.head_sha }}
RED_WATCH_EVENT: ${{ github.event.workflow_run.event }}
RED_WATCH_URL: ${{ github.event.workflow_run.html_url }}
RED_WATCH_ACTOR: ${{ github.event.workflow_run.actor.login }}
RED_WATCH_TITLE: ${{ github.event.workflow_run.head_commit.message }}
RED_WATCH_AUTHOR: ${{ github.event.workflow_run.head_commit.author.name }}
run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl

View file

@ -10,10 +10,12 @@
#
# Where it runs: `pow` and `sims` go to the box's runner (igneum-build-1, rustc pinned, sccache read-only, 48 jobs)
# when the repository variable IGNEUM_CI_RUNNER is `box`, else to ubuntu-latest (docs/plans/ci-self-hosted.md; GitHub
# has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The `red` job
# runs on the box after any failed master or release-* run and records the failure for the watcher
# (tools/ci/red-watch.mjs; infra/build-server/ci-red): one line per run to the hidden updates channel and to
# /srv/ci-red/red.jsonl, so nobody opens the Actions page to learn master is red.
# has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The red watcher
# is its own workflow, .github/workflows/ci-red.yml (workflow_run, so the copy on master watches every branch's run
# whatever ci.yml that branch carries): one line per failed run, naming the branch, the commit, the red check and the
# pushing author, to the hidden updates channel and to /srv/ci-red/red.jsonl (tools/ci/red-watch.mjs;
# infra/build-server/ci-red), so nobody opens the Actions page to learn a branch is red (the inline `red` job here
# watched master and release-* only until 7 October 2026, when eight red runs on ca3-v4-node went unseen).
#
# What does not run, on purpose: the node fork (vendor/igneum-node*, a rusty-kaspa fork of about 500 crates with
# rocksdb, blst and the execution layer) is gitignored here and too big for the free runners today (a cold build is
@ -75,27 +77,3 @@ jobs:
- name: public stats API answers with the documented fields (the live site; master only, the endpoints exist there after the merge)
if: github.ref == 'refs/heads/master'
run: node tools/ci/public-api-check.mjs https://igneum.network
red:
# Runs only when a master or release-* run has a failed job, on the box's own runner (not a GitHub-hosted machine:
# the billing block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told).
# tools/ci/red-watch.mjs record appends ONE line for this run to /srv/ci-red/red.jsonl (idempotent per run attempt);
# the box's igneum-ci-red.timer posts each new line once to the hidden updates channel. Never blocks a release:
# it reads the run, writes one line, and ends.
name: red watcher (master and release-* only; one line per failed run to the updates channel and the box file)
needs: [pow, sims, site]
if: ${{ failure() && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) }}
runs-on: [self-hosted, linux, x64, igneum-build-1]
timeout-minutes: 5
permissions:
actions: read # the run's jobs API (the first real red run, 21:19Z: the default token answered 403 and the line carried no step)
contents: read
steps:
- uses: actions/checkout@v4
with:
sparse-checkout: tools/ci
- name: record this run (one line, the failed jobs and their first failed step, from the run's own API)
env:
GITHUB_TOKEN: ${{ github.token }}
RED_WATCH_TITLE: ${{ github.event.head_commit.message }}
run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl

View file

@ -355,7 +355,19 @@ the project lead gave the go in advance for tonight: the shipper runs publish 1
| p25 | 1.28x | |
| the 53 passing | 0.9915x to 1.16x | no predicted source |
Three residual classes, all a constant delivered through a writer the rule admits: (1) saturation or zero preserved through rotl, rotr, load or mad (p31, p6, p23, p26, p34); (2) zero from mulhi (p45) and zero preserved by rotates; (3) the iteration boundary, the rule's writer state starting fresh at instruction 0 so an or at 63 feeds a load at 1 (p11). Sub-version 2's dataflow-freshness rule closes all three IF the freshness is computed as a fixpoint over the loop (the state after instruction 63 feeds instruction 0 of the next iteration), with the dynamic (c') count on load sources as the backstop; sent to the hash lane. "No lossy-sourced load by construction" is not true of sub-version 1 and stays held. Chip consequence on sub-version 1 by the 1.4 arithmetic: the hot set is still one item at one site, under 1 percent of rate for a chip caching it, so the ship is safe on the rate side; the auditor's flag is what sub-version 2 removes. F9's hot-set harness cannot be the second re-gate (its metric counts the era's designed half and quarter windows as hot; its chain-path 8.3 percent on sub-version 1 is the window model, verified on its worst seed); F8's census is the single re-gate instrument, re-run on sub-version 2 within the hour of its commit. THE INTEROP FACT stands from the void run: the 5899f603 hub accepted 235 object-byte-5 blocks from the 8097d600 node with 0 rejected, one digest on all five nodes on the live sixteen-field file. The gates: the digest test and the kaspa-pow vector test (the amended devnet epoch-0 id 1a4230699a6b9c60 must equal, c120d7963abdcd96 must differ, the v3 control unchanged) on the box; the mixed-version Devnet 2 gate (the amended 0.3.20 node beside a 5899f603 node for ten minutes on the live file without the v4 fields) after the Mac build; the fresh-join canary the 0.3.20 cut's |
Three residual classes, all a constant delivered through a writer the rule admits: (1) saturation or zero preserved through rotl, rotr, load or mad (p31, p6, p23, p26, p34); (2) zero from mulhi (p45) and zero preserved by rotates; (3) the iteration boundary, the rule's writer state starting fresh at instruction 0 so an or at 63 feeds a load at 1 (p11). Sub-version 2's dataflow-freshness rule closes all three IF the freshness is computed as a fixpoint over the loop (the state after instruction 63 feeds instruction 0 of the next iteration), with the dynamic (c') count on load sources as the backstop; sent to the hash lane. "No lossy-sourced load by construction" is not true of sub-version 1 and stays held. Chip consequence on sub-version 1 by the 1.4 arithmetic: the hot set is still one item at one site, under 1 percent of rate for a chip caching it, so the ship is safe on the rate side; the auditor's flag is what sub-version 2 removes. F9's hot-set harness cannot be the second re-gate (its metric counts the era's designed half and quarter windows as hot; its chain-path 8.3 percent on sub-version 1 is the window model, verified on its worst seed); F8's census is the single re-gate instrument, re-run on sub-version 2 within the hour of its commit. SUB-VERSION 2 COMMITTED: ca3-v4-amend 07a809a7, 13:01Z (origin and build), the string with the attack-pass lane. The loop fixpoint is in as rule (a') in accept.rs (the freshness run to its fixpoint over base then shadow block; every load's source fresh in the steady state, else the candidate rejected and the next attempt drawn; it closes the iteration boundary the draw's fallback cannot see); (c') counts zero and all-ones alike (v == 0 or v == MAX) per load site over the 16,384 evaluations, rejected at 164 or more; both and the draw rule keyed on the class v4 shape on every draw path, so v2 and v3 do not move; mulhi never fresh. Epoch-0 id a788661687db4bb3 (the devnet seed's attempt 0 rejected by the new rules, attempt 1 accepted); must-differ c120d7963abdcd96 and 1a4230699a6b9c60 pinned; object byte 7 in recheck.rs. The seven fingerprints (Metal = Apple OpenCL on the M5 Max, 13:01:17 to 13:01:49Z, the control 90f794dd556f7a3b unchanged):
| Pack | Fingerprint |
|---|---|
| v4-devnet-epoch0 | e370fb2080b7dbb1 |
| era-0 | b7237555d31fc3cf |
| era-1 | b6b167fa15dfe2c9 |
| era-2 | 28bdf65eff33f2c4 |
| era-3 | e26d38c46f3f1b16 |
| era-4 | dd8fdf6ff4f59eed |
| era-5 | 8bf40f5cb858d835 |
Packs zip (eight packs, packs-ca3-v4-sub2) sha256 69c36772cd79e44e2ddd589466d9c64a94a13c9e970e9f27bd76feabb9b4581b. The suite re-runs through master's build-remote on box 2 (the worktree's own script predates --box; the first run died on the flag), line to follow; G1 on PC 2 under --cards-off after it. The sub-version-2 pairing waits on the node lane's re-pin to a788661687db4bb3 and byte 7. For the record, sub-version 1's pairing: igneum-pow 8c728ca3 against b7cc37e7 (8097d600's assert_ne in) 17 passed, 0 failed, rc 0, 12:21Z. 0.3.20's CUT SET AS IT STANDS (the shipper, 14:1x UK): pin c4459193, igneum-pow 8c728ca3 at object byte 5 (sub-version 1), the floor-moved file publishing with it (the project lead's word; the floor from the live DAA at the publish plus 604,800, the digest read on c4459193), publish about 15:15Z (16:15 BST) on CASES END, the sweep from then with PC 1 first. 0.3.21's clock tonight: the node lane stages release-0.3.21-node at the shipper's sweep-end word (about 15:45Z, 16:45 BST) with the sub-version-2 re-pin (07a809a7, byte 7, id a788661687db4bb3) as its own commit, held pending the F8 census on 07a809a7; the census's clock about 14:00Z (15:00 BST) by the attack-pass lane's within-the-hour line from 13:01Z; the pass line every one of the 64 seeds under 1.2x of the window model on the chain path. If the census fails or slips past 19:00Z (20:00 BST), main's standing ruling applies (nothing on sub-version 2 is proposed until the census is green): 0.3.21's node ships byte 5 again with the re-pin dropped and the rest of its line kept. CI NOTE (13:1x UTC): master's ci runs since 12dc5c97 (nineteen of mine) sit queued behind one self-hosted runner (igneum-build-1, busy; 31 queued across branches, one in progress); the last completed master runs (439a233f to 5f990a09) are success; no red exists, the conclusions are unread until the queue drains. MAIN'S WORD (14:2x UK): the floor move is the project lead's word already and ships with 0.3.20 at the cut; the CI lever is a second self-hosted runner on build-2 plus ubuntu-latest for docs-only pushes, ordered to the CI lane; the rule reads "own a red when the conclusion lands", never holding pushes; the census verdict about 14:00Z (15:00 UK) decides 0.3.21's byte. SUB-VERSION 2's STATIC CENSUS (the hash lane, tools/ca3-v4-uniform on box 2, 13:12Z, 1,024 chain-shaped seeds plus F8's p1 to p3): 0 lossy-sourced load sites of 16,432 (14,329 injecting, 2,103 bijective); 0 programs with an or-, mul- or mulhi-sourced load; the no-era draw path gives the devnet epoch-0 seed the pack's own id a788661687db4bb3, so every draw path reads one stream. Cost of (a') and (c'): 1.99 attempts per seed on average against 0.05 before (p2's seed five), about 2 ms of generation per rejected attempt on one core; nothing a miner or node notices. Ledger entry 715f14be. Master 36e08c80 merged into ca3-v4-amend as f5244ad7 (igneum-pow untouched, re-export 0 differing files), pushed with the gate GREEN, its CI runs queued; the box-2 suite runs through the merged tools (the first attempt died on test initialisers missing the (c') field, fixed in the same push; library and packs unaffected). G1 BLOCKED: PC 2 has not picked up fetch-ca3-v4-sub2-20261007 and run-ca3-v4-sub2-g1-pc2-20261007 (published 13:04:02Z, signature OK); the intake shows nothing from 1ccfe586 since job-update-now-0319 at 10:34:21Z; the PC 2 lock releases when the job closes or in 30 minutes; the run is republished when the app reports. PC 2 READS SILENT on the console (the shipper, 14:4x UK): last seen 2 h ago, app 0.3.19 on node 5899f603, its last line the 0.3.19 update-now at 10:34:21Z; the app went down or stopped polling on that update (the UI lane's update-now; PC 1 took the same update and reports). The build-server lane's PC 2 kept-datadir job (run-20261007-125433, published 12:54Z) is unpicked for the same reason, and it is the Windows kept-datadir gate for 0.3.20's PC 1 step. The sweep cannot bring PC 2 back (the app's poller applies updates; a silent app does not poll); a Windows restart of the app is a hand action, the project lead's or by main's word; the shipper has asked main. The hash lane's G1 and the Windows kept-datadir line wait on that answer; the PC 2 lock stays. THE INTEROP FACT stands from the void run: the 5899f603 hub accepted 235 object-byte-5 blocks from the 8097d600 node with 0 rejected, one digest on all five nodes on the live sixteen-field file. The gates: the digest test and the kaspa-pow vector test (the amended devnet epoch-0 id 1a4230699a6b9c60 must equal, c120d7963abdcd96 must differ, the v3 control unchanged) on the box; the mixed-version Devnet 2 gate (the amended 0.3.20 node beside a 5899f603 node for ten minutes on the live file without the v4 fields) after the Mac build; the fresh-join canary the 0.3.20 cut's |
| Main's rulings (7 October, morning) | no generator change to v4 on the live devnet; the record's null is the window model with numbers, sent by the hash lane to the attack-pass lane so AP-F8-1 re-gates against it; a fault beyond the model (a low-entropy source at site 15) stops at the coordinator with the two options priced (a 0.3.19 class amendment before the flip, or the flip held at the floor), nothing shipping without the project lead's word; the tighter tail, an acceptance bound on the hot-set share, is a CLASS V5 item (sent to the v5 lane a6410f3b8abefb762 with the 64-seed census as its gate; the bound's number follows from the model) |
### AP-F4-1, the weak-day MUL draw (the attack-pass lane, 7 October, morning): PASS against v4, a class v5 rule

View file

@ -0,0 +1,132 @@
#!/usr/bin/env node
// Harness summaries never carry a raw key. Every fast-time runner that writes a summary JSON writes it through
// writeSummary(): a 64-hex value under a key-shaped field (a name ending in key, keys, token, secret, password or
// passphrase, or anything nested under one) is shortened to its first 8 hex characters and an ellipsis, and the
// serialised text is then checked with the same rule as tools/ci/no-secrets-check.sh before it touches the disk.
// A summary that would fail the no-secrets gate is refused here, at the source, with the line named.
//
// The class (7 October 2026, 11:26 to 12:47 UK): fork-gate.mjs wrote the two miners' vote-key hashes into
// joint_sinks.*.key and split_sinks.*.key of three gate summaries under docs/plans/counter-asic-3-gate; eight CI runs
// on ca3-v4-node went red on "no secret file names and no 64-hex secrets in the tree" and no pushing lane saw it,
// because the feature-branch hook ran only the structural checks and the red watcher posted master and release-* only.
//
// import { writeSummary } from './lib/redact-keys.mjs'; writeSummary(OUT, summary);
// node infra/fast-time/lib/redact-keys.mjs --self-test a key field is shortened, a hash field is left alone,
// a raw key in the text fails the writer's own check
// node infra/fast-time/lib/redact-keys.mjs --check <file>... exit 1 if any file carries a raw key line (the hits named)
import { mkdirSync, writeFileSync, readFileSync, existsSync, rmSync, mkdtempSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { tmpdir } from 'node:os';
// the content rule of tools/ci/no-secrets-check.sh, verbatim in spirit: a 64-hex value (0x optional) assigned to a name
// ending in token, key, secret, password or passphrase
export const RAW_KEY_LINE = /(token|key|secret|password|passphrase)["']?\s*[:=]\s*["']?(0x)?[0-9a-fA-F]{64}(?![0-9a-fA-F])/i;
export const KEY_FIELD = /(key|keys|token|secret|password|passphrase)$/i;
const HEX64 = /^(0x)?[0-9a-fA-F]{64}$/;
export function shortHex(v) {
if (typeof v !== 'string' || !HEX64.test(v)) return v;
const head = v.startsWith('0x') ? 10 : 8;
return v.slice(0, head) + '…';
}
// A copy of `value` with every 64-hex string under a key-shaped field shortened. Nothing else changes: hashes, digests
// and ids under other names stay whole. The input object is never mutated (the runner keeps comparing live keys).
export function redactKeys(value, underKeyField = false) {
if (Array.isArray(value)) return value.map((v) => redactKeys(v, underKeyField));
if (value && typeof value === 'object') {
const out = {};
for (const [k, v] of Object.entries(value)) out[k] = redactKeys(v, underKeyField || KEY_FIELD.test(k));
return out;
}
return underKeyField ? shortHex(value) : value;
}
// The line numbers (1-based) of `text` that the no-secrets gate would flag.
export function rawKeyLines(text) {
return text.split('\n').map((l, i) => (RAW_KEY_LINE.test(l) ? i + 1 : 0)).filter(Boolean);
}
// The text a summary is written as: redacted, then checked. Throws when a raw key survives (a key inside a free-text
// field such as a quoted log line), so the runner fails before the tree does.
export function summaryText(summary) {
const text = JSON.stringify(redactKeys(summary), null, 2);
const lines = rawKeyLines(text);
if (lines.length) throw new Error(`summary carries a raw 64-hex key at line${lines.length > 1 ? 's' : ''} ${lines.join(', ')}: the no-secrets gate would refuse it; shorten it with redactKeys or drop the field`);
return text;
}
export function writeSummary(file, summary) {
mkdirSync(dirname(file), { recursive: true });
const text = summaryText(summary);
writeFileSync(file, text);
return text;
}
const hex = (c) => c.repeat(64);
function selfTest() {
const fails = [];
const summary = {
pass: true, keys: { a: hex('a'), b: hex('b') },
joint_sinks: { a: { hash: hex('c'), blocks: 1, key: hex('a') }, b: { hash: hex('d'), blocks: 2, key: '0x' + hex('b') } },
samples: [{ t: 1, a: { sink: hex('c').slice(0, 16), key: hex('a') } }],
vote_key: hex('e'), signingKey: hex('f'), a_token: hex('1'), digest: hex('2'), exec_restart_hash: hex('3'), id32: 'a'.repeat(32),
};
const before = JSON.stringify(summary);
const out = redactKeys(summary);
if (JSON.stringify(summary) !== before) fails.push('redactKeys mutated its input');
if (out.keys.a !== 'aaaaaaaa…' || out.keys.b !== 'bbbbbbbb…') fails.push(`keys.* not shortened: ${out.keys.a} ${out.keys.b}`);
if (out.joint_sinks.a.key !== 'aaaaaaaa…') fails.push(`joint_sinks.a.key not shortened: ${out.joint_sinks.a.key}`);
if (out.joint_sinks.b.key !== '0xbbbbbbbb…') fails.push(`a 0x key kept its prefix wrong: ${out.joint_sinks.b.key}`);
if (out.samples[0].a.key !== 'aaaaaaaa…') fails.push('a key inside an array element was not shortened');
if (out.vote_key !== 'eeeeeeee…' || out.signingKey !== 'ffffffff…' || out.a_token !== '11111111…') fails.push('a name ending in key/Key/token was not shortened');
if (out.joint_sinks.a.hash !== hex('c') || out.digest !== hex('2') || out.exec_restart_hash !== hex('3')) fails.push('a hash, digest or *_hash field was changed');
if (out.id32 !== 'a'.repeat(32) || out.joint_sinks.a.blocks !== 1) fails.push('a non-key value was changed');
let text;
try { text = summaryText(summary); } catch (e) { fails.push(`summaryText refused a redactable summary: ${e.message}`); }
if (text && /[0-9a-f]{64}/.test(text.replace(new RegExp(`"(hash|digest|exec_restart_hash)": "(0x)?[0-9a-f]{64}"`, 'g'), ''))) fails.push('a raw 64-hex key survived in the written text');
if (text && rawKeyLines(text).length) fails.push('the written text would fail the no-secrets rule');
// the writer's own check: a raw key line in the text fails, under each shape the gate catches
for (const t of [`{\n "key": "${hex('9')}"\n}`, `KEY=0x${hex('8')}`, `x-igneum-key: ${hex('7')}`, `const signingKey = "${hex('6')}";`]) {
if (!rawKeyLines(t).length) fails.push(`rawKeyLines missed: ${t.slice(0, 30)}`);
}
if (rawKeyLines(`{\n "hash": "${hex('5')}",\n "id": "${'4'.repeat(32)}"\n}`).length) fails.push('rawKeyLines flagged a hash or a 32-hex id');
// a key hiding in free text under a field the redactor does not know: summaryText refuses it, and names the line
let refused = false;
try { summaryText({ ok: true, refusal_example: `Fork choice: refused block by key: ${hex('9')}` }); } catch (e) { refused = /line 3/.test(e.message); }
if (!refused) fails.push('summaryText did not refuse (or did not name the line of) a raw key inside a free-text field');
// writeSummary writes the redacted text, and --check on a raw file fails
const dir = mkdtempSync(join(tmpdir(), 'redact-keys-'));
try {
const good = join(dir, 'nested', 'good.json'); writeSummary(good, summary);
if (!existsSync(good) || rawKeyLines(readFileSync(good, 'utf8')).length) fails.push('writeSummary wrote a raw key or nothing');
if (JSON.parse(readFileSync(good, 'utf8')).keys.a !== 'aaaaaaaa…') fails.push('the written file does not parse back to the shortened key');
const bad = join(dir, 'bad.json'); writeFileSync(bad, JSON.stringify({ keys: { a: hex('a') }, sink: { key: hex('a') } }, null, 2));
if (checkFiles([good]).length) fails.push('--check flagged the redacted file');
const hits = checkFiles([bad]);
if (hits.length !== 1 || !/bad\.json:\d+/.test(hits[0])) fails.push(`--check did not name the raw file and line: ${hits.join(' | ')}`);
if (hits.some((h) => /[0-9a-f]{64}/.test(h))) fails.push('--check printed a 64-hex value');
} finally { rmSync(dir, { recursive: true, force: true }); }
if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); }
console.log('self-test passed: key-shaped fields shorten to 8 hex and an ellipsis, hashes stay whole, the input is not mutated, a raw key in the text fails the writer\'s own check with its line, --check names a raw file without printing the key');
}
export function checkFiles(files) {
const hits = [];
for (const f of files) {
if (!existsSync(f)) { hits.push(`${f}: missing`); continue; }
for (const n of rawKeyLines(readFileSync(f, 'utf8'))) hits.push(`${f}:${n}: a raw 64-hex key under a key-shaped field`);
}
return hits;
}
if (import.meta.url === `file://${process.argv[1]}`) {
const args = process.argv.slice(2);
if (args[0] === '--self-test') selfTest();
else if (args[0] === '--check') {
const hits = checkFiles(args.slice(1));
if (hits.length) { for (const h of hits) console.error(h); process.exit(1); }
console.log(`redact-keys: ${args.length - 1} file(s), no raw key`);
} else { console.error('usage: redact-keys.mjs --self-test | --check <file>...'); process.exit(2); }
}

View file

@ -5,10 +5,11 @@
#
# tools/ci/pre-push.sh # the full gate over this working tree (any worktree; reads the tree, writes nothing in it)
# tools/ci/pre-push.sh --ci # the same, with the site built in place (a CI checkout is disposable)
# tools/ci/pre-push.sh --hook # from .git/hooks/pre-push: full gate for master and release-*, the two structural
# # checks (conflict markers, Windows paths) for every other ref
# tools/ci/pre-push.sh --self-test # the runner fires on a known failure, passes a known success, and the hook picks the
# # right gate from the ref lines
# tools/ci/pre-push.sh --hook # from .git/hooks/pre-push: full gate for master and release-*; the light gate for every
# # other ref: the two structural checks (conflict markers, Windows paths) and the two
# # never-push classes (the no-secrets check, the identity grep), about 20 s on the Mac
# tools/ci/pre-push.sh --self-test # the runner fires on a known failure, passes a known success, the hook picks the
# # right gate from the ref lines, and the light gate carries the never-push classes
# tools/ci/pre-push.sh --list # the check names, one per line
#
# What is NOT here, on purpose: the three compile-or-compute jobs (igneum-pow tests, igneum-census, the simulators) and
@ -51,12 +52,20 @@ structural_checks() {
run "every tracked path is valid on Windows (colon, trailing dot, reserved names, length)" bash tools/ci/windows-paths-check.sh
}
never_push_checks() {
# The two never-push classes, on EVERY ref (7 October 2026: three gate summaries on ca3-v4-node carried a 64-hex key
# through eight red CI runs in 80 minutes; the feature-branch hook ran only the structural checks, so no lane saw it).
# A secret or an identity leak must not reach the remote on any branch; together about 20 s on the Mac.
run "identity grep of the public export list and the served site" bash tools/ci/identity-check.sh
run "no secret file names and no 64-hex secrets in the tree" bash -c 'bash tools/ci/no-secrets-check.sh --self-test && bash tools/ci/no-secrets-check.sh'
}
tree_checks() {
run "site build (in a temporary copy locally, in place in CI)" site_build
run "internal link check of site/*.html" node tools/ci/link-check.mjs
run "every served page carries the full header (six items, Download, burger)" node tools/ci/site-nav-check.mjs
run "ledger sentences present verbatim on their public pages" node tools/ci/ledger-text-check.mjs
run "identity grep of the public export list and the served site" bash tools/ci/identity-check.sh
never_push_checks
run "shell inside .github/workflows parses (bash -n, the PowerShell 5.1 rule)" node tools/ci/check-workflow-shell.mjs
run "PowerShell drive-reference check (\$name: in a double-quoted string)" bash tools/ci/ps-drive-ref-check.sh
run "copied sources are re-stamped before a build" bash tools/ci/copied-sources-check.sh
@ -90,10 +99,10 @@ tree_checks() {
run "ship tool self-test" node tools/ship-app.mjs --self-test
run "relay unit tests" node --test relay/test/parse.test.mjs relay/test/auth.test.mjs relay/test/wake.test.mjs relay/test/ember.test.mjs
run "miner app notice strip and update card tests" node --test app/igneum-app/ui/notices.test.mjs app/igneum-app/ui/update-card.test.mjs app/igneum-app/ui/view.test.mjs app/igneum-app/ui/tune-line.test.mjs
run "no secret file names and no 64-hex secrets in the tree" bash -c 'bash tools/ci/no-secrets-check.sh --self-test && bash tools/ci/no-secrets-check.sh'
run "launch gates: every row with its check, the handoff text clean (self-test, then the tree)" bash -c 'node tools/ci/launch-gates-check.mjs --self-test && node tools/ci/launch-gates-check.mjs'
run "income per tier: the public table equals its inputs, the schedule arithmetic" bash -c 'node tools/launch/income-tiers.mjs --check && node --test tools/launch/income-tiers.test.mjs'
run "hash-origin report: a known-finished day and a known-failed day" node --test tools/observer/hash-origin.test.mjs
run "harness summaries never carry a raw 64-hex key (the writer's own redaction and check)" node infra/fast-time/lib/redact-keys.mjs --self-test
}
gated_refs() {
@ -126,7 +135,12 @@ case "$MODE" in
[ "$(printf 'refs/heads/x 1 refs/heads/release-0.3.15 2\n' | gated_refs)" = full ] || { echo "self-test failed: a push to release-0.3.15 did not select the full gate"; fails=1; }
[ "$(printf 'refs/heads/x 1 refs/heads/x 2\n' | gated_refs)" = light ] || { echo "self-test failed: a push to a feature branch selected the full gate"; fails=1; }
[ "$(printf '' | gated_refs)" = light ] || { echo "self-test failed: a push with no refs selected the full gate"; fails=1; }
[ "$fails" = 0 ] && echo "self-test passed: a failing check is RED and fails the gate, a passing one is ok; master and release-* select the full gate, other refs the light one"
# the light gate carries the two never-push classes beside the structural checks, and the full gate runs them too
declare -f never_push_checks | grep -q 'tools/ci/no-secrets-check.sh' || { echo "self-test failed: the never-push checks do not run the no-secrets check"; fails=1; }
declare -f never_push_checks | grep -q 'tools/ci/identity-check.sh' || { echo "self-test failed: the never-push checks do not run the identity grep"; fails=1; }
grep -qE '^\s+structural_checks; never_push_checks; finish "feature branch"' "$0" || { echo "self-test failed: the hook's light gate does not run the never-push checks"; fails=1; }
declare -f tree_checks | grep -q 'never_push_checks' || { echo "self-test failed: the full gate does not run the never-push checks"; fails=1; }
[ "$fails" = 0 ] && echo "self-test passed: a failing check is RED and fails the gate, a passing one is ok; master and release-* select the full gate, other refs the light one (structural checks, no-secrets, identity grep)"
exit $fails ;;
list)
grep -E '^\s+run "' "$0" | sed -E 's/^\s+run "([^"]+)".*/\1/' ;;
@ -136,8 +150,8 @@ case "$MODE" in
echo "pre-push gate: a push to master or release-*, the full gate (the same checks CI runs):"
structural_checks; tree_checks; finish "push to master or release-*"
else
echo "pre-push gate: a feature branch, the two structural checks:"
structural_checks; finish "feature branch"
echo "pre-push gate: a feature branch, the light gate (the two structural checks, the no-secrets check, the identity grep):"
structural_checks; never_push_checks; finish "feature branch"
fi ;;
ci|local)
[ "$MODE" = ci ] && echo "pre-push gate in CI (the same script as the local hook):" || echo "pre-push gate over this working tree:"

View file

@ -1,11 +1,15 @@
#!/usr/bin/env node
// The red-master watcher. One line per failed master or release-* run, so nobody opens the Actions page to learn CI is red.
// The red watcher. One line per failed run on ANY branch (master and release-* only until 7 October 2026: eight red runs on
// ca3-v4-node went unseen that morning), naming the branch, the commit, the red check and the pushing author, so nobody
// opens the Actions page to learn a branch is red.
// Node 22, standard library only.
//
// node tools/ci/red-watch.mjs record --file <red.jsonl> in the workflow's `red` job (runs on igneum-build-1 after a
// failed run): reads the run from the GitHub environment and
// node tools/ci/red-watch.mjs record --file <red.jsonl> in .github/workflows/ci-red.yml (a workflow_run job on
// igneum-build-1 after a failed ci run on any branch): reads the
// FAILED run from RED_WATCH_* (the workflow_run payload; the
// GITHUB_* variables there describe the watcher's own run) and
// the failed jobs and steps from the API with the job's own
// token, appends ONE JSON line for this run id (idempotent)
// token, appends ONE JSON line for that run id (idempotent)
// node tools/ci/red-watch.mjs post --file <red.jsonl> [--live] on the box, every minute as `build` (igneum-ci-red.timer):
// every recorded run not yet posted goes as one line to the
// hidden updates channel (DISCORD_WEBHOOK_UPDATES in the
@ -41,7 +45,7 @@ const STATE_FILE = process.env.IGNEUM_CI_RED_STATE || path.join(os.homedir(), '.
const WEBHOOK_KEY = 'DISCORD_WEBHOOK_UPDATES';
// what stops each class now (named in the digest so the line teaches, not just counts); docs/analysis/ci-failures-2026-10-06.md
export const GUARDS = {
'ci': 'the pre-push gate (tools/ci/pre-push.sh, the same checks CI runs, before any push to master or release-*)',
'ci': 'the pre-push gate (tools/ci/pre-push.sh: the full gate before a push to master or release-*, the never-push checks before a push to any branch)',
'instant': 'pre-flight in remote-run.sh (manifest, -p package, feature, subcommand checked in a second) and the kept run log',
'preflight-manifest': 'refused before the slot: the manifest did not parse',
'preflight-package': 'refused before the slot: the -p package does not exist',
@ -61,14 +65,22 @@ export function readLines(file) {
return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
}
// The run being recorded: the FAILED run from RED_WATCH_* when the watcher runs as a workflow_run job (ci-red.yml), else
// the job's own run from GITHUB_* (the inline shape, kept for a branch whose ci.yml still carries the old `red` job).
export const watchedRunId = (env = process.env) => env.RED_WATCH_RUN_ID || env.GITHUB_RUN_ID;
export function runFromEnv(env = process.env) {
const need = ['GITHUB_RUN_ID', 'GITHUB_REPOSITORY', 'GITHUB_REF_NAME', 'GITHUB_SHA', 'GITHUB_WORKFLOW'];
const need = ['GITHUB_REPOSITORY', 'GITHUB_RUN_ID'];
for (const k of need) if (!env[k]) throw new Error(`record: ${k} is not set (this command runs inside a GitHub Actions job)`);
const pick = (own, fallback) => env[own] || env[fallback] || '';
const server = env.GITHUB_SERVER_URL || 'https://github.com';
const id = String(watchedRunId(env));
const sha = pick('RED_WATCH_SHA', 'GITHUB_SHA');
if (!sha) throw new Error('record: neither RED_WATCH_SHA nor GITHUB_SHA is set');
return {
run_id: String(env.GITHUB_RUN_ID), attempt: Number(env.GITHUB_RUN_ATTEMPT || 1), workflow: env.GITHUB_WORKFLOW,
branch: env.GITHUB_REF_NAME, sha: env.GITHUB_SHA.slice(0, 7), event: env.GITHUB_EVENT_NAME || '',
url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`, at: new Date().toISOString(),
run_id: id, attempt: Number(pick('RED_WATCH_ATTEMPT', 'GITHUB_RUN_ATTEMPT') || 1), workflow: pick('RED_WATCH_WORKFLOW', 'GITHUB_WORKFLOW'),
branch: pick('RED_WATCH_BRANCH', 'GITHUB_REF_NAME'), sha: sha.slice(0, 7), event: pick('RED_WATCH_EVENT', 'GITHUB_EVENT_NAME'),
actor: pick('RED_WATCH_ACTOR', 'GITHUB_ACTOR'), author: env.RED_WATCH_AUTHOR || '', // who pushed (the GitHub login), who the head commit names
url: env.RED_WATCH_URL || `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${id}`, at: new Date().toISOString(),
};
}
@ -76,7 +88,7 @@ export function runFromEnv(env = process.env) {
// (the caller) is skipped by name. Any API trouble gives an empty list and a note, never a thrown error: the line is
// the thing that must land.
export async function failedJobs(env = process.env, fetchImpl = fetch) {
const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; const id = env.GITHUB_RUN_ID;
const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; const id = watchedRunId(env);
const api = env.GITHUB_API_URL || 'https://api.github.com';
if (!token) return { failed: [], note: 'no GITHUB_TOKEN; failed steps not read' };
try {
@ -115,7 +127,8 @@ export async function record(file, env = process.env, fetchImpl = fetch, title =
export function formatLine(l) {
const where = l.failed.length ? l.failed.map((f) => `${f.job.replace(/,.*$/, '')} at "${f.step}"`).join('; ') : (l.note || 'no step detail');
const title = l.title ? ` "${l.title}"` : '';
return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}: ${where} ${l.url}`;
const who = l.actor ? ` pushed by ${l.actor}${l.author && l.author !== l.actor ? ` (commit by ${l.author})` : ''};` : '';
return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}:${who} ${where} ${l.url}`;
}
function readCredentials(file) {
@ -208,7 +221,8 @@ async function selfTest() {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'red-watch-'));
const file = path.join(dir, 'red.jsonl'); const stateFile = path.join(dir, 'posted.json'); const credFile = path.join(dir, 'discord');
const env = { GITHUB_RUN_ID: '424242', GITHUB_RUN_ATTEMPT: '1', GITHUB_REPOSITORY: 'igneum-network/igneum', GITHUB_REF_NAME: 'master',
GITHUB_SHA: '0f0abc6deadbeef', GITHUB_WORKFLOW: 'ci', GITHUB_TOKEN: 'x', RED_WATCH_TITLE: 'Merge box-work 2bd3bec: the repro script re-stamps its clones' };
GITHUB_SHA: '0f0abc6deadbeef', GITHUB_WORKFLOW: 'ci', GITHUB_TOKEN: 'x', RED_WATCH_TITLE: 'Merge box-work 2bd3bec: the repro script re-stamps its clones',
GITHUB_ACTOR: 'igneum-labs', RED_WATCH_AUTHOR: 'igneum-labs' };
const jobs = { jobs: [
{ name: 'site build, link check, identity grep', conclusion: 'failure', steps: [{ name: 'site build', conclusion: 'success' }, { name: 'identity grep of the public export list', conclusion: 'failure' }] },
{ name: 'igneum-pow tests, igneum-census build', conclusion: 'success', steps: [] },
@ -226,6 +240,25 @@ async function selfTest() {
if (lines[0].failed[1].step !== '(job never started: runner or billing)') fails.push('record: a job with no steps was not named as never started');
const text = formatLine(lines[0]);
if (!/^CI red: ci on master @0f0abc6 "Merge box-work/.test(text) || !text.includes('actions/runs/424242')) fails.push(`format: ${text}`);
if (!text.includes('pushed by igneum-labs (commit by igneum-labs);') || !text.includes('site build at "identity grep of the public export list"')) fails.push(`format: the line does not name the pushing author and the red check: ${text}`);
// a feature branch is recorded and formatted like master (every branch since 7 October 2026)
const envFeature = { ...env, GITHUB_RUN_ID: '424299', GITHUB_REF_NAME: 'ca3-v4-node', RED_WATCH_AUTHOR: 'igneum-labs' };
const fileFeature = path.join(dir, 'feature.jsonl');
await record(fileFeature, envFeature, fakeFetch);
const textFeature = formatLine(readLines(fileFeature)[0]);
if (!/^CI red: ci on ca3-v4-node @0f0abc6 "Merge box-work[^"]*": pushed by igneum-labs; site build at/.test(textFeature)) fails.push(`format on a feature branch: ${textFeature}`);
// the workflow_run shape (ci-red.yml): GITHUB_* describe the watcher's own run, RED_WATCH_* the failed one; the line is the failed run's
const envRun = { GITHUB_RUN_ID: '999', GITHUB_RUN_ATTEMPT: '1', GITHUB_REPOSITORY: 'igneum-network/igneum', GITHUB_REF_NAME: 'master', GITHUB_SHA: 'ffffffffffff', GITHUB_WORKFLOW: 'ci-red', GITHUB_ACTOR: 'igneum-labs', GITHUB_TOKEN: 'x',
RED_WATCH_RUN_ID: '37620364667', RED_WATCH_ATTEMPT: '2', RED_WATCH_WORKFLOW: 'ci', RED_WATCH_BRANCH: 'ca3-v4-node', RED_WATCH_SHA: '26a4b0f1deadbeef', RED_WATCH_EVENT: 'push',
RED_WATCH_URL: 'https://github.com/igneum-network/igneum/actions/runs/37620364667', RED_WATCH_ACTOR: 'igneum-labs', RED_WATCH_TITLE: 'Counter ASIC 3.0 node plan 6.7', RED_WATCH_AUTHOR: 'igneum-labs' };
const fileRun = path.join(dir, 'workflow-run.jsonl'); const asked = [];
const askingFetch = async (url) => { asked.push(url); return { ok: true, status: 200, json: async () => jobs }; };
await record(fileRun, envRun, askingFetch);
const lr = readLines(fileRun)[0];
if (lr.run_id !== '37620364667' || lr.attempt !== 2 || lr.branch !== 'ca3-v4-node' || lr.sha !== '26a4b0f' || lr.workflow !== 'ci') fails.push(`workflow_run shape: recorded ${JSON.stringify({ run_id: lr.run_id, attempt: lr.attempt, branch: lr.branch, sha: lr.sha, workflow: lr.workflow })}, expected the failed run, not the watcher's`);
if (!asked[0] || !asked[0].includes('/actions/runs/37620364667/jobs')) fails.push(`workflow_run shape: the jobs API was asked for ${asked[0]}, not the failed run`);
const textRun = formatLine(lr);
if (!/^CI red: ci on ca3-v4-node @26a4b0f "Counter ASIC 3.0 node plan 6.7": pushed by igneum-labs \(commit by igneum-labs\); site build at "identity grep of the public export list"; simulators at "\(job never started: runner or billing\)" https:\/\/github.com\/igneum-network\/igneum\/actions\/runs\/37620364667$/.test(textRun)) fails.push(`workflow_run line: ${textRun}`);
// post, dry run: prints, sends nothing, marks nothing
let printed = []; const log = (s) => printed.push(s);
const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; };
@ -274,7 +307,7 @@ async function selfTest() {
if (!d3.sent) fails.push('digest: not sent the next day');
fs.rmSync(dir, { recursive: true, force: true });
if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); }
console.log('self-test passed: one line per run however often record runs; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author, the failed run and not the watcher\'s own under workflow_run; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
}
const cmd = args[0];