diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c895324b2..0efaddd2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,19 @@ # CI on every push and pull request (private repository, free runner minutes). # # What runs: the lottery-hash crate's tests (igneum-pow, release profile), the census tool's build, the two Python -# simulators' --quick modes (each under two minutes), the site build with an internal link check, the gh-free -# identity grep of the public export list (tools/ci/forbidden-strings.txt), and the no-secrets check of the tree -# (tools/ci/no-secrets-check.sh: no file named like a key of ~/.config/igneum, no 64-hex value assigned to a -# token/key/secret name outside tests and the allowlist; docs/security/keys.md). +# simulators' --quick modes (each under two minutes), and the tree gate: every fast check in ONE script, +# tools/ci/pre-push.sh (site build and link check, the ledger sentences, the identity grep of the public export list +# and the served site, Windows-valid paths, workflow shell syntax, the copied-sources and playbook classes, the unit +# tests, the no-secrets check). The pre-push hook runs the SAME script before a push to master or release-*, so the +# local gate and CI cannot drift (6 October 2026: 131 red `ci` runs in three days, 92 of them on master, every one a +# tree check that would have failed on the pushing machine in under 25 s; docs/analysis/ci-failures-2026-10-06.md). +# +# Where it runs: `pow` and `sims` go to the box's runner (igneum-build-1, rustc pinned, sccache read-only, 48 jobs) +# when the repository variable IGNEUM_CI_RUNNER is `box`, else to ubuntu-latest (docs/plans/ci-self-hosted.md; GitHub +# has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The `red` job +# runs on the box after any failed master or release-* run and records the failure for the watcher +# (tools/ci/red-watch.mjs; infra/build-server/ci-red): one line per run to the hidden updates channel and to +# /srv/ci-red/red.jsonl, so nobody opens the Actions page to learn master is red. # # What does not run, on purpose: the node fork (vendor/igneum-node*, a rusty-kaspa fork of about 500 crates with # rocksdb, blst and the execution layer) is gitignored here and too big for the free runners today (a cold build is @@ -17,7 +26,7 @@ on: jobs: pow: name: igneum-pow tests, igneum-census build - runs-on: ubuntu-latest + runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 - name: toolchain @@ -32,13 +41,15 @@ jobs: run: cargo build --release sims: name: simulators, quick modes - runs-on: ubuntu-latest + runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + if: vars.IGNEUM_CI_RUNNER != 'box' # the box has python3 and numpy from provision.sh with: python-version: '3.12' - run: python3 -m pip install --quiet numpy + if: vars.IGNEUM_CI_RUNNER != 'box' - name: finality_v2.py --quick (under two minutes) working-directory: sim run: time timeout 120 python3 finality_v2.py --quick > finality_quick.md @@ -59,56 +70,29 @@ jobs: - uses: actions/setup-node@v4 with: node-version: '22' - - name: site build - run: node site/build.mjs - - name: internal link check of site/*.html - run: node tools/ci/link-check.mjs - - name: identity grep of the public export list - run: bash tools/ci/identity-check.sh - - name: no conflict markers in tracked files - run: bash tools/ci/no-conflict-markers.sh - - name: PowerShell drive-reference check (a `$name:` inside a double-quoted string is a 5.1 parse error) - run: bash tools/ci/ps-drive-ref-check.sh - - name: copied sources are re-stamped before a build - run: bash tools/ci/copied-sources-check.sh - - name: override params files parse with no duplicate key (the duplicate-field class, 6 October 2026) - run: bash tools/ci/override-json-check.sh - - name: second-engine playbooks log to a file and end their tree (C35) - run: bash tools/ci/second-engine-check.sh - - name: no playbook quits, pauses or resumes the installed app (self-test first, then the tree) - run: bash tools/ci/playbook-quit-check.sh --self-test && bash tools/ci/playbook-quit-check.sh - - name: no script writes into another worktree or walks Projects (tools/ci/no-foreign-tree-writes.sh) - run: bash tools/ci/no-foreign-tree-writes.sh --self-test && bash tools/ci/no-foreign-tree-writes.sh - - name: the signer is never piped into head - run: bash tools/ci/signer-pipe-check.sh - - name: bash bodies in PowerShell job scripts pass bash -n, the lost-quote class (self-test first, then the tree) - run: bash tools/ci/bash-body-check.sh --self-test && bash tools/ci/bash-body-check.sh - - name: run jobs test their fetched kit before use, the wiped-jobs-folder class (self-test first, then the tree) - run: bash tools/ci/kit-path-check.sh --self-test && bash tools/ci/kit-path-check.sh - - name: every Windows spawn of the app runs with a hidden console (self-test first, then the tree) - run: node tools/ci/windows-spawn-check.mjs --self-test && node tools/ci/windows-spawn-check.mjs - - name: pinned guest programs match their manifest and are built only by pin-guests.sh - run: bash tools/ci/pinned-guests-check.sh - - name: root prover playbooks kill the GPU server and unlink its socket (the root-socket class, 5 October 2026) - run: bash tools/ci/prover-socket-check.sh - - name: commit-string gate self-test (the empty-commit class of 6 October 2026; the gate itself runs in build-remote.sh, cross-remote.sh and cross-build.sh on every node binary) - run: bash tools/ci/commit-string-check.sh --self-test - - name: build server remote checkout self-test (the stale-overlay class of 6 October 2026) - run: bash infra/build-server/remote-run.sh --self-test - - name: no shell assignment hides behind a trailing comment (the swallowed-defaults class of 6 October 2026) - run: bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh - - name: no secret file names and no 64-hex secrets in the tree (self-test first, then the tree) - run: bash tools/ci/no-secrets-check.sh --self-test && bash tools/ci/no-secrets-check.sh - - name: faucet unit tests (validation, the daily limits, the signed transaction; keccak, RLP and secp256k1 vectors) - run: node --test site/api/faucet.test.mjs - - name: explorer and public stats unit tests (search router, formatters, emission rule against the node's own test values, the documented API fields from a fixture) - run: node --test site/lib/explorer.test.mjs site/lib/emission.test.mjs site/api/public-stats.test.mjs + - name: the tree gate, tools/ci/pre-push.sh --ci (the same script the pre-push hook runs; one line per check, a red check prints its output) + run: bash tools/ci/pre-push.sh --ci - name: public stats API answers with the documented fields (the live site; master only, the endpoints exist there after the merge) if: github.ref == 'refs/heads/master' run: node tools/ci/public-api-check.mjs https://igneum.network - - name: ship tool self-test (version bump, the dl-both and public manifest helpers) - run: node tools/ship-app.mjs --self-test - - name: relay unit tests (parsers, secret compare, the wake endpoint) - run: node --test relay/test/parse.test.mjs relay/test/auth.test.mjs relay/test/wake.test.mjs relay/test/ember.test.mjs - - name: miner app notice strip and update card (ordering, keys, wording, timers, when the card shows) - run: node --test app/igneum-app/ui/notices.test.mjs app/igneum-app/ui/update-card.test.mjs app/igneum-app/ui/view.test.mjs app/igneum-app/ui/tune-line.test.mjs + + red: + # Runs only when a master or release-* run has a failed job, on the box's own runner (not a GitHub-hosted machine: + # the billing block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told). + # tools/ci/red-watch.mjs record appends ONE line for this run to /srv/ci-red/red.jsonl (idempotent per run attempt); + # the box's igneum-ci-red.timer posts each new line once to the hidden updates channel. Never blocks a release: + # it reads the run, writes one line, and ends. + name: red watcher (master and release-* only; one line per failed run to the updates channel and the box file) + needs: [pow, sims, site] + if: ${{ failure() && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) }} + runs-on: [self-hosted, linux, x64, igneum-build-1] + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: tools/ci + - name: record this run (one line, the failed jobs and their first failed step, from the run's own API) + env: + GITHUB_TOKEN: ${{ github.token }} + RED_WATCH_TITLE: ${{ github.event.head_commit.message }} + run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 377b452e9..cfe19eeed 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -293,3 +293,21 @@ jobs: path: build/inputs-artifact/ retention-days: 90 if-no-files-found: error + + red: + # The red watcher for the Windows pipeline (see ci.yml `red`): one line per failed master or release-* run to the + # hidden updates channel and /srv/ci-red/red.jsonl on the box, recorded by the box's own runner. + name: red watcher (master and release-* only; one line per failed run to the updates channel and the box file) + needs: [parse, build] + if: ${{ failure() && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) }} + runs-on: [self-hosted, linux, x64, igneum-build-1] + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: tools/ci + - name: record this run (one line, the failed jobs and their first failed step, from the run's own API) + env: + GITHUB_TOKEN: ${{ github.token }} + RED_WATCH_TITLE: ${{ github.event.head_commit.message }} + run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl diff --git a/CLAUDE.md b/CLAUDE.md index dc43ea8ef..bffc70c15 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -118,6 +118,13 @@ Josh's ruling after the DAA 198,000 incident (a fixed-height activation crossed - Standing fleet (Josh, 6 October 2026, 20:0x UK: "cant we keep rented cards up longer"): 16 live-devnet boxes and 6 Devnet 2 boxes are kept up permanently and re-rented on host death; only benchmark and wave boxes are one-shot; a standing box never leaves the live devnet for an experiment (the class v4 rehearsal took the 15 prover boxes and paused live finality at 18:42Z; experiments use wave boxes only). The Mac runs nothing the network depends on: node 1 and the observer move to igneum-build-1 as systemd units (docs/plans/hands-on-build-1.md). - Secrets on igneum-build-1: none that sign releases, move funds or reach the hands. Exception recorded 6 October 2026: Discord webhook URLs at /srv/discord-hooks/env (mode 600, rotatable in one click) for tools/community/discord-hooks.mjs. +## CI red is stop-the-line (standing rule, 6 October 2026, 22:0x UK) +- Whoever's merge turns master or a release-* branch red owns the fix inside 15 minutes or reverts the merge; the red watcher posts every failed run to the hidden updates channel and to /srv/ci-red/red.jsonl on the box (tools/ci/red-watch.mjs). +- The pre-push gate is the same script CI runs: `tools/ci/pre-push.sh` (installed by `tools/ci/install-hooks.sh`; `--hook` before a push to master or release-*, `--ci` in the workflow). A check is added there, never only in ci.yml. +- Research and operations documents live outside the public export list: name them in `tools/ci/export-exclude.txt` (read by the identity check and by the mirror's sync.sh). What stays in the list is read by the public and must pass the identity grep. +- A path Windows cannot hold (colon, trailing dot or space, reserved name, over 240 characters) never enters a commit: the pre-commit hook runs `tools/ci/windows-paths-check.sh --staged`. +- Record: docs/analysis/ci-failures-2026-10-06.md (168 non-green runs in three days classified; 126 on master; all but two classes were tree checks that the gate now runs locally first). + ## A rule row closes only with its check (standing rule, 6 October 2026, 18:4x UK) Josh, after the pgrep self-match hit twice in one day (the shipper's hands script at lunchtime, the fleet's wave script at 17:1xZ: a `pgrep -f ""` whose literal sat in the calling shell's own command line, so the check always passed and no node ever started): "again wasted time". Rules: diff --git a/docs/analysis/ci-failures-2026-10-06.md b/docs/analysis/ci-failures-2026-10-06.md new file mode 100644 index 000000000..6f7bccbcc --- /dev/null +++ b/docs/analysis/ci-failures-2026-10-06.md @@ -0,0 +1,115 @@ +# CI failures, 4 to 6 October 2026: every non-green run classified, the fixes, the guards + +Written 6 October 2026, 22:0x UK, from `gh run list` (430 runs, every run since the first workflow run at 09:57Z on +4 October) and `gh run view --log-failed` on one run per class. Times UTC (UK was UTC+1). This document is an operations +record and is listed in `tools/ci/export-exclude.txt`: it is not exported to the public mirror. + +## 1. The totals + +| Workflow | Runs | Green | Failed | Cancelled | +|---|---:|---:|---:|---:| +| ci | 336 | 205 | 131 | 0 | +| windows-ci | 94 | 57 | 20 | 17 | +| total | 430 | 262 | 151 | 17 | + +126 of the 168 non-green runs were on master. Master was red without a break from 18:37Z to the end of the evening +(20:23Z, run 37526027569) across three causes in a row: the billing block, the copied-sources check, the identity grep. + +## 2. Every failure by root cause + +One line per class. "Guard" is what now stops the class before it reaches master. + +| Class | Runs | First | Last | Cause | Fix | Guard | +|---|---:|---|---|---|---|---| +| A1 identity grep | 56 | 04 13:26Z | 05 01:46Z | A simulator's run log committed under `sim/difficulty/records` carried the Mac's home path in its first line; 43 master pushes in twelve hours each failed the same step | The log was scrubbed; `.log` files joined the generic scrub (mirror e18256d) | The pre-push gate runs the identity grep locally before any push to master or release-* (`tools/ci/pre-push.sh --hook`), so the hit lands on the pushing machine, not on master | +| A2 identity grep | 17 | 05 02:17Z | 05 10:36Z | vmmap dumps under `docs/benchmarks/memory-floods-2026-10-04/vmmap/` carried a local time offset on their Date/Time lines | The dumps were re-stamped to UTC | Same gate | +| A3 identity grep | 1 | 06 20:23Z | 06 20:23Z | `docs/analysis/horizon/polish.md`, a research review of internal tooling, quotes the overlay network's product name, the intake key's variable name and the identity guard's own regexes as text; docs/analysis is in the export list, so the grep is right to flag it | The document is listed in `tools/ci/export-exclude.txt`; the identity check and the mirror's `sync.sh` both prune that list, so the guard's purpose (nothing the public reads carries these strings) is intact and the research text is untouched | The exclusion list, plus the gate; `identity-check.sh --self-test` shows an excluded path may quote the patterns and an exported one may not | +| B1 hosted runner refused | 32 | 06 18:37Z | 06 20:05Z | "The job was not started because recent account payments have failed or your spending limit needs to be increased": every GitHub-hosted job of every run failed at start with zero steps, 26 master runs and 6 release-0.3.15 runs, and nothing told anyone | Cleared on the billing page by 20:08Z | The `red` job runs on the box's own runner after any failed master or release-* run and posts one line per run (section 4); the `pow` and `sims` jobs can move to the box with one repository variable (section 5) | +| C1 copied-sources | 1 | 06 18:00Z | 06 18:00Z | `tools/workers/collect.mjs` mentioned rsync and cargo in a comment; the check read comments | The check skips comment lines | The gate | +| C2 copied-sources | 12 | 06 20:08Z | 06 20:19Z | `infra/build-server/repro/rebuild-on-box.sh` clones sources and runs cargo without `touch`; 10 master runs and 2 release-0.3.15 runs | 0f0abc6 (box-work 2bd3bec): the repro script re-stamps its clones. Release-0.3.15 still carries the old script at c25a3ca and will fail this step again until it takes master (or cherry-picks 2bd3bec) | The gate | +| D no-foreign-tree-writes | 2 | 06 18:20Z | 06 18:24Z | The new check's warning pipeline (`grep | grep -v | sed | cut`) fails under `pipefail` on a file with no hit, and `set -e` ends the script silently with exit 1 after "self-test passed"; the two runs right after it landed died this way, and the Mac's bash 3.2 died the same way on every tree | `|| true` on the warning pipeline (this change) | The gate runs the check locally, where it would have shown | +| E Windows checkout | 3 | 04 13:53Z | 06 20:15Z | Nine screenshot files under `docs/plans/site-ui-3-shots/` carried a colon from an address; git on windows-latest refuses the path, so `actions/checkout` died and with it every Windows build of the tree (the 0.3.15 installer waited on it) | 61f46cc renamed the nine files | `tools/ci/windows-paths-check.sh`: colon and the other forbidden characters, trailing dot or space, reserved device names, over 240 characters; as the pre-commit hook on the staged paths and in the gate on every tracked path | +| F1 site build | 5 | 04 13:46Z | 04 13:53Z | `site/scrub-bench.sh` failed on `docs/bench-log.md` and `build.mjs` threw from the execSync | Fixed in the bench log the same afternoon | The gate builds the site in a temporary copy before the push | +| F2 site build | 2 | 05 16:42Z | 05 16:43Z | Conflict markers left in `site/journey.json`; `build.mjs` parsed it as JSON | Resolved by hand; `no-conflict-markers.sh` and the first pre-push hook (fb076de) followed | The gate's first check, on every push | +| G link check | 1 | 05 09:28Z | 05 09:28Z | `/#wallet` linked from every page with no such id (release-0.3.6) | Anchor added | The gate | +| H windows payload inputs | 4 | 05 16:30Z | 06 18:15Z | The signed inputs manifest on the downloads host pinned one node commit and `packaging/windows/node-source.pin` in the tree another: the Mac had pushed new inputs without committing the pin, or committed a pin without pushing inputs | Each time, the pin and the inputs were brought level | Not a tree check and not in the gate: the shipper's push-inputs.sh writes the pin and the commit must carry it; the `red` job now reports the mismatch within a minute instead of the next person opening the Actions page | +| I windows installer | 1 | 04 10:32Z | 04 10:32Z | The runner image's Inno Setup was older than 6.3 | The workflow installs Inno Setup when the image's is too old | Resolved in the workflow | +| J windows engine | 2 | 04 13:53Z | 04 13:56Z | Rust that did not compile pushed to master (`expected identifier, found keyword let`) | Fixed in the next push | A compile is not a 25-second check; the owner's merge rule (CLAUDE.md, "CI red is stop-the-line") covers it: the merger fixes or reverts inside 15 minutes | +| K igneum-census | 1 | 05 23:18Z | 05 23:18Z | igneum-pow's `Instr`, `Program` and a layout argument changed; igneum-census was not rebuilt (release-0.3.11) | Updated with the crate | As J | +| L prover-socket | 1 | 06 08:49Z | 06 08:49Z | `tools/proving-v1/pc2-agg-cost.ps1` ran the prover host as root without killing sp1-gpu-server (release-0.3.12) | The playbook was fixed | The gate | +| M public API check | 1 | 06 15:12Z | 06 15:12Z | The live observer was 969 s stale when the master-only live check ran | The observer recovered; the hands moved to the box that evening | A live check stays in CI only, master only; it is not a tree fact and not in the gate | +| N no-secrets | 2 | 06 15:56Z | 06 16:23Z | A 64-hex test vector next to `private_key` in `app/igneum-wallet/src/vault.rs` (wallet-0.1.5) | Allow-listed as a test value | The gate | +| P swallowed defaults line (no CI run: a silent class) | 0 | 06 19:5xZ | 06 21:xxZ | A comment appended to a line of shell assignments in `tools/build-remote.sh` and then `tools/cross-remote.sh` turned every assignment after the `#` into comment text; `bash -n` and shellcheck are silent on it; the default cross-build never ran and its chain kept the previous exes from about 20:40 to 22:00 UK | 36e4ee7 on master: the lines split; `tools/ci/defaults-line-check.sh` with its self-test | In ci.yml at 36e4ee7 and in the gate from this change, so it runs on the pushing machine before the push | +| O1 windows-ci cancelled | 16 | 04 10:42Z | 06 18:12Z | `concurrency: cancel-in-progress` on windows.yml: a newer master push superseded the run. Not a failure | None needed | None; they are listed because `gh run list` counts them as non-green | +| O2 hosted runner not acquired | 8 | 05 19:26Z | 05 20:54Z | "The job was not acquired by Runner of type hosted even after multiple attempts" on release-0.3.10 (7) and master (1): GitHub capacity, retried by hand | Re-run | The `red` job reports it; the box runner for `pow` and `sims` (section 5) takes those jobs off the hosted pool | + +Sum: 168 runs, plus class P, which never reached CI because nothing checked for it. Classes A1 to A3, C1, C2, D, E, +F1, F2, G, L and N are 102 runs (61 percent), every one a tree check that finishes in under 25 s on the pushing machine. B1 and O2 are 40 runs (24 percent) of GitHub-side refusals that nobody +saw until the Actions page was opened. O1 is 16 runs (10 percent) of expected cancellations. H, I, J, K and M are the +remaining 10. + +## 3. The gate: one script, local and CI (`tools/ci/pre-push.sh`) + +Every fast tree check CI runs is in one script. The `site` job of ci.yml calls `tools/ci/pre-push.sh --ci`; the pre-push +hook calls `tools/ci/pre-push.sh --hook`. The two cannot drift because there is one list. A check added to ci.yml alone +is the wrong place; it goes in the script. + +| Mode | When | What | +|---|---|---| +| `--hook` on a push to master or release-* | installed by `tools/ci/install-hooks.sh` into the shared hooks directory (one set for every worktree) | all 31 checks; a red check refuses the push and prints its output | +| `--hook` on any other ref | same | the two structural checks only (conflict markers, Windows paths) | +| `--ci` | the `site` job | all 31 checks, with the site built in place | +| default | by hand in any worktree | all 31 checks | +| `--self-test` | in the gate itself | a known failure is RED and fails the gate; a known success is ok; master and release-* select the full gate, other refs the light one | + +Measured 6 October 2026, 21:5x UK, on the Mac: 30 checks, GREEN, 25 s (no-secrets 11 s, identity grep 3 s, the rest +under 2 s each). The hook never writes into the worktree: the site is built in a temporary copy with +`SITE_DOWNLOADS_OFFLINE=1` (063bbca: the earlier hook built in place and rewrote the downloads snapshot in five +worktrees); `git status` before and after the full gate is identical. + +Checks that joined CI through the gate and were not in ci.yml before: the ledger sentence check +(`ledger-text-check.mjs`), the workflow shell parse (`check-workflow-shell.mjs`), the Windows paths check, and the three +self-tests (identity, Windows paths, the red watcher). The swallowed-defaults check (36e4ee7) is in the gate too. + +## 4. The red watcher (`tools/ci/red-watch.mjs`, `infra/build-server/ci-red/`) + +A `red` job in ci.yml and windows.yml runs only when a master or release-* run has a failed job. It runs on the box's +own runner (`igneum-build-1`), not on a GitHub-hosted machine, because the hosted pool is the thing that was refused in +B1 and O2. It appends one JSON line for the run (id, workflow, branch, commit, title, the failed jobs and each one's first +failed step from the run's own API, the URL) to `/srv/ci-red/red.jsonl`, idempotent per run attempt. On the box, +`igneum-ci-red.timer` runs the poster every minute as `build`: each line not yet posted goes once to the hidden updates +channel through `DISCORD_WEBHOOK_UPDATES` in `/srv/discord-hooks/env`, then its run id is recorded in +`/srv/discord-hooks/ci-red-posted.json`. The orchestrator reads the file (`ssh build@ cat /srv/ci-red/red.jsonl`) +or the channel. No URL is ever printed; a missing key is logged by name. + +Shown on 6 October 2026: the self-test (one line however often `record` runs; the dry run sends nothing; a missing key +is named, never a URL; one live send per run; a webhook error keeps the run pending). The poster is installed and +active on the box (22:55 CEST, "nothing to post (0 recorded)"). Open: the updates channel has no webhook yet, so the +first real red run will land in `red.jsonl` and the poster will log the missing key until `DISCORD_WEBHOOK_UPDATES` is +added to `~/.config/igneum/discord` on the Mac and `infra/build-server/discord-hooks/install.sh` is re-run. The +Actions-side trigger has not fired on a real red run yet (master was made green in the same change); the first red +master or release-* run is its known-failed case. + +## 5. Where CI runs, and why `ci` takes about three minutes + +| Job | Where today | Time on ubuntu-latest | Time on the box (measured 6 October) | Note | +|---|---|---|---|---| +| pow (igneum-pow `cargo test --release`, packfile test, igneum-census build) | ubuntu-latest | 2 min 30 s to 3 min, cold every run (no cache action) | 42 s cold as the runner user (99 tests), sccache read-only hits after the first build | the long pole | +| sims (two Python simulators, --quick) | ubuntu-latest | about 1 min with setup-python and pip | python3 and numpy are on the box from provision.sh | | +| site (the gate) | ubuntu-latest | under 1 min | not moved: the live public API check belongs on a neutral egress | | +| red | the box's runner | | seconds | only after a failed master or release-* run | + +The workflow now reads the repository variable `IGNEUM_CI_RUNNER`: `box` sends `pow` and `sims` to +`[self-hosted, linux, x64, igneum-build-1]`, anything else keeps `ubuntu-latest` (docs/plans/ci-self-hosted.md: GitHub +has no fallback in `runs-on`, so a variable is the switch; `gh variable set IGNEUM_CI_RUNNER --body box` as igneum-josh, +`gh variable delete IGNEUM_CI_RUNNER` to come back). Recommendation: flip it. The two compile-or-compute jobs are what +GitHub's minutes and the billing block were spent on, the box compiles the agents' own pinned rustc 1.99.0, and a `ci` +run drops from about three minutes to about one. The hosted runner then serves only the gate and the Windows +pipeline (MSVC, WebView2, Inno Setup, PowerShell 5.1, which a Linux box cannot provide). The flip is main's call after the +0.3.15 cut, per build-server.md section 7.1. + +## 6. What belongs on another branch + +| Branch | One-line change | +|---|---| +| release-0.3.15 | take master (or cherry-pick 2bd3bec): `infra/build-server/repro/rebuild-on-box.sh` re-stamps its clones, else the copied-sources step fails again at the next push. Its own `tools/ci/windows-paths-check.sh` (c25a3ca) is superseded by master's: on merge keep master's file and drop the extra ci.yml step, the gate runs it | diff --git a/infra/build-server/ci-red/igneum-ci-red.service b/infra/build-server/ci-red/igneum-ci-red.service new file mode 100644 index 000000000..79b8700d5 --- /dev/null +++ b/infra/build-server/ci-red/igneum-ci-red.service @@ -0,0 +1,28 @@ +# The red-master watcher's poster on igneum-build-1: one pass a minute from igneum-ci-red.timer. +# The workflow's `red` job (ci.yml, windows.yml; runs on this box's runner after a failed master or release-* run) appends +# one JSON line per run to /srv/ci-red/red.jsonl as the runner user. This pass, as build, posts every line not yet posted +# to the hidden updates channel (DISCORD_WEBHOOK_UPDATES in /srv/discord-hooks/env) and records the run id in +# /srv/discord-hooks/ci-red-posted.json. One line per run, however many passes. `journalctl -u igneum-ci-red -n 30`. +# Installed by infra/build-server/ci-red/install.sh. +[Unit] +Description=Igneum CI red watcher (post failed master and release runs to the updates channel) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +User=build +Group=build +Environment=HOME=/home/build +Environment=PATH=/usr/local/bin:/usr/bin:/bin +Environment=IGNEUM_DISCORD_ENV=/srv/discord-hooks/env +Environment=IGNEUM_CI_RED_STATE=/srv/discord-hooks/ci-red-posted.json +WorkingDirectory=/srv/discord-hooks +ExecStart=/usr/local/bin/node /srv/discord-hooks/bin/red-watch.mjs post --file /srv/ci-red/red.jsonl --live +TimeoutStartSec=50 +Nice=10 +# the unit reads one secret file; nothing else on the box may +PrivateTmp=yes +NoNewPrivileges=yes +ProtectSystem=strict +ReadWritePaths=/srv/discord-hooks diff --git a/infra/build-server/ci-red/igneum-ci-red.timer b/infra/build-server/ci-red/igneum-ci-red.timer new file mode 100644 index 000000000..8d441bdf0 --- /dev/null +++ b/infra/build-server/ci-red/igneum-ci-red.timer @@ -0,0 +1,13 @@ +# Fires the CI red watcher's poster every minute. `systemctl list-timers igneum-ci-red.timer` shows the next pass. +[Unit] +Description=Igneum CI red watcher, a pass every minute + +[Timer] +OnBootSec=60s +OnCalendar=*-*-* *:*:30 +AccuracySec=5s +Persistent=true +Unit=igneum-ci-red.service + +[Install] +WantedBy=timers.target diff --git a/infra/build-server/ci-red/install.sh b/infra/build-server/ci-red/install.sh new file mode 100644 index 000000000..7e111eca9 --- /dev/null +++ b/infra/build-server/ci-red/install.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Install or refresh the CI red watcher's poster on igneum-build-1 from this Mac. +# infra/build-server/ci-red/install.sh (re-run whenever tools/ci/red-watch.mjs or the units change) +# Copies tools/ci/red-watch.mjs to /srv/discord-hooks/bin (beside the Discord scheduler, which already holds the webhook +# file), creates /srv/ci-red (owner runner, 755: the workflow's `red` job writes red.jsonl there as the runner user, the +# poster and anyone on the box read it), installs the two units and enables the timer. No secret moves here: the poster +# reads the webhook file the Discord scheduler's install.sh already placed (DISCORD_WEBHOOK_UPDATES is the key it needs; +# until that key is in ~/.config/igneum/discord and that install.sh is re-run, every pass logs the missing key by name +# and the lines wait in red.jsonl). Needs root over ssh (root@ with ~/.ssh/igneum_ed25519); the host ip comes from +# ~/.config/igneum/build-server (build@). +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../../.." && pwd)" +KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}" +HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')" +IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server" >&2; exit 1; } +node "$ROOT/tools/ci/red-watch.mjs" --self-test >/dev/null || { echo "red-watch.mjs fails its own self-test; not installing" >&2; exit 1; } + +SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10) +"${SSH[@]}" "root@$IP" 'install -d -o build -g build -m 750 /srv/discord-hooks /srv/discord-hooks/bin && install -d -o runner -g runner -m 755 /srv/ci-red && [ -f /srv/ci-red/red.jsonl ] || install -o runner -g runner -m 644 /dev/null /srv/ci-red/red.jsonl' +scp -q -i "$KEY" "$ROOT/tools/ci/red-watch.mjs" "build@$IP:/srv/discord-hooks/bin/" +scp -q -i "$KEY" "$HERE/igneum-ci-red.service" "$HERE/igneum-ci-red.timer" "root@$IP:/etc/systemd/system/" +"${SSH[@]}" "root@$IP" 'systemctl daemon-reload && systemctl enable --now igneum-ci-red.timer >/dev/null 2>&1; systemctl is-active igneum-ci-red.timer; systemctl list-timers igneum-ci-red.timer --no-pager | sed -n 2p' +# one dry pass as the unit's user: what would be posted, names only, never a URL +"${SSH[@]}" "build@$IP" 'IGNEUM_DISCORD_ENV=/srv/discord-hooks/env IGNEUM_CI_RED_STATE=/srv/discord-hooks/ci-red-posted.json /usr/local/bin/node /srv/discord-hooks/bin/red-watch.mjs post --file /srv/ci-red/red.jsonl' +echo "installed; the poster runs at :30 every minute: journalctl -u igneum-ci-red -n 20; the record file: ssh build@$IP cat /srv/ci-red/red.jsonl" diff --git a/infra/build-server/provision.sh b/infra/build-server/provision.sh index ff07635cf..017393d50 100755 --- a/infra/build-server/provision.sh +++ b/infra/build-server/provision.sh @@ -482,6 +482,10 @@ step_runner() { done as_runner "$rustup component list --installed --toolchain $RUST_TOOLCHAIN" | grep -q '^clippy' || { as_runner "$rustup component add clippy rustfmt --toolchain $RUST_TOOLCHAIN" >/dev/null; any=1; } as_runner "git config --global --get safe.directory >/dev/null 2>&1 || git config --global --add safe.directory '*'" # the mirrors are owned by build + # 3b. the CI red watcher's record file: the workflow's `red` job appends one line per failed master or release run here + # (tools/ci/red-watch.mjs record); the poster (infra/build-server/ci-red) reads it as build + if [ ! -d /srv/ci-red ]; then install -d -o "$RUNNER_USER" -g "$RUNNER_USER" -m 755 /srv/ci-red; any=1; fi + [ -f /srv/ci-red/red.jsonl ] || { install -o "$RUNNER_USER" -g "$RUNNER_USER" -m 644 /dev/null /srv/ci-red/red.jsonl; any=1; } # 4. sccache: the build user's binary copied system-wide (the runner cannot read /home/build), a read-only view of /srv/sccache if [ ! -x /usr/local/bin/sccache ] || ! cmp -s "$BUILD_HOME/.cargo/bin/sccache" /usr/local/bin/sccache; then install -m 755 "$BUILD_HOME/.cargo/bin/sccache" /usr/local/bin/sccache; any=1 diff --git a/tools/ci/export-exclude.txt b/tools/ci/export-exclude.txt new file mode 100644 index 000000000..7b1f244a5 --- /dev/null +++ b/tools/ci/export-exclude.txt @@ -0,0 +1,14 @@ +# Paths under the public export list that are NOT exported: research and operations documents written for the +# team, not for the public spec mirror (one path per line, relative to the repository root; a directory excludes its +# tree; # comments ignored). Read by tools/ci/identity-check.sh (pruned from the export copy before the grep) and by +# igneum-public/tools/sync.sh (pruned from the mirror after the copy), so the two can never disagree. +# +# Rule (CLAUDE.md, "CI red is stop-the-line", 6 October 2026): a research document that reviews internal operations or +# quotes the identity patterns as text lives here, outside the export list. A document that IS meant for the mirror is +# not listed here and must pass the identity grep like everything else the public reads. +# +# 6 October 2026, 21:2x UK: the Horizon lane's polish review (lane 6) quotes the overlay network's product name, the +# intake key variable name and the identity guard's own regexes as text; it blocked every master run from 20:23Z. +docs/analysis/horizon/polish.md +# the CI failure classification of 6 October 2026 (an operations document: run ids, step names, the fixes) +docs/analysis/ci-failures-2026-10-06.md diff --git a/tools/ci/identity-check.sh b/tools/ci/identity-check.sh index 5b1b04ea3..e14fdf38c 100755 --- a/tools/ci/identity-check.sh +++ b/tools/ci/identity-check.sh @@ -8,10 +8,35 @@ # The private rules of the mirror (sync.local.sed, identity.local) are not here; they run at export time. # # tools/ci/identity-check.sh # exit 1 on any hit, with file:line +# tools/ci/identity-check.sh --self-test # a forbidden word in an exported path fails; the same word in an excluded path passes +# +# Excluded from the export (6 October 2026): the paths in tools/ci/export-exclude.txt, research and operations documents +# written for the team. igneum-public/tools/sync.sh prunes the same file after its copy, so what this check skips never +# reaches the mirror either. IDENTITY_CHECK_REPO points the check at another tree (the self-test's fixture). set -euo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" -REPO="$(cd "$HERE/../.." && pwd)" +REPO="${IDENTITY_CHECK_REPO:-$(cd "$HERE/../.." && pwd)}" PATTERNS="$HERE/forbidden-strings.txt" +EXCLUDE="$HERE/export-exclude.txt" + +if [ "${1:-}" = "--self-test" ]; then + fx="$(mktemp -d)"; trap 'rm -rf "$fx"' EXIT + mkdir -p "$fx/docs/analysis/horizon" "$fx/docs/spec" "$fx/site" + echo "# spec" > "$fx/docs/spec/clean.md" + # the excluded path (the first entry of export-exclude.txt) carrying a forbidden word, and a clean exported tree: must pass + first="$(grep -vE '^\s*(#|$)' "$EXCLUDE" | head -1)" + mkdir -p "$fx/$(dirname "$first")"; printf 'quotes the overlay name: Tailscale, and the key: LOG_INTAKE\n' > "$fx/$first" + if ! IDENTITY_CHECK_REPO="$fx" bash "$0" >/dev/null 2>&1; then echo "self-test failed: a forbidden word in the excluded path $first was reported"; exit 1; fi + # the same word in an exported path: must fail + printf 'the overlay name: Tailscale\n' > "$fx/docs/spec/leak.md" + if IDENTITY_CHECK_REPO="$fx" bash "$0" >/dev/null 2>&1; then echo "self-test failed: a forbidden word in docs/spec/leak.md passed"; exit 1; fi + rm -f "$fx/docs/spec/leak.md" + # a served site file carrying a pattern: must fail, unscrubbed + printf '

a home path /Users/someone/x

\n' > "$fx/site/leak.html" + if IDENTITY_CHECK_REPO="$fx" bash "$0" >/dev/null 2>&1; then echo "self-test failed: a home path in site/leak.html passed"; exit 1; fi + echo "self-test passed: the excluded research path may quote the patterns; an exported document and a served page may not" + exit 0 +fi # The export list of igneum-public/tools/sync.sh (keep in step with it), plus the two files published with the repository # at the public testnet (decision of 5 October 2026: the criticism ledger and its fixes file). They are not in sync.sh, @@ -25,7 +50,11 @@ FILES=(site/ledger.html docs/provenance.md docs/bench-log.md docs/evidence.md do TMP="$(mktemp -d)"; trap 'rm -rf "$TMP"' EXIT for d in "${DIRS[@]}"; do [ -d "$REPO/$d" ] && { mkdir -p "$TMP/$(dirname "$d")"; cp -R "$REPO/$d" "$TMP/$d"; }; done for f in "${FILES[@]}"; do [ -f "$REPO/$f" ] && { mkdir -p "$TMP/$(dirname "$f")"; cp "$REPO/$f" "$TMP/$f"; }; done -# prune what sync.sh prunes +# prune what sync.sh prunes: the excluded research paths first (tools/ci/export-exclude.txt), then build output +while IFS= read -r x; do + x="${x%%#*}"; x="$(printf '%s' "$x" | sed -E 's/^[[:space:]]+|[[:space:]]+$//g')"; [ -n "$x" ] || continue + rm -rf "${TMP:?}/$x" +done < "$EXCLUDE" find "$TMP" \( -name target -o -name __pycache__ -o -name out -o -name 'build-*' -o -name node_modules -o -name results -o -name runs \) -prune -exec rm -rf {} + 2>/dev/null || true find "$TMP" \( -name .DS_Store -o -name '*.pyc' \) -type f -delete @@ -77,4 +106,4 @@ if [ -n "$SITE_HITS" ]; then printf '%s\n' "$SITE_HITS" | sed "s#^$REPO/##" | cut -c1-200 exit 1 fi -echo "identity grep: 0 hits over $(printf '%s\n' "$TEXT_FILES" | grep -c .) export files and $(printf '%s\n' "$SITE_FILES" | grep -c .) served site files" +echo "identity grep: 0 hits over $(printf '%s\n' "$TEXT_FILES" | grep -c .) export files ($(grep -cvE '^\s*(#|$)' "$EXCLUDE") research paths excluded by tools/ci/export-exclude.txt) and $(printf '%s\n' "$SITE_FILES" | grep -c .) served site files" diff --git a/tools/ci/install-hooks.sh b/tools/ci/install-hooks.sh index f138ab7c5..59b55fcee 100755 --- a/tools/ci/install-hooks.sh +++ b/tools/ci/install-hooks.sh @@ -1,12 +1,28 @@ #!/usr/bin/env bash -# Installs the repository's git hooks into this checkout (pre-push: no conflict markers, the site builds). +# Installs the repository's git hooks into this checkout's shared hooks directory (one set for the main checkout and +# every worktree, since worktrees share .git/hooks). Each hook is a two-line delegate to a versioned script, so a +# change to the gate is a commit, never a reinstall: +# pre-commit -> tools/ci/windows-paths-check.sh --staged (a path Windows cannot check out never enters a commit) +# pre-push -> tools/ci/pre-push.sh --hook (the full CI gate before a push to master or release-*, +# the two structural checks before any other push) +# Neither hook writes into the worktree (the site is built in a temporary copy; 063bbca, 6 October 2026). +# A tree that predates the scripts (an old branch) falls back to the conflict-marker check alone. set -euo pipefail -cd "$(dirname "$0")/../.." -cat > .git/hooks/pre-push <<'HOOK' -#!/usr/bin/env bash -set -e cd "$(git rev-parse --show-toplevel)" -bash tools/ci/no-conflict-markers.sh -(cd site && node build.mjs >/dev/null) || { echo "pre-push: the site build fails; fix it before pushing" >&2; exit 1; } +hooks="$(git rev-parse --git-common-dir)/hooks"; mkdir -p "$hooks" +cat > "$hooks/pre-push" <<'HOOK' +#!/usr/bin/env bash +# installed by tools/ci/install-hooks.sh; the gate itself is versioned in tools/ci/pre-push.sh (same script as CI) +cd "$(git rev-parse --show-toplevel)" || exit 1 +if [ -f tools/ci/pre-push.sh ]; then exec bash tools/ci/pre-push.sh --hook "$@"; fi +exec bash tools/ci/no-conflict-markers.sh HOOK -chmod +x .git/hooks/pre-push; echo "pre-push hook installed" +cat > "$hooks/pre-commit" <<'HOOK' +#!/usr/bin/env bash +# installed by tools/ci/install-hooks.sh; the check itself is versioned in tools/ci/windows-paths-check.sh +cd "$(git rev-parse --show-toplevel)" || exit 1 +[ -f tools/ci/windows-paths-check.sh ] || exit 0 +exec bash tools/ci/windows-paths-check.sh --staged +HOOK +chmod +x "$hooks/pre-push" "$hooks/pre-commit" +echo "hooks installed in $hooks: pre-commit (Windows paths of the staged files), pre-push (tools/ci/pre-push.sh --hook)" diff --git a/tools/ci/no-foreign-tree-writes.sh b/tools/ci/no-foreign-tree-writes.sh index 969032bc8..a99e8b387 100755 --- a/tools/ci/no-foreign-tree-writes.sh +++ b/tools/ci/no-foreign-tree-writes.sh @@ -35,7 +35,9 @@ if [ "${1:-}" = "--self-test" ]; then echo "self-test passed: a Projects path fails, a worktree-list loop with a write fails, an own-toplevel write passes"; exit 0 fi fail=0 -while IFS= read -r f; do grep -nE "$WARN" "$f" | grep -vE '^[0-9]+:\s*(#|//)' | sed "s|^|foreign-tree (warning, env-only default owed): $f:|" | cut -c1-200; done < <(git ls-files 'tools/**' 'packaging/**' 'infra/**' 'relay/**' | grep -E '\.(sh|mjs|js|py|ps1)$' | grep -v '^tools/ci/no-foreign-tree-writes.sh$') +# `|| true`: with pipefail a file without a warning hit fails this pipeline, and set -e then ends the script silently with +# exit 1 (bash 3.2 on the Mac; the two CI runs right after this check landed on 6 October 2026 died the same way) +while IFS= read -r f; do grep -nE "$WARN" "$f" | grep -vE '^[0-9]+:\s*(#|//)' | sed "s|^|foreign-tree (warning, env-only default owed): $f:|" | cut -c1-200 || true; done < <(git ls-files 'tools/**' 'packaging/**' 'infra/**' 'relay/**' | grep -E '\.(sh|mjs|js|py|ps1)$' | grep -v '^tools/ci/no-foreign-tree-writes.sh$') while IFS= read -r f; do check_file "$f" || fail=1; done < <(git ls-files 'tools/**' 'packaging/**' 'site/*.mjs' 'infra/**' 'relay/**' | grep -E '\.(sh|mjs|js|py|ps1)$' | grep -v '^tools/ci/no-foreign-tree-writes.sh$') [ "$fail" = 0 ] && echo "foreign-tree: every script writes under its own toplevel" exit $fail diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh new file mode 100755 index 000000000..f596c9e51 --- /dev/null +++ b/tools/ci/pre-push.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# The one gate. Every fast tree check CI runs, in one script, so the local gate and CI can never drift: the `site` job +# of .github/workflows/ci.yml calls `tools/ci/pre-push.sh --ci`, and the pre-push hook (tools/ci/install-hooks.sh) calls +# `tools/ci/pre-push.sh --hook` before any push to master or a release-* branch and refuses the push on red. +# +# tools/ci/pre-push.sh # the full gate over this working tree (any worktree; reads the tree, writes nothing in it) +# tools/ci/pre-push.sh --ci # the same, with the site built in place (a CI checkout is disposable) +# tools/ci/pre-push.sh --hook # from .git/hooks/pre-push: full gate for master and release-*, the two structural +# # checks (conflict markers, Windows paths) for every other ref +# tools/ci/pre-push.sh --self-test # the runner fires on a known failure, passes a known success, and the hook picks the +# # right gate from the ref lines +# tools/ci/pre-push.sh --list # the check names, one per line +# +# What is NOT here, on purpose: the three compile-or-compute jobs (igneum-pow tests, igneum-census, the simulators) and +# the live public API check (master only, a network call), which stay separate steps in ci.yml. Everything here finished +# in under 25 s on the Mac on 6 October 2026 (no-secrets 10 s, everything else under 3 s each). +# +# Local mode never writes into the worktree: the site is built in a temporary copy with SITE_DOWNLOADS_OFFLINE=1 +# (063bbca, 6 October 2026: a hook that built in place rewrote the downloads snapshot in five worktrees). The link, ledger +# and identity checks then read the committed pages; CI builds in place and checks the rebuilt pages, the one difference. +set -uo pipefail +cd "$(git rev-parse --show-toplevel)" || exit 1 +MODE="${1:-local}"; MODE="${MODE#--}" +RED=0; N=0; LOG="$(mktemp)"; trap 'rm -rf "$LOG" "${SITE_TMP:-}"' EXIT +T0=$(date +%s) + +run() { + # run : one line per check; the output of a red check is shown in full + local name="$1"; shift; N=$((N + 1)) + local s=$(date +%s) + if "$@" >"$LOG" 2>&1; then + printf ' ok %3ds %s\n' "$(( $(date +%s) - s ))" "$name" + else + printf ' RED %3ds %s\n' "$(( $(date +%s) - s ))" "$name"; sed 's/^/ /' "$LOG" | cut -c1-240; RED=1 + fi +} +run_quiet() { "$@" >/dev/null 2>&1; } + +site_build() { + if [ "$MODE" = ci ]; then node site/build.mjs; return; fi + SITE_TMP="$(mktemp -d)"; cp -R site "$SITE_TMP/site" + (cd "$SITE_TMP/site" && SITE_DOWNLOADS_OFFLINE=1 node build.mjs) +} + +structural_checks() { + run "no conflict markers in tracked files" bash tools/ci/no-conflict-markers.sh + run "every tracked path is valid on Windows (colon, trailing dot, reserved names, length)" bash tools/ci/windows-paths-check.sh +} + +tree_checks() { + run "site build (in a temporary copy locally, in place in CI)" site_build + run "internal link check of site/*.html" node tools/ci/link-check.mjs + run "ledger sentences present verbatim on their public pages" node tools/ci/ledger-text-check.mjs + run "identity grep of the public export list and the served site" bash tools/ci/identity-check.sh + run "shell inside .github/workflows parses (bash -n, the PowerShell 5.1 rule)" node tools/ci/check-workflow-shell.mjs + run "PowerShell drive-reference check (\$name: in a double-quoted string)" bash tools/ci/ps-drive-ref-check.sh + run "copied sources are re-stamped before a build" bash tools/ci/copied-sources-check.sh + run "override params files parse with no duplicate key" bash tools/ci/override-json-check.sh + run "second-engine playbooks log to a file and end their tree (C35)" bash tools/ci/second-engine-check.sh + run "no playbook quits, pauses or resumes the installed app" bash -c 'bash tools/ci/playbook-quit-check.sh --self-test && bash tools/ci/playbook-quit-check.sh' + run "no script writes into another worktree or walks Projects" bash -c 'bash tools/ci/no-foreign-tree-writes.sh --self-test && bash tools/ci/no-foreign-tree-writes.sh' + run "the signer is never piped into head" bash tools/ci/signer-pipe-check.sh + run "bash bodies in PowerShell job scripts pass bash -n" bash -c 'bash tools/ci/bash-body-check.sh --self-test && bash tools/ci/bash-body-check.sh' + run "run jobs test their fetched kit before use" bash -c 'bash tools/ci/kit-path-check.sh --self-test && bash tools/ci/kit-path-check.sh' + run "every Windows spawn of the app runs with a hidden console" bash -c 'node tools/ci/windows-spawn-check.mjs --self-test && node tools/ci/windows-spawn-check.mjs' + run "pinned guest programs match their manifest" bash tools/ci/pinned-guests-check.sh + run "root prover playbooks kill the GPU server and unlink its socket" bash tools/ci/prover-socket-check.sh + run "commit-string gate self-test" bash tools/ci/commit-string-check.sh --self-test + run "build server remote checkout self-test" bash infra/build-server/remote-run.sh --self-test + run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh' + run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test + run "the Windows paths check's own self-test" bash tools/ci/windows-paths-check.sh --self-test + run "the red watcher's own self-test (one line per run, posted once)" node tools/ci/red-watch.mjs --self-test + run "faucet unit tests" node --test site/api/faucet.test.mjs + run "explorer, emission and public stats unit tests" node --test site/lib/explorer.test.mjs site/lib/emission.test.mjs site/api/public-stats.test.mjs + run "ship tool self-test" node tools/ship-app.mjs --self-test + run "relay unit tests" node --test relay/test/parse.test.mjs relay/test/auth.test.mjs relay/test/wake.test.mjs relay/test/ember.test.mjs + run "miner app notice strip and update card tests" node --test app/igneum-app/ui/notices.test.mjs app/igneum-app/ui/update-card.test.mjs app/igneum-app/ui/view.test.mjs app/igneum-app/ui/tune-line.test.mjs + run "no secret file names and no 64-hex secrets in the tree" bash -c 'bash tools/ci/no-secrets-check.sh --self-test && bash tools/ci/no-secrets-check.sh' +} + +gated_refs() { + # stdin: the pre-push hook's lines " ". Prints "full" when any remote + # ref is master or release-*, else "light". + local lref lsha rref rsha full=0 + while read -r lref lsha rref rsha; do + case "$rref" in refs/heads/master|refs/heads/release-*) full=1 ;; esac + done + [ "$full" = 1 ] && echo full || echo light +} + +finish() { + local what="$1" secs=$(( $(date +%s) - T0 )) + if [ "$RED" = 0 ]; then echo "pre-push gate ($what): GREEN, $N checks in ${secs}s"; exit 0; fi + echo "pre-push gate ($what): RED after $N checks in ${secs}s. Fix it before pushing (every check above is one CI runs; the same script runs there)." >&2 + exit 1 +} + +case "$MODE" in + self-test) + fails=0 + st="$(mktemp)" # run in this shell, not a $(...) subshell, so RED is visible here + run "known failure" false >"$st" 2>&1; out="$(cat "$st")"; case "$out" in *"RED"*"known failure"*) ;; *) echo "self-test failed: a failing check was not reported RED"; fails=1 ;; esac + [ "$RED" = 1 ] || { echo "self-test failed: a failing check did not set RED"; fails=1; } + RED=0 + run "known success" true >"$st" 2>&1; out="$(cat "$st")"; rm -f "$st"; case "$out" in *"ok"*"known success"*) ;; *) echo "self-test failed: a passing check was not reported ok"; fails=1 ;; esac + [ "$RED" = 0 ] || { echo "self-test failed: a passing check set RED"; fails=1; } + [ "$(printf 'refs/heads/x 1 refs/heads/master 2\n' | gated_refs)" = full ] || { echo "self-test failed: a push to master did not select the full gate"; fails=1; } + [ "$(printf 'refs/heads/x 1 refs/heads/release-0.3.15 2\n' | gated_refs)" = full ] || { echo "self-test failed: a push to release-0.3.15 did not select the full gate"; fails=1; } + [ "$(printf 'refs/heads/x 1 refs/heads/x 2\n' | gated_refs)" = light ] || { echo "self-test failed: a push to a feature branch selected the full gate"; fails=1; } + [ "$(printf '' | gated_refs)" = light ] || { echo "self-test failed: a push with no refs selected the full gate"; fails=1; } + [ "$fails" = 0 ] && echo "self-test passed: a failing check is RED and fails the gate, a passing one is ok; master and release-* select the full gate, other refs the light one" + exit $fails ;; + list) + grep -E '^\s+run "' "$0" | sed -E 's/^\s+run "([^"]+)".*/\1/' ;; + hook) + which="$(gated_refs)" + if [ "$which" = full ]; then + echo "pre-push gate: a push to master or release-*, the full gate (the same checks CI runs):" + structural_checks; tree_checks; finish "push to master or release-*" + else + echo "pre-push gate: a feature branch, the two structural checks:" + structural_checks; finish "feature branch" + fi ;; + ci|local) + [ "$MODE" = ci ] && echo "pre-push gate in CI (the same script as the local hook):" || echo "pre-push gate over this working tree:" + structural_checks; tree_checks; finish "$MODE" ;; + *) echo "usage: tools/ci/pre-push.sh [--ci|--hook|--self-test|--list]" >&2; exit 2 ;; +esac diff --git a/tools/ci/red-watch.mjs b/tools/ci/red-watch.mjs new file mode 100755 index 000000000..1dc6714e7 --- /dev/null +++ b/tools/ci/red-watch.mjs @@ -0,0 +1,199 @@ +#!/usr/bin/env node +// The red-master watcher. One line per failed master or release-* run, so nobody opens the Actions page to learn CI is red. +// Node 22, standard library only. +// +// node tools/ci/red-watch.mjs record --file in the workflow's `red` job (runs on igneum-build-1 after a +// failed run): reads the run from the GitHub environment and +// the failed jobs and steps from the API with the job's own +// token, appends ONE JSON line for this run id (idempotent) +// node tools/ci/red-watch.mjs post --file [--live] on the box, every minute as `build` (igneum-ci-red.timer): +// every recorded run not yet posted goes as one line to the +// hidden updates channel (DISCORD_WEBHOOK_UPDATES in the +// credentials file), then is marked posted in the state file; +// without --live the line is printed, not sent +// node tools/ci/red-watch.mjs --self-test record twice = one line; post = one send; post again = none +// +// Files: the record file is written by the runner user (one object per line: run_id, workflow, branch, sha, title, failed, +// url, at); the poster's state (which run ids were posted, when) is $IGNEUM_CI_RED_STATE, default +// ~/.config/igneum/ci-red-posted.json, so the two users never write the same file. Credentials: $IGNEUM_DISCORD_ENV +// (default ~/.config/igneum/discord), KEY=VALUE lines, mode 600, never printed: a webhook URL never appears in any output, +// only the key's name. The orchestrator reads the record file (ssh build@ cat /srv/ci-red/red.jsonl) or the channel. +import fs from 'node:fs'; +import path from 'node:path'; +import os from 'node:os'; + +const args = process.argv.slice(2); +const flag = (name) => { const i = args.indexOf(name); return i >= 0 ? args[i + 1] : undefined; }; +const has = (name) => args.includes(name); +const CRED_FILE = process.env.IGNEUM_DISCORD_ENV || path.join(os.homedir(), '.config', 'igneum', 'discord'); +const STATE_FILE = process.env.IGNEUM_CI_RED_STATE || path.join(os.homedir(), '.config', 'igneum', 'ci-red-posted.json'); +const WEBHOOK_KEY = 'DISCORD_WEBHOOK_UPDATES'; + +export function readLines(file) { + if (!fs.existsSync(file)) return []; + return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean); +} + +export function runFromEnv(env = process.env) { + const need = ['GITHUB_RUN_ID', 'GITHUB_REPOSITORY', 'GITHUB_REF_NAME', 'GITHUB_SHA', 'GITHUB_WORKFLOW']; + for (const k of need) if (!env[k]) throw new Error(`record: ${k} is not set (this command runs inside a GitHub Actions job)`); + const server = env.GITHUB_SERVER_URL || 'https://github.com'; + return { + run_id: String(env.GITHUB_RUN_ID), attempt: Number(env.GITHUB_RUN_ATTEMPT || 1), workflow: env.GITHUB_WORKFLOW, + branch: env.GITHUB_REF_NAME, sha: env.GITHUB_SHA.slice(0, 7), event: env.GITHUB_EVENT_NAME || '', + url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`, at: new Date().toISOString(), + }; +} + +// The failed jobs and their first failed step, from the run's jobs API with the job's own token. The `red` job itself +// (the caller) is skipped by name. Any API trouble gives an empty list and a note, never a thrown error: the line is +// the thing that must land. +export async function failedJobs(env = process.env, fetchImpl = fetch) { + const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; const id = env.GITHUB_RUN_ID; + const api = env.GITHUB_API_URL || 'https://api.github.com'; + if (!token) return { failed: [], note: 'no GITHUB_TOKEN; failed steps not read' }; + try { + const r = await fetchImpl(`${api}/repos/${repo}/actions/runs/${id}/jobs?per_page=100`, { + headers: { Authorization: `Bearer ${token}`, Accept: 'application/vnd.github+json', 'User-Agent': 'igneum-red-watch' }, + }); + if (!r.ok) return { failed: [], note: `jobs API ${r.status}` }; + const j = await r.json(); + const failed = []; + for (const job of j.jobs || []) { + if (job.name === (env.GITHUB_JOB_NAME || 'red watcher') || /^red watcher/.test(job.name)) continue; + if (job.conclusion === 'success' || job.conclusion === 'skipped' || job.conclusion === null) continue; + const step = (job.steps || []).find((s) => s.conclusion && s.conclusion !== 'success' && s.conclusion !== 'skipped'); + const zeroSteps = !(job.steps || []).length; + failed.push({ job: job.name, conclusion: job.conclusion, step: step ? step.name : (zeroSteps ? '(job never started: runner or billing)' : '(no step)') }); + } + return { failed, note: '' }; + } catch (e) { + return { failed: [], note: `jobs API: ${e.message}` }; + } +} + +export async function record(file, env = process.env, fetchImpl = fetch, title = '') { + const run = runFromEnv(env); + const existing = readLines(file); + if (existing.some((l) => l.run_id === run.run_id && l.attempt === run.attempt)) { + return { written: false, run }; // one line per run attempt, however many times the job is re-run or retried + } + const { failed, note } = await failedJobs(env, fetchImpl); + const line = { ...run, title: (title || env.RED_WATCH_TITLE || '').slice(0, 100), failed, note }; + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.appendFileSync(file, JSON.stringify(line) + '\n'); + return { written: true, run: line }; +} + +export function formatLine(l) { + const where = l.failed.length ? l.failed.map((f) => `${f.job.replace(/,.*$/, '')} at "${f.step}"`).join('; ') : (l.note || 'no step detail'); + const title = l.title ? ` "${l.title}"` : ''; + return `CI red: ${l.workflow} on ${l.branch} @${l.sha}${title}: ${where} ${l.url}`; +} + +function readCredentials(file) { + if (!fs.existsSync(file)) return {}; + const out = {}; + for (const raw of fs.readFileSync(file, 'utf8').split('\n')) { + const line = raw.trim(); if (!line || line.startsWith('#')) continue; + const i = line.indexOf('='); if (i < 0) continue; + out[line.slice(0, i).trim()] = line.slice(i + 1).trim(); + } + return out; +} + +function readState(file) { try { return JSON.parse(fs.readFileSync(file, 'utf8')); } catch { return { posted: {} }; } } +function writeState(file, state) { fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, JSON.stringify(state, null, 1) + '\n', { mode: 0o600 }); } + +export async function post(file, { live = false, stateFile = STATE_FILE, credFile = CRED_FILE, fetchImpl = fetch, log = console.log } = {}) { + const lines = readLines(file); + const state = readState(stateFile); + const pending = lines.filter((l) => !state.posted[`${l.run_id}.${l.attempt || 1}`]); + if (!pending.length) { log(`ci-red: nothing to post (${lines.length} recorded, all posted)`); return { sent: 0, pending: 0 }; } + const creds = readCredentials(credFile); + const hook = creds[WEBHOOK_KEY]; + let sent = 0; + for (const l of pending) { + const text = formatLine(l); + if (!live) { log(`ci-red (dry run, not sent): ${text}`); continue; } + if (!hook) { log(`ci-red: ${WEBHOOK_KEY} is not in the credentials file; ${pending.length} line(s) wait (the line itself is in ${file})`); return { sent: 0, pending: pending.length, missingKey: true }; } + try { + const r = await fetchImpl(hook, { method: 'POST', headers: { 'Content-Type': 'application/json', 'User-Agent': 'igneum-red-watch' }, + body: JSON.stringify({ username: 'Igneum CI', content: text.slice(0, 1900), allowed_mentions: { parse: [] } }) }); + if (!r.ok && r.status !== 204) { log(`ci-red: the webhook answered ${r.status} for run ${l.run_id}; retried next tick`); continue; } + state.posted[`${l.run_id}.${l.attempt || 1}`] = new Date().toISOString(); sent += 1; + log(`ci-red: posted run ${l.run_id} (${l.workflow} on ${l.branch} @${l.sha})`); + } catch (e) { + log(`ci-red: send failed for run ${l.run_id}: ${e.message.replace(/https?:\/\/\S+/g, '')}; retried next tick`); + } + } + if (live) writeState(stateFile, state); + return { sent, pending: pending.length - sent }; +} + +async function selfTest() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'red-watch-')); + const file = path.join(dir, 'red.jsonl'); const stateFile = path.join(dir, 'posted.json'); const credFile = path.join(dir, 'discord'); + const env = { GITHUB_RUN_ID: '424242', GITHUB_RUN_ATTEMPT: '1', GITHUB_REPOSITORY: 'igneum-network/igneum', GITHUB_REF_NAME: 'master', + GITHUB_SHA: '0f0abc6deadbeef', GITHUB_WORKFLOW: 'ci', GITHUB_TOKEN: 'x', RED_WATCH_TITLE: 'Merge box-work 2bd3bec: the repro script re-stamps its clones' }; + const jobs = { jobs: [ + { name: 'site build, link check, identity grep', conclusion: 'failure', steps: [{ name: 'site build', conclusion: 'success' }, { name: 'identity grep of the public export list', conclusion: 'failure' }] }, + { name: 'igneum-pow tests, igneum-census build', conclusion: 'success', steps: [] }, + { name: 'simulators, quick modes', conclusion: 'failure', steps: [] }, + { name: 'red watcher (master and release-* only)', conclusion: null, steps: [] }, + ] }; + const fakeFetch = async () => ({ ok: true, status: 200, json: async () => jobs }); + const fails = []; + const a = await record(file, env, fakeFetch); const b = await record(file, env, fakeFetch); + if (!a.written || b.written) fails.push('record: the second call for the same run wrote a second line'); + const lines = readLines(file); + if (lines.length !== 1) fails.push(`record: ${lines.length} lines, expected 1`); + if (lines[0].failed.length !== 2) fails.push(`record: ${lines[0].failed.length} failed jobs, expected 2 (the watcher itself and the green job skipped)`); + if (lines[0].failed[0].step !== 'identity grep of the public export list') fails.push('record: the failed step was not the first non-success step'); + if (lines[0].failed[1].step !== '(job never started: runner or billing)') fails.push('record: a job with no steps was not named as never started'); + const text = formatLine(lines[0]); + if (!/^CI red: ci on master @0f0abc6 "Merge box-work/.test(text) || !text.includes('actions/runs/424242')) fails.push(`format: ${text}`); + // post, dry run: prints, sends nothing, marks nothing + let printed = []; const log = (s) => printed.push(s); + const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; }; + await post(file, { live: false, stateFile, credFile, fetchImpl: hookFetch, log }); + if (sends.length !== 0 || !printed.some((s) => s.includes('dry run'))) fails.push('post: the dry run sent or did not print'); + // post, live, no key: says which key is missing, names no URL, sends nothing + fs.writeFileSync(credFile, 'DISCORD_WEBHOOK_NUMBERS=https://discord.example/api/webhooks/1/secret\n', { mode: 0o600 }); + printed = []; + const r0 = await post(file, { live: true, stateFile, credFile, fetchImpl: hookFetch, log }); + if (!r0.missingKey || sends.length !== 0 || !printed.some((s) => s.includes(WEBHOOK_KEY))) fails.push('post: a missing updates key was not reported by name'); + if (printed.some((s) => s.includes('secret'))) fails.push('post: a webhook URL leaked into the log'); + // post, live, with the key: one send with the line, then marked posted; a second pass sends nothing + fs.writeFileSync(credFile, `${WEBHOOK_KEY}=https://discord.example/api/webhooks/2/secret2\n`, { mode: 0o600 }); + printed = []; + const r1 = await post(file, { live: true, stateFile, credFile, fetchImpl: hookFetch, log }); + if (r1.sent !== 1 || sends.length !== 1 || sends[0].body.content !== text) fails.push(`post: expected one send of the line, got ${sends.length}`); + if (sends[0].body.allowed_mentions?.parse?.length !== 0) fails.push('post: mentions are not disabled'); + if (printed.some((s) => s.includes('secret2'))) fails.push('post: the webhook URL leaked into the log'); + const r2 = await post(file, { live: true, stateFile, credFile, fetchImpl: hookFetch, log }); + if (r2.sent !== 0 || sends.length !== 1) fails.push('post: the second pass sent the same run again'); + // a failing webhook leaves the run pending for the next tick + const env2 = { ...env, GITHUB_RUN_ID: '424243' }; + await record(file, env2, fakeFetch); + const badFetch = async () => ({ ok: false, status: 500 }); + const r3 = await post(file, { live: true, stateFile, credFile, fetchImpl: badFetch, log }); + if (r3.sent !== 0 || r3.pending !== 1) fails.push('post: a 500 from the webhook did not keep the run pending'); + fs.rmSync(dir, { recursive: true, force: true }); + if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); } + console.log('self-test passed: one line per run however often record runs; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending'); +} + +const cmd = args[0]; +if (cmd === '--self-test') { + await selfTest(); +} else if (cmd === 'record') { + const file = flag('--file'); if (!file) { console.error('record: --file is required'); process.exit(2); } + const r = await record(file, process.env, fetch, flag('--title') || ''); + console.log(r.written ? `ci-red: recorded ${formatLine(r.run)}` : `ci-red: run ${r.run.run_id} attempt ${r.run.attempt} already recorded`); +} else if (cmd === 'post') { + const file = flag('--file'); if (!file) { console.error('post: --file is required'); process.exit(2); } + await post(file, { live: has('--live') }); +} else { + console.error('usage: red-watch.mjs record --file | post --file [--live] | --self-test'); process.exit(2); +} diff --git a/tools/ci/windows-paths-check.sh b/tools/ci/windows-paths-check.sh new file mode 100755 index 000000000..c32712a33 --- /dev/null +++ b/tools/ci/windows-paths-check.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Every tracked path must be one Windows can hold. 6 October 2026: a screenshot named after an address carried a colon +# (docs/plans/site-ui-3-shots/after/address_igneumdev:qz9h....jpg), actions/checkout on windows-latest failed with +# "invalid path" (git exit 128), and every Windows build of the tree died at the checkout for forty minutes. +# +# Rules (NTFS and the Win32 namespace): +# no : * ? " < > | in a name, and no control character; +# no name ending in a dot or a space; +# no reserved device name as a name or as the stem of one (CON, PRN, AUX, NUL, COM1-9, LPT1-9, any case); +# no path longer than 240 characters (MAX_PATH is 260 and a checkout prefix takes the rest). +# +# tools/ci/windows-paths-check.sh every tracked path (CI, the pre-push gate) +# tools/ci/windows-paths-check.sh --staged the paths being committed (the pre-commit hook) +# tools/ci/windows-paths-check.sh --self-test the rules fire on each bad shape and pass a good one +# Exit 1 with the offending paths listed. +set -euo pipefail + +check_paths() { + # stdin: one path per line. Prints one line per offence. Returns 1 if any. One awk pass (a subprocess per path took + # 23 s over 2,500 files on the Mac; this takes well under a second). + awk ' + function reserved(name, stem) { stem = name; sub(/\..*$/, "", stem); return toupper(stem) ~ /^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])$/ } + { + p = $0; if (p == "") next + if (p ~ /[:*?"<>|]/ || p ~ /[\001-\037\177]/) { print " " p " (a character Windows forbids: one of : * ? \" < > | or a control character)"; bad = 1; next } + if (p ~ /[. ]$/ || p ~ /(^|\/)[^\/]*[. ]\//) { print " " p " (a name ending in a dot or a space)"; bad = 1; next } + if (length(p) > 240) { print " " p " (" length(p) " characters; over 240)"; bad = 1; next } + n = split(p, parts, "/") + for (i = 1; i <= n; i++) if (reserved(parts[i])) { print " " p " (reserved device name " parts[i] ")"; bad = 1; break } + } + END { exit bad ? 1 : 0 }' +} + +if [ "${1:-}" = "--self-test" ]; then + fails=0 + for bad in 'docs/shots/address_igneumdev:qz9h.jpg' 'a/b/what?.md' 'a/trailing./x' 'a/trailing ' 'docs/nul.txt' 'x/COM1' 'x/LpT3.log' "$(printf 'd/%0.s' $(seq 1 125))f.txt"; do + if printf '%s\n' "$bad" | check_paths >/dev/null; then echo "self-test failed: accepted '$bad'"; fails=1; fi + done + for good in 'docs/plans/site-ui-3-shots/after/address_igneumdev-qz9h.jpg' 'tools/ci/windows-paths-check.sh' 'a/console.log' 'a/null.rs' 'a/com10.txt' 'a/.gitignore' 'a/b.c.d'; do + if ! printf '%s\n' "$good" | check_paths >/dev/null; then echo "self-test failed: rejected '$good'"; fails=1; fi + done + [ "$fails" = 0 ] && echo "self-test passed: colon, question mark, trailing dot, trailing space, NUL, COM1, LpT3 and a 250-character path fail; seven ordinary paths pass" + exit $fails +fi + +cd "$(git rev-parse --show-toplevel)" +if [ "${1:-}" = "--staged" ]; then + list="$(git diff --cached --name-only --diff-filter=ACR -z | tr '\0' '\n')"; what="staged paths" +else + list="$(git ls-files -z | tr '\0' '\n')"; what="tracked paths" +fi +if out="$(printf '%s\n' "$list" | check_paths)"; then + echo "windows-paths: every one of $(printf '%s\n' "$list" | grep -c . || true) $what is valid on Windows" +else + echo "windows-paths: these $what cannot exist on Windows (rename them before committing):"; printf '%s\n' "$out"; exit 1 +fi