Merge remote-tracking branch 'origin/master' into ca3-coord
This commit is contained in:
igneum-josh
commit
bdbc154beb
7 changed files with 115 additions and 9 deletions
5
.github/workflows/ci-red.yml
vendored
5
.github/workflows/ci-red.yml
vendored
|
|
@ -17,7 +17,10 @@ jobs:
|
|||
red:
|
||||
name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file)
|
||||
if: ${{ github.event.workflow_run.conclusion == 'failure' }}
|
||||
runs-on: [self-hosted, linux, x64, igneum-build-1]
|
||||
# the label ci-red is on igneum-build-1 only (added through the runners API on 7 October 2026; the default of
|
||||
# RUNNER_LABELS in provision.sh carries it): the record file and the poster (igneum-ci-red.timer, the webhook file)
|
||||
# live on that box, and the pool label igneum-build-1 is shared with igneum-build-2 since the same day
|
||||
runs-on: [self-hosted, linux, x64, ci-red]
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
actions: read # the failed run's jobs API (the first real red run, 21:19Z on 6 October: the default token answered 403 and the line carried no step)
|
||||
|
|
|
|||
37
.github/workflows/ci.yml
vendored
37
.github/workflows/ci.yml
vendored
|
|
@ -8,7 +8,9 @@
|
|||
# local gate and CI cannot drift (6 October 2026: 131 red `ci` runs in three days, 92 of them on master, every one a
|
||||
# tree check that would have failed on the pushing machine in under 25 s; docs/analysis/ci-failures-2026-10-06.md).
|
||||
#
|
||||
# Where it runs: `pow` and `sims` go to the box's runner (igneum-build-1, rustc pinned, sccache read-only, 48 jobs)
|
||||
# Where it runs: `pow` and `sims` go to the self-hosted pool (label igneum-build-1: the runners on igneum-build-1 and, since
|
||||
# 7 October 2026, igneum-build-2, which carries that label too; rustc pinned, sccache read-only) and only when the push
|
||||
# touched code (the `changes` job; a docs-only push skips them)
|
||||
# when the repository variable IGNEUM_CI_RUNNER is `box`, else to ubuntu-latest (docs/plans/ci-self-hosted.md; GitHub
|
||||
# has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The red watcher
|
||||
# is its own workflow, .github/workflows/ci-red.yml (workflow_run, so the copy on master watches every branch's run
|
||||
|
|
@ -26,8 +28,39 @@ on:
|
|||
push:
|
||||
pull_request:
|
||||
jobs:
|
||||
changes:
|
||||
# What the push touched (tools/ci/docs-only-check.sh): a push of documents only (docs/, site/, *.md) skips the two
|
||||
# compile-or-compute jobs below, which read none of those paths, so the self-hosted queue carries only runs that can
|
||||
# change their result (7 October 2026: 31 runs queued on one runner, most of them status-document pushes). The tree
|
||||
# gate (the `site` job) runs on ubuntu-latest for every push. A pull request, a new branch or a force push answers
|
||||
# code=true (no `before` to compare from), as does any error reading the compare API: when in doubt, run.
|
||||
name: what the push touched (docs-only runs skip the Rust and simulator jobs)
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
code: ${{ steps.classify.outputs.code }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: tools/ci
|
||||
- id: classify
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BEFORE: ${{ github.event.before }}
|
||||
AFTER: ${{ github.sha }}
|
||||
REPO: ${{ github.repository }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
if [ "$EVENT" != push ] || [ -z "$BEFORE" ] || [ "$BEFORE" = 0000000000000000000000000000000000000000 ]; then
|
||||
echo "code=true" >> "$GITHUB_OUTPUT"; echo "no base to compare from ($EVENT): the compile jobs run"; exit 0
|
||||
fi
|
||||
files="$(gh api "repos/$REPO/compare/$BEFORE...$AFTER" --paginate --jq '.files[].filename' 2>/dev/null || true)"
|
||||
line="$(printf '%s\n' "$files" | bash tools/ci/docs-only-check.sh)"
|
||||
echo "$line" >> "$GITHUB_OUTPUT"
|
||||
echo "$line: $(printf '%s\n' "$files" | grep -c .) changed path(s) between ${BEFORE:0:8} and ${AFTER:0:8}"
|
||||
pow:
|
||||
name: igneum-pow tests, igneum-census build
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.code == 'true' }}
|
||||
runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
@ -43,6 +76,8 @@ jobs:
|
|||
run: cargo build --release
|
||||
sims:
|
||||
name: simulators, quick modes
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.code == 'true' }}
|
||||
runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ The Mac crashed and rebooted under agent load (about twenty lanes, local cargo t
|
|||
## CI red is stop-the-line (standing rule, 6 October 2026, 22:0x UK)
|
||||
- Whoever's merge turns master or a release-* branch red owns the fix inside 15 minutes or reverts the merge; the red watcher posts every failed run to the hidden updates channel and to /srv/ci-red/red.jsonl on the box (tools/ci/red-watch.mjs); the box's own red builds land in the same file with a class (remote-run.sh pre-flight, kept run logs) and the 09:00 UK digest counts them per class with each class's guard.
|
||||
- The pre-push gate is the same script CI runs: `tools/ci/pre-push.sh` (installed by `tools/ci/install-hooks.sh`; `--hook` before a push to master or release-*, `--ci` in the workflow). A check is added there, never only in ci.yml. On a feature branch the hook runs the light gate: the two structural checks plus the two never-push classes, the no-secrets check and the identity grep (7 October 2026).
|
||||
- After every push, the pushing lane reads its CI conclusion (`gh run list --branch <branch> --limit 1` until it is not in progress, then `gh run view <id> --log-failed` on a red) and owns any red before its next push; a red on any branch is reported to main the moment it is seen. The red watcher posts every failed run on every branch (branch, commit, red check, pushing author). Record: 7 October 2026, 11:26 to 12:47 UK, eight red runs on ca3-v4-node over three gate summaries carrying a 64-hex key, read by nobody.
|
||||
- After every push, the pushing lane reads the conclusion when it lands (`gh run list --branch <branch> --limit 1`; `gh run view <id> --log-failed` on a red) and owns a red before the next push; pushes are never held for it. A red on any branch is reported to main the moment it is seen. The red watcher (.github/workflows/ci-red.yml, from master, every branch) posts each failed run with its branch, commit, red check and pushing author. Record: 7 October 2026, 11:26 to 12:47 UK, eight red runs on ca3-v4-node over three gate summaries carrying a 64-hex key, read by nobody; 31 runs queued on one runner at 13:15 UK, so igneum-build-2 joined the pool and docs-only pushes skip the compile jobs.
|
||||
- Research and operations documents live outside the public export list: name them in `tools/ci/export-exclude.txt` (read by the identity check and by the mirror's sync.sh). What stays in the list is read by the public and must pass the identity grep.
|
||||
- A path Windows cannot hold (colon, trailing dot or space, reserved name, over 240 characters) never enters a commit: the pre-commit hook runs `tools/ci/windows-paths-check.sh --staged`.
|
||||
- Record: docs/analysis/ci-failures-2026-10-06.md (168 non-green runs in three days classified; 126 on master; all but two classes were tree checks that the gate now runs locally first).
|
||||
|
|
|
|||
|
|
@ -69,7 +69,11 @@ RUNNER_VERSION="${RUNNER_VERSION:-2.338.0}" # github.com/actions
|
|||
RUNNER_SHA256="${RUNNER_SHA256:-af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32}" # the release note's linux-x64 line
|
||||
RUNNER_REPO_URL="${RUNNER_REPO_URL:-https://github.com/igneum-network/igneum}"
|
||||
RUNNER_NAME="${RUNNER_NAME:-$BOX_HOSTNAME}"
|
||||
RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1}" # added to the defaults self-hosted, linux, x64
|
||||
RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1,ci-red}" # added to the defaults self-hosted, linux, x64. igneum-build-1 is the POOL label
|
||||
# (every box that takes pow and sims carries it); ci-red marks the one box that
|
||||
# holds the red watcher's record file and poster. A second box: BOX_HOSTNAME=igneum-build-2
|
||||
# RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 (register.sh --host)
|
||||
RUNNER_CPUS="${RUNNER_CPUS:-}" # AllowedCPUs for the runner's service when set (a second box is bounded like a suite: 32 cores, nice 10)
|
||||
RUNNER_JOBS="${RUNNER_JOBS:-48}" # cargo jobs for a CI job: half the box, the agents' builds keep the rest
|
||||
RUNNER_TOKEN="${RUNNER_TOKEN:-}" # a registration token (1 h), from infra/build-server/runner/register.sh over stdin; never logged
|
||||
RUNNER_SCCACHE_PORT="${RUNNER_SCCACHE_PORT:-4227}" # the runner's own sccache server; 4226 is the build user's
|
||||
|
|
@ -465,7 +469,7 @@ step_ufw() {
|
|||
# documentation as remembered on 6 October 2026, the docs host answered 404 to the fetch that evening).
|
||||
runner_env_file() {
|
||||
cat <<EOF
|
||||
# igneum-build-1 (infra/build-server/provision.sh step_runner): the environment every CI job on this runner starts with
|
||||
# $BOX_HOSTNAME (infra/build-server/provision.sh step_runner): the environment every CI job on this runner starts with
|
||||
RUSTC_WRAPPER=/usr/local/bin/sccache
|
||||
SCCACHE_CONF=$RUNNER_HOME/.config/sccache/config
|
||||
SCCACHE_SERVER_PORT=$RUNNER_SCCACHE_PORT
|
||||
|
|
@ -536,7 +540,7 @@ step_runner() {
|
|||
RUNNER_TOKEN="$RUNNER_TOKEN" runuser -u "$RUNNER_USER" -- bash -c "cd '$RUNNER_DIR' && ./config.sh --unattended --replace --url '$RUNNER_REPO_URL' --token \"\$RUNNER_TOKEN\" --name '$RUNNER_NAME' --labels '$RUNNER_LABELS' --work _work" >/dev/null \
|
||||
|| die "runner: config.sh failed (an expired token? register.sh fetches a fresh one)"
|
||||
any=1
|
||||
log "runner: registered as $RUNNER_NAME with labels self-hosted, linux, x64, $RUNNER_LABELS"
|
||||
log "runner: registered as $RUNNER_NAME with labels self-hosted, linux, x64, $RUNNER_LABELS${RUNNER_CPUS:+, AllowedCPUs $RUNNER_CPUS}"
|
||||
fi
|
||||
# 7. the service: GitHub's unit (User=runner, KillMode=process) plus Nice and a restart on failure
|
||||
svc="actions.runner.$(sed -n 's/.*"gitHubUrl": *"https:\/\/github.com\/\([^"]*\)".*/\1/p' "$RUNNER_DIR/.runner" | tr '/' '-').$RUNNER_NAME.service"
|
||||
|
|
@ -552,7 +556,8 @@ step_runner() {
|
|||
rm -f "$tmp"
|
||||
dropin="/etc/systemd/system/$svc.d/igneum.conf"
|
||||
tmp=$(mktemp)
|
||||
printf '# igneum-build-1 (infra/build-server/provision.sh step_runner)\n[Service]\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nRestart=on-failure\nRestartSec=30\n' > "$tmp"
|
||||
printf '# %s (infra/build-server/provision.sh step_runner)\n[Service]\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nRestart=on-failure\nRestartSec=30\n' "$BOX_HOSTNAME" > "$tmp"
|
||||
[ -z "$RUNNER_CPUS" ] || printf 'AllowedCPUs=%s\n' "$RUNNER_CPUS" >> "$tmp"
|
||||
install -d -m 755 "$(dirname "$dropin")"
|
||||
if ! cmp -s "$tmp" "$dropin"; then install -m 644 "$tmp" "$dropin"; systemctl daemon-reload; any=1; fi
|
||||
rm -f "$tmp"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,12 @@
|
|||
# Register (or re-register) the GitHub Actions self-hosted runner on igneum-build-1 from this Mac.
|
||||
# infra/build-server/runner/register.sh fetch a registration token with gh, run provision.sh on the box with it
|
||||
# infra/build-server/runner/register.sh --status list the repository's runners (name, status, labels) and the box's unit
|
||||
# infra/build-server/runner/register.sh --host <ip> another box (7 October 2026, igneum-build-2): the same, against that ip;
|
||||
# BOX_HOSTNAME, RUNNER_NAME, RUNNER_LABELS, RUNNER_CPUS and RUNNER_JOBS
|
||||
# from this shell's environment travel with it (provision.sh would
|
||||
# otherwise rename the box igneum-build-1), e.g.
|
||||
# BOX_HOSTNAME=igneum-build-2 RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 \
|
||||
# infra/build-server/runner/register.sh --host 142.132.249.238
|
||||
#
|
||||
# The token: `gh api -X POST repos/igneum-network/igneum/actions/runners/registration-token` as igneum-josh (the CLAUDE.md gh
|
||||
# rule: that account must be ACTIVE; any other active account fails here before anything is fetched). It is a one-hour
|
||||
|
|
@ -14,7 +20,20 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||
REPO_SLUG="${IGNEUM_GH_REPO:-igneum-network/igneum}"
|
||||
KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
|
||||
HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')"
|
||||
IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server (infra/build-server/run-from-mac.sh writes it)" >&2; exit 1; }
|
||||
IP="${HOST_LINE#*@}"
|
||||
if [ "${1:-}" = --host ]; then
|
||||
IP="${2:-}"; [ -n "$IP" ] || { echo "--host needs an ip" >&2; exit 1; }; shift 2
|
||||
[ -n "${BOX_HOSTNAME:-}" ] || { echo "--host: set BOX_HOSTNAME (provision.sh would otherwise rename the box igneum-build-1)" >&2; exit 1; }
|
||||
fi
|
||||
[ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server (infra/build-server/run-from-mac.sh writes it)" >&2; exit 1; }
|
||||
# the provisioning variables a second box needs, forwarded as assignments in front of the remote shell (values are plain
|
||||
# words: a hostname, a label list, a cpu range, a number; anything else is refused)
|
||||
FWD=""
|
||||
for v in BOX_HOSTNAME RUNNER_NAME RUNNER_LABELS RUNNER_CPUS RUNNER_JOBS; do
|
||||
val="${!v:-}"; [ -n "$val" ] || continue
|
||||
printf '%s' "$val" | grep -qE '^[A-Za-z0-9,._-]+$' || { echo "$v='$val' is not a plain word" >&2; exit 1; }
|
||||
FWD="$FWD $v=$val"
|
||||
done
|
||||
SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 "root@$IP")
|
||||
|
||||
gh_josh() {
|
||||
|
|
@ -43,7 +62,7 @@ echo "token received (not shown); running provision.sh on root@$IP with it (firs
|
|||
# dropped before it is shown (provision.sh never prints it; this is the belt)
|
||||
OUT=$(mktemp); trap 'rm -f "$OUT"' EXIT
|
||||
set +e
|
||||
{ printf '%s\n' "$TOKEN"; cat "$HERE/../provision.sh"; } | "${SSH[@]}" 'IFS= read -r RUNNER_TOKEN; export RUNNER_TOKEN; MODE=provision bash -s' > "$OUT" 2>&1
|
||||
{ printf '%s\n' "$TOKEN"; cat "$HERE/../provision.sh"; } | "${SSH[@]}" "IFS= read -r RUNNER_TOKEN; export RUNNER_TOKEN; MODE=provision$FWD bash -s" > "$OUT" 2>&1
|
||||
RC=$?
|
||||
set -e
|
||||
grep -v -F "$TOKEN" "$OUT" || true
|
||||
|
|
|
|||
43
tools/ci/docs-only-check.sh
Executable file
43
tools/ci/docs-only-check.sh
Executable file
|
|
@ -0,0 +1,43 @@
|
|||
#!/usr/bin/env bash
|
||||
# Did a push touch code, or only documents? The `changes` job of .github/workflows/ci.yml feeds the changed paths on
|
||||
# stdin; the answer is one line, `code=true` or `code=false`. A docs-only push (every path under docs/ or site/, or a
|
||||
# *.md anywhere) skips the two compile-or-compute jobs (igneum-pow tests, the simulators), which read none of those
|
||||
# paths, so the box's one runner queue carries only runs that can change their result (7 October 2026, 13:03Z to
|
||||
# 14:15Z: 31 runs queued on igneum-build-1, most of them status-document pushes, and no lane could read a conclusion).
|
||||
# The tree gate (site build, link check, identity grep, the unit tests) runs on ubuntu-latest for every push as before.
|
||||
#
|
||||
# An empty list (the compare API answered nothing, a new branch, a force push) answers code=true: when in doubt, run.
|
||||
#
|
||||
# printf 'docs/a.md\nsite/x.html\n' | tools/ci/docs-only-check.sh # code=false
|
||||
# printf 'docs/a.md\nigneum-pow/src/lib.rs\n' | tools/ci/docs-only-check.sh # code=true
|
||||
# tools/ci/docs-only-check.sh --self-test
|
||||
set -euo pipefail
|
||||
|
||||
classify() { # stdin: paths, one per line; prints code=true|false
|
||||
local p any=0 code=0
|
||||
while IFS= read -r p; do
|
||||
[ -n "$p" ] || continue
|
||||
any=1
|
||||
case "$p" in
|
||||
docs/*|site/*|*.md) ;;
|
||||
*) code=1 ;;
|
||||
esac
|
||||
done
|
||||
if [ "$any" = 0 ] || [ "$code" = 1 ]; then echo code=true; else echo code=false; fi
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--self-test" ]; then
|
||||
fails=0
|
||||
t() { local want="$1" got; shift; got="$(printf '%b' "$1" | classify)"; [ "$got" = "$want" ] || { echo "self-test failed: paths [$1] gave $got, expected $want"; fails=1; }; }
|
||||
t code=false 'docs/plans/x.md\nsite/index.html\nREADME.md\ndocs/plans/counter-asic-3-gate/a.json\n'
|
||||
t code=true 'docs/plans/x.md\nigneum-pow/src/lib.rs\n'
|
||||
t code=true 'tools/ci/pre-push.sh\n'
|
||||
t code=true 'site/build.mjs\n.github/workflows/ci.yml\n'
|
||||
t code=true 'sim/finality_v2.py\n'
|
||||
t code=true ''
|
||||
t code=true 'docs.rs/x.md\nproto-cuda/host.cu\n'
|
||||
t code=false 'CLAUDE.md\n'
|
||||
[ "$fails" = 0 ] && echo "self-test passed: docs/, site/ and *.md alone answer code=false; any other path, or no path at all, answers code=true"
|
||||
exit $fails
|
||||
fi
|
||||
classify
|
||||
|
|
@ -103,6 +103,7 @@ tree_checks() {
|
|||
run "income per tier: the public table equals its inputs, the schedule arithmetic" bash -c 'node tools/launch/income-tiers.mjs --check && node --test tools/launch/income-tiers.test.mjs'
|
||||
run "hash-origin report: a known-finished day and a known-failed day" node --test tools/observer/hash-origin.test.mjs
|
||||
run "harness summaries never carry a raw 64-hex key (the writer's own redaction and check)" node infra/fast-time/lib/redact-keys.mjs --self-test
|
||||
run "docs-only pushes skip the compile-or-compute CI jobs (the changes job's classifier)" bash tools/ci/docs-only-check.sh --self-test
|
||||
}
|
||||
|
||||
gated_refs() {
|
||||
|
|
|
|||
Loading…
Reference in a new issue