diff --git a/.github/workflows/ci-red.yml b/.github/workflows/ci-red.yml index dddb758b0..48b3deff7 100644 --- a/.github/workflows/ci-red.yml +++ b/.github/workflows/ci-red.yml @@ -17,7 +17,10 @@ jobs: red: name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file) if: ${{ github.event.workflow_run.conclusion == 'failure' }} - runs-on: [self-hosted, linux, x64, igneum-build-1] + # the label ci-red is on igneum-build-1 only (added through the runners API on 7 October 2026; the default of + # RUNNER_LABELS in provision.sh carries it): the record file and the poster (igneum-ci-red.timer, the webhook file) + # live on that box, and the pool label igneum-build-1 is shared with igneum-build-2 since the same day + runs-on: [self-hosted, linux, x64, ci-red] timeout-minutes: 5 permissions: actions: read # the failed run's jobs API (the first real red run, 21:19Z on 6 October: the default token answered 403 and the line carried no step) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5311c6dc0..4b3570203 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,9 @@ # local gate and CI cannot drift (6 October 2026: 131 red `ci` runs in three days, 92 of them on master, every one a # tree check that would have failed on the pushing machine in under 25 s; docs/analysis/ci-failures-2026-10-06.md). # -# Where it runs: `pow` and `sims` go to the box's runner (igneum-build-1, rustc pinned, sccache read-only, 48 jobs) +# Where it runs: `pow` and `sims` go to the self-hosted pool (label igneum-build-1: the runners on igneum-build-1 and, since +# 7 October 2026, igneum-build-2, which carries that label too; rustc pinned, sccache read-only) and only when the push +# touched code (the `changes` job; a docs-only push skips them) # when the repository variable IGNEUM_CI_RUNNER is `box`, else to ubuntu-latest (docs/plans/ci-self-hosted.md; GitHub # has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The red watcher # is its own workflow, .github/workflows/ci-red.yml (workflow_run, so the copy on master watches every branch's run @@ -26,8 +28,39 @@ on: push: pull_request: jobs: + changes: + # What the push touched (tools/ci/docs-only-check.sh): a push of documents only (docs/, site/, *.md) skips the two + # compile-or-compute jobs below, which read none of those paths, so the self-hosted queue carries only runs that can + # change their result (7 October 2026: 31 runs queued on one runner, most of them status-document pushes). The tree + # gate (the `site` job) runs on ubuntu-latest for every push. A pull request, a new branch or a force push answers + # code=true (no `before` to compare from), as does any error reading the compare API: when in doubt, run. + name: what the push touched (docs-only runs skip the Rust and simulator jobs) + runs-on: ubuntu-latest + outputs: + code: ${{ steps.classify.outputs.code }} + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: tools/ci + - id: classify + env: + GH_TOKEN: ${{ github.token }} + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.sha }} + REPO: ${{ github.repository }} + EVENT: ${{ github.event_name }} + run: | + if [ "$EVENT" != push ] || [ -z "$BEFORE" ] || [ "$BEFORE" = 0000000000000000000000000000000000000000 ]; then + echo "code=true" >> "$GITHUB_OUTPUT"; echo "no base to compare from ($EVENT): the compile jobs run"; exit 0 + fi + files="$(gh api "repos/$REPO/compare/$BEFORE...$AFTER" --paginate --jq '.files[].filename' 2>/dev/null || true)" + line="$(printf '%s\n' "$files" | bash tools/ci/docs-only-check.sh)" + echo "$line" >> "$GITHUB_OUTPUT" + echo "$line: $(printf '%s\n' "$files" | grep -c .) changed path(s) between ${BEFORE:0:8} and ${AFTER:0:8}" pow: name: igneum-pow tests, igneum-census build + needs: changes + if: ${{ needs.changes.outputs.code == 'true' }} runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 @@ -43,6 +76,8 @@ jobs: run: cargo build --release sims: name: simulators, quick modes + needs: changes + if: ${{ needs.changes.outputs.code == 'true' }} runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 diff --git a/infra/build-server/provision.sh b/infra/build-server/provision.sh index 5ec888592..54082d2f5 100755 --- a/infra/build-server/provision.sh +++ b/infra/build-server/provision.sh @@ -69,7 +69,11 @@ RUNNER_VERSION="${RUNNER_VERSION:-2.338.0}" # github.com/actions RUNNER_SHA256="${RUNNER_SHA256:-af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32}" # the release note's linux-x64 line RUNNER_REPO_URL="${RUNNER_REPO_URL:-https://github.com/igneum-network/igneum}" RUNNER_NAME="${RUNNER_NAME:-$BOX_HOSTNAME}" -RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1}" # added to the defaults self-hosted, linux, x64 +RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1,ci-red}" # added to the defaults self-hosted, linux, x64. igneum-build-1 is the POOL label + # (every box that takes pow and sims carries it); ci-red marks the one box that + # holds the red watcher's record file and poster. A second box: BOX_HOSTNAME=igneum-build-2 + # RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 (register.sh --host) +RUNNER_CPUS="${RUNNER_CPUS:-}" # AllowedCPUs for the runner's service when set (a second box is bounded like a suite: 32 cores, nice 10) RUNNER_JOBS="${RUNNER_JOBS:-48}" # cargo jobs for a CI job: half the box, the agents' builds keep the rest RUNNER_TOKEN="${RUNNER_TOKEN:-}" # a registration token (1 h), from infra/build-server/runner/register.sh over stdin; never logged RUNNER_SCCACHE_PORT="${RUNNER_SCCACHE_PORT:-4227}" # the runner's own sccache server; 4226 is the build user's @@ -465,7 +469,7 @@ step_ufw() { # documentation as remembered on 6 October 2026, the docs host answered 404 to the fetch that evening). runner_env_file() { cat </dev/null \ || die "runner: config.sh failed (an expired token? register.sh fetches a fresh one)" any=1 - log "runner: registered as $RUNNER_NAME with labels self-hosted, linux, x64, $RUNNER_LABELS" + log "runner: registered as $RUNNER_NAME with labels self-hosted, linux, x64, $RUNNER_LABELS${RUNNER_CPUS:+, AllowedCPUs $RUNNER_CPUS}" fi # 7. the service: GitHub's unit (User=runner, KillMode=process) plus Nice and a restart on failure svc="actions.runner.$(sed -n 's/.*"gitHubUrl": *"https:\/\/github.com\/\([^"]*\)".*/\1/p' "$RUNNER_DIR/.runner" | tr '/' '-').$RUNNER_NAME.service" @@ -552,7 +556,8 @@ step_runner() { rm -f "$tmp" dropin="/etc/systemd/system/$svc.d/igneum.conf" tmp=$(mktemp) - printf '# igneum-build-1 (infra/build-server/provision.sh step_runner)\n[Service]\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nRestart=on-failure\nRestartSec=30\n' > "$tmp" + printf '# %s (infra/build-server/provision.sh step_runner)\n[Service]\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nRestart=on-failure\nRestartSec=30\n' "$BOX_HOSTNAME" > "$tmp" + [ -z "$RUNNER_CPUS" ] || printf 'AllowedCPUs=%s\n' "$RUNNER_CPUS" >> "$tmp" install -d -m 755 "$(dirname "$dropin")" if ! cmp -s "$tmp" "$dropin"; then install -m 644 "$tmp" "$dropin"; systemctl daemon-reload; any=1; fi rm -f "$tmp" diff --git a/infra/build-server/runner/register.sh b/infra/build-server/runner/register.sh index b27e20e66..056b6030c 100755 --- a/infra/build-server/runner/register.sh +++ b/infra/build-server/runner/register.sh @@ -2,6 +2,12 @@ # Register (or re-register) the GitHub Actions self-hosted runner on igneum-build-1 from this Mac. # infra/build-server/runner/register.sh fetch a registration token with gh, run provision.sh on the box with it # infra/build-server/runner/register.sh --status list the repository's runners (name, status, labels) and the box's unit +# infra/build-server/runner/register.sh --host another box (7 October 2026, igneum-build-2): the same, against that ip; +# BOX_HOSTNAME, RUNNER_NAME, RUNNER_LABELS, RUNNER_CPUS and RUNNER_JOBS +# from this shell's environment travel with it (provision.sh would +# otherwise rename the box igneum-build-1), e.g. +# BOX_HOSTNAME=igneum-build-2 RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 \ +# infra/build-server/runner/register.sh --host 142.132.249.238 # # The token: `gh api -X POST repos/igneum-network/igneum/actions/runners/registration-token` as igneum-labs (the CLAUDE.md gh # rule: that account must be ACTIVE; any other active account fails here before anything is fetched). It is a one-hour @@ -14,7 +20,20 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_SLUG="${IGNEUM_GH_REPO:-igneum-network/igneum}" KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}" HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')" -IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server (infra/build-server/run-from-mac.sh writes it)" >&2; exit 1; } +IP="${HOST_LINE#*@}" +if [ "${1:-}" = --host ]; then + IP="${2:-}"; [ -n "$IP" ] || { echo "--host needs an ip" >&2; exit 1; }; shift 2 + [ -n "${BOX_HOSTNAME:-}" ] || { echo "--host: set BOX_HOSTNAME (provision.sh would otherwise rename the box igneum-build-1)" >&2; exit 1; } +fi +[ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server (infra/build-server/run-from-mac.sh writes it)" >&2; exit 1; } +# the provisioning variables a second box needs, forwarded as assignments in front of the remote shell (values are plain +# words: a hostname, a label list, a cpu range, a number; anything else is refused) +FWD="" +for v in BOX_HOSTNAME RUNNER_NAME RUNNER_LABELS RUNNER_CPUS RUNNER_JOBS; do + val="${!v:-}"; [ -n "$val" ] || continue + printf '%s' "$val" | grep -qE '^[A-Za-z0-9,._-]+$' || { echo "$v='$val' is not a plain word" >&2; exit 1; } + FWD="$FWD $v=$val" +done SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 "root@$IP") gh_josh() { @@ -43,7 +62,7 @@ echo "token received (not shown); running provision.sh on root@$IP with it (firs # dropped before it is shown (provision.sh never prints it; this is the belt) OUT=$(mktemp); trap 'rm -f "$OUT"' EXIT set +e -{ printf '%s\n' "$TOKEN"; cat "$HERE/../provision.sh"; } | "${SSH[@]}" 'IFS= read -r RUNNER_TOKEN; export RUNNER_TOKEN; MODE=provision bash -s' > "$OUT" 2>&1 +{ printf '%s\n' "$TOKEN"; cat "$HERE/../provision.sh"; } | "${SSH[@]}" "IFS= read -r RUNNER_TOKEN; export RUNNER_TOKEN; MODE=provision$FWD bash -s" > "$OUT" 2>&1 RC=$? set -e grep -v -F "$TOKEN" "$OUT" || true diff --git a/tools/ci/docs-only-check.sh b/tools/ci/docs-only-check.sh new file mode 100755 index 000000000..3c5b0906a --- /dev/null +++ b/tools/ci/docs-only-check.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Did a push touch code, or only documents? The `changes` job of .github/workflows/ci.yml feeds the changed paths on +# stdin; the answer is one line, `code=true` or `code=false`. A docs-only push (every path under docs/ or site/, or a +# *.md anywhere) skips the two compile-or-compute jobs (igneum-pow tests, the simulators), which read none of those +# paths, so the box's one runner queue carries only runs that can change their result (7 October 2026, 13:03Z to +# 14:15Z: 31 runs queued on igneum-build-1, most of them status-document pushes, and no lane could read a conclusion). +# The tree gate (site build, link check, identity grep, the unit tests) runs on ubuntu-latest for every push as before. +# +# An empty list (the compare API answered nothing, a new branch, a force push) answers code=true: when in doubt, run. +# +# printf 'docs/a.md\nsite/x.html\n' | tools/ci/docs-only-check.sh # code=false +# printf 'docs/a.md\nigneum-pow/src/lib.rs\n' | tools/ci/docs-only-check.sh # code=true +# tools/ci/docs-only-check.sh --self-test +set -euo pipefail + +classify() { # stdin: paths, one per line; prints code=true|false + local p any=0 code=0 + while IFS= read -r p; do + [ -n "$p" ] || continue + any=1 + case "$p" in + docs/*|site/*|*.md) ;; + *) code=1 ;; + esac + done + if [ "$any" = 0 ] || [ "$code" = 1 ]; then echo code=true; else echo code=false; fi +} + +if [ "${1:-}" = "--self-test" ]; then + fails=0 + t() { local want="$1" got; shift; got="$(printf '%b' "$1" | classify)"; [ "$got" = "$want" ] || { echo "self-test failed: paths [$1] gave $got, expected $want"; fails=1; }; } + t code=false 'docs/plans/x.md\nsite/index.html\nREADME.md\ndocs/plans/counter-asic-3-gate/a.json\n' + t code=true 'docs/plans/x.md\nigneum-pow/src/lib.rs\n' + t code=true 'tools/ci/pre-push.sh\n' + t code=true 'site/build.mjs\n.github/workflows/ci.yml\n' + t code=true 'sim/finality_v2.py\n' + t code=true '' + t code=true 'docs.rs/x.md\nproto-cuda/host.cu\n' + t code=false 'CLAUDE.md\n' + [ "$fails" = 0 ] && echo "self-test passed: docs/, site/ and *.md alone answer code=false; any other path, or no path at all, answers code=true" + exit $fails +fi +classify diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh index 016baca3b..aafefb760 100755 --- a/tools/ci/pre-push.sh +++ b/tools/ci/pre-push.sh @@ -103,6 +103,7 @@ tree_checks() { run "income per tier: the public table equals its inputs, the schedule arithmetic" bash -c 'node tools/launch/income-tiers.mjs --check && node --test tools/launch/income-tiers.test.mjs' run "hash-origin report: a known-finished day and a known-failed day" node --test tools/observer/hash-origin.test.mjs run "harness summaries never carry a raw 64-hex key (the writer's own redaction and check)" node infra/fast-time/lib/redact-keys.mjs --self-test + run "docs-only pushes skip the compile-or-compute CI jobs (the changes job's classifier)" bash tools/ci/docs-only-check.sh --self-test } gated_refs() {