Merge remote-tracking branch 'origin/master' into ca3-v4-node
This commit is contained in:
igneum-labs
commit
3f8479ec27
10 changed files with 215 additions and 51 deletions
45
.github/workflows/ci-red.yml
vendored
Normal file
45
.github/workflows/ci-red.yml
vendored
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
# The red watcher as its own workflow, on workflow_run, so the copy on master watches EVERY branch's ci run whatever
|
||||
# ci.yml that branch carries: GitHub runs a workflow_run workflow from the default branch only, and the branch's own
|
||||
# ci.yml never enters it (7 October 2026: the inline `red` job of ci.yml was conditioned on master and release-*, and
|
||||
# a feature branch would have waited for a merge of master before its reds were posted at all).
|
||||
#
|
||||
# One line per failed run (tools/ci/red-watch.mjs record, idempotent per run attempt) to /srv/ci-red/red.jsonl on the
|
||||
# box; the box's igneum-ci-red.timer posts each new line once to the hidden updates channel, naming the branch, the
|
||||
# commit, the red check and the pushing author. Runs on the box's own runner (not a GitHub-hosted machine: the billing
|
||||
# block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told). Never blocks a
|
||||
# release: it reads the run, writes one line, and ends.
|
||||
name: ci-red
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [ci]
|
||||
types: [completed]
|
||||
jobs:
|
||||
red:
|
||||
name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file)
|
||||
if: ${{ github.event.workflow_run.conclusion == 'failure' }}
|
||||
# the label ci-red is on igneum-build-1 only (added through the runners API on 7 October 2026; the default of
|
||||
# RUNNER_LABELS in provision.sh carries it): the record file and the poster (igneum-ci-red.timer, the webhook file)
|
||||
# live on that box, and the pool label igneum-build-1 is shared with igneum-build-2 since the same day
|
||||
runs-on: [self-hosted, linux, x64, ci-red]
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
actions: read # the failed run's jobs API (the first real red run, 21:19Z on 6 October: the default token answered 403 and the line carried no step)
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: tools/ci
|
||||
- name: record the failed run (one line, the branch, the commit, the failed jobs and their first failed step from the run's own API, the pushing author)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
RED_WATCH_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RED_WATCH_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
|
||||
RED_WATCH_WORKFLOW: ${{ github.event.workflow_run.name }}
|
||||
RED_WATCH_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
RED_WATCH_SHA: ${{ github.event.workflow_run.head_sha }}
|
||||
RED_WATCH_EVENT: ${{ github.event.workflow_run.event }}
|
||||
RED_WATCH_URL: ${{ github.event.workflow_run.html_url }}
|
||||
RED_WATCH_ACTOR: ${{ github.event.workflow_run.actor.login }}
|
||||
RED_WATCH_TITLE: ${{ github.event.workflow_run.head_commit.message }}
|
||||
RED_WATCH_AUTHOR: ${{ github.event.workflow_run.head_commit.author.name }}
|
||||
run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl
|
||||
73
.github/workflows/ci.yml
vendored
73
.github/workflows/ci.yml
vendored
|
|
@ -8,13 +8,16 @@
|
|||
# local gate and CI cannot drift (6 October 2026: 131 red `ci` runs in three days, 92 of them on master, every one a
|
||||
# tree check that would have failed on the pushing machine in under 25 s; docs/analysis/ci-failures-2026-10-06.md).
|
||||
#
|
||||
# Where it runs: `pow` and `sims` go to the box's runner (igneum-build-1, rustc pinned, sccache read-only, 48 jobs)
|
||||
# Where it runs: `pow` and `sims` go to the self-hosted pool (label igneum-build-1: the runners on igneum-build-1 and, since
|
||||
# 7 October 2026, igneum-build-2, which carries that label too; rustc pinned, sccache read-only) and only when the push
|
||||
# touched code (the `changes` job; a docs-only push skips them)
|
||||
# when the repository variable IGNEUM_CI_RUNNER is `box`, else to ubuntu-latest (docs/plans/ci-self-hosted.md; GitHub
|
||||
# has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The `red` job
|
||||
# runs on the box after any failed run on ANY branch and records the failure for the watcher
|
||||
# (tools/ci/red-watch.mjs; infra/build-server/ci-red): one line per run, naming the branch, the commit, the red check
|
||||
# and the pushing author, to the hidden updates channel and to /srv/ci-red/red.jsonl, so nobody opens the Actions page
|
||||
# to learn a branch is red (master and release-* only until 7 October 2026, when eight red runs on ca3-v4-node went unseen).
|
||||
# has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The red watcher
|
||||
# is its own workflow, .github/workflows/ci-red.yml (workflow_run, so the copy on master watches every branch's run
|
||||
# whatever ci.yml that branch carries): one line per failed run, naming the branch, the commit, the red check and the
|
||||
# pushing author, to the hidden updates channel and to /srv/ci-red/red.jsonl (tools/ci/red-watch.mjs;
|
||||
# infra/build-server/ci-red), so nobody opens the Actions page to learn a branch is red (the inline `red` job here
|
||||
# watched master and release-* only until 7 October 2026, when eight red runs on ca3-v4-node went unseen).
|
||||
#
|
||||
# What does not run, on purpose: the node fork (vendor/igneum-node*, a rusty-kaspa fork of about 500 crates with
|
||||
# rocksdb, blst and the execution layer) is gitignored here and too big for the free runners today (a cold build is
|
||||
|
|
@ -25,8 +28,39 @@ on:
|
|||
push:
|
||||
pull_request:
|
||||
jobs:
|
||||
changes:
|
||||
# What the push touched (tools/ci/docs-only-check.sh): a push of documents only (docs/, site/, *.md) skips the two
|
||||
# compile-or-compute jobs below, which read none of those paths, so the self-hosted queue carries only runs that can
|
||||
# change their result (7 October 2026: 31 runs queued on one runner, most of them status-document pushes). The tree
|
||||
# gate (the `site` job) runs on ubuntu-latest for every push. A pull request, a new branch or a force push answers
|
||||
# code=true (no `before` to compare from), as does any error reading the compare API: when in doubt, run.
|
||||
name: what the push touched (docs-only runs skip the Rust and simulator jobs)
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
code: ${{ steps.classify.outputs.code }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: tools/ci
|
||||
- id: classify
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
BEFORE: ${{ github.event.before }}
|
||||
AFTER: ${{ github.sha }}
|
||||
REPO: ${{ github.repository }}
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
if [ "$EVENT" != push ] || [ -z "$BEFORE" ] || [ "$BEFORE" = 0000000000000000000000000000000000000000 ]; then
|
||||
echo "code=true" >> "$GITHUB_OUTPUT"; echo "no base to compare from ($EVENT): the compile jobs run"; exit 0
|
||||
fi
|
||||
files="$(gh api "repos/$REPO/compare/$BEFORE...$AFTER" --paginate --jq '.files[].filename' 2>/dev/null || true)"
|
||||
line="$(printf '%s\n' "$files" | bash tools/ci/docs-only-check.sh)"
|
||||
echo "$line" >> "$GITHUB_OUTPUT"
|
||||
echo "$line: $(printf '%s\n' "$files" | grep -c .) changed path(s) between ${BEFORE:0:8} and ${AFTER:0:8}"
|
||||
pow:
|
||||
name: igneum-pow tests, igneum-census build
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.code == 'true' }}
|
||||
runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
@ -42,6 +76,8 @@ jobs:
|
|||
run: cargo build --release
|
||||
sims:
|
||||
name: simulators, quick modes
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.code == 'true' }}
|
||||
runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
@ -76,28 +112,3 @@ jobs:
|
|||
- name: public stats API answers with the documented fields (the live site; master only, the endpoints exist there after the merge)
|
||||
if: github.ref == 'refs/heads/master'
|
||||
run: node tools/ci/public-api-check.mjs https://igneum.network
|
||||
|
||||
red:
|
||||
# Runs when a run on any branch has a failed job, on the box's own runner (not a GitHub-hosted machine:
|
||||
# the billing block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told).
|
||||
# tools/ci/red-watch.mjs record appends ONE line for this run to /srv/ci-red/red.jsonl (idempotent per run attempt);
|
||||
# the box's igneum-ci-red.timer posts each new line once to the hidden updates channel. Never blocks a release:
|
||||
# it reads the run, writes one line, and ends.
|
||||
name: red watcher (every branch; one line per failed run, with the branch, commit, red check and pushing author, to the updates channel and the box file)
|
||||
needs: [pow, sims, site]
|
||||
if: ${{ failure() }}
|
||||
runs-on: [self-hosted, linux, x64, igneum-build-1]
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
actions: read # the run's jobs API (the first real red run, 21:19Z: the default token answered 403 and the line carried no step)
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
sparse-checkout: tools/ci
|
||||
- name: record this run (one line, the branch, the commit, the failed jobs and their first failed step from the run's own API, the pushing author)
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
RED_WATCH_TITLE: ${{ github.event.head_commit.message }}
|
||||
RED_WATCH_AUTHOR: ${{ github.event.head_commit.author.name }}
|
||||
run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -45,9 +45,9 @@ bs_box_state() { # <box> -> "free=<n> slots=<n> load1=<x>" | "absent" | "down"
|
|||
v=$(eval "printf '%s' \"\${BS_ROUTE_STATE_$b:-}\""); if [ -n "$v" ]; then printf '%s' "$v"; return 0; fi
|
||||
f=$(bs_box_file "$b"); [ -s "$f" ] || { printf 'absent'; return 0; }
|
||||
h="$(head -1 "$f" | tr -d '[:space:]')"
|
||||
# the route check runs before bs_host fills BS_SSH_OPTS; the Mac's bash 3.2 treats the empty array as unbound under
|
||||
# set -u (every lane on a Mac died here at 13:20Z, 7 October 2026), so the key and the batch options are named here
|
||||
ssh -i "${BS_KEY:-$HOME/.ssh/igneum_ed25519}" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 "$h" 'd=/srv/builds/_locks; n=$(cat $d/slots 2>/dev/null || echo 1); free=0; k=0; while [ $k -lt $n ]; do exec 9>>$d/build-$k; if flock -n 9; then free=$((free+1)); fi; exec 9>&-; k=$((k+1)); done; printf "free=%s slots=%s load1=%s" $free $n "$(cut -d" " -f1 /proc/loadavg)"' 2>/dev/null || printf 'down'
|
||||
# the probe runs BEFORE bs_host builds BS_SSH_OPTS (the pool lane, 7 Oct 2026 15:1x UK: bash 3.2 under set -u refuses an
|
||||
# unset array), so it carries its own options: the ops key, batch mode, a short connect timeout, no control socket
|
||||
ssh -i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=8 "$h" 'd=/srv/builds/_locks; n=$(cat $d/slots 2>/dev/null || echo 1); free=0; k=0; while [ $k -lt $n ]; do exec 9>>$d/build-$k; if flock -n 9; then free=$((free+1)); fi; exec 9>&-; k=$((k+1)); done; printf "free=%s slots=%s load1=%s" $free $n "$(cut -d" " -f1 /proc/loadavg)"' 2>/dev/null || printf 'down'
|
||||
}
|
||||
bs_state_ok() { # <state> -> 0 when the box can take a job now (a free slot, load1 at or under the line)
|
||||
local st="$1" free load
|
||||
|
|
|
|||
|
|
@ -69,7 +69,11 @@ RUNNER_VERSION="${RUNNER_VERSION:-2.338.0}" # github.com/actions
|
|||
RUNNER_SHA256="${RUNNER_SHA256:-af4b794c1bc41d73d40535e3fe092a39f9679cd8d965954c2aca25a05ca41d32}" # the release note's linux-x64 line
|
||||
RUNNER_REPO_URL="${RUNNER_REPO_URL:-https://github.com/igneum-network/igneum}"
|
||||
RUNNER_NAME="${RUNNER_NAME:-$BOX_HOSTNAME}"
|
||||
RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1}" # added to the defaults self-hosted, linux, x64
|
||||
RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1,ci-red}" # added to the defaults self-hosted, linux, x64. igneum-build-1 is the POOL label
|
||||
# (every box that takes pow and sims carries it); ci-red marks the one box that
|
||||
# holds the red watcher's record file and poster. A second box: BOX_HOSTNAME=igneum-build-2
|
||||
# RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 (register.sh --host)
|
||||
RUNNER_CPUS="${RUNNER_CPUS:-}" # AllowedCPUs for the runner's service when set (a second box is bounded like a suite: 32 cores, nice 10)
|
||||
RUNNER_JOBS="${RUNNER_JOBS:-48}" # cargo jobs for a CI job: half the box, the agents' builds keep the rest
|
||||
RUNNER_TOKEN="${RUNNER_TOKEN:-}" # a registration token (1 h), from infra/build-server/runner/register.sh over stdin; never logged
|
||||
RUNNER_SCCACHE_PORT="${RUNNER_SCCACHE_PORT:-4227}" # the runner's own sccache server; 4226 is the build user's
|
||||
|
|
@ -465,7 +469,7 @@ step_ufw() {
|
|||
# documentation as remembered on 6 October 2026, the docs host answered 404 to the fetch that evening).
|
||||
runner_env_file() {
|
||||
cat <<EOF
|
||||
# igneum-build-1 (infra/build-server/provision.sh step_runner): the environment every CI job on this runner starts with
|
||||
# $BOX_HOSTNAME (infra/build-server/provision.sh step_runner): the environment every CI job on this runner starts with
|
||||
RUSTC_WRAPPER=/usr/local/bin/sccache
|
||||
SCCACHE_CONF=$RUNNER_HOME/.config/sccache/config
|
||||
SCCACHE_SERVER_PORT=$RUNNER_SCCACHE_PORT
|
||||
|
|
@ -536,7 +540,7 @@ step_runner() {
|
|||
RUNNER_TOKEN="$RUNNER_TOKEN" runuser -u "$RUNNER_USER" -- bash -c "cd '$RUNNER_DIR' && ./config.sh --unattended --replace --url '$RUNNER_REPO_URL' --token \"\$RUNNER_TOKEN\" --name '$RUNNER_NAME' --labels '$RUNNER_LABELS' --work _work" >/dev/null \
|
||||
|| die "runner: config.sh failed (an expired token? register.sh fetches a fresh one)"
|
||||
any=1
|
||||
log "runner: registered as $RUNNER_NAME with labels self-hosted, linux, x64, $RUNNER_LABELS"
|
||||
log "runner: registered as $RUNNER_NAME with labels self-hosted, linux, x64, $RUNNER_LABELS${RUNNER_CPUS:+, AllowedCPUs $RUNNER_CPUS}"
|
||||
fi
|
||||
# 7. the service: GitHub's unit (User=runner, KillMode=process) plus Nice and a restart on failure
|
||||
svc="actions.runner.$(sed -n 's/.*"gitHubUrl": *"https:\/\/github.com\/\([^"]*\)".*/\1/p' "$RUNNER_DIR/.runner" | tr '/' '-').$RUNNER_NAME.service"
|
||||
|
|
@ -552,7 +556,8 @@ step_runner() {
|
|||
rm -f "$tmp"
|
||||
dropin="/etc/systemd/system/$svc.d/igneum.conf"
|
||||
tmp=$(mktemp)
|
||||
printf '# igneum-build-1 (infra/build-server/provision.sh step_runner)\n[Service]\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nRestart=on-failure\nRestartSec=30\n' > "$tmp"
|
||||
printf '# %s (infra/build-server/provision.sh step_runner)\n[Service]\nNice=10\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\nRestart=on-failure\nRestartSec=30\n' "$BOX_HOSTNAME" > "$tmp"
|
||||
[ -z "$RUNNER_CPUS" ] || printf 'AllowedCPUs=%s\n' "$RUNNER_CPUS" >> "$tmp"
|
||||
install -d -m 755 "$(dirname "$dropin")"
|
||||
if ! cmp -s "$tmp" "$dropin"; then install -m 644 "$tmp" "$dropin"; systemctl daemon-reload; any=1; fi
|
||||
rm -f "$tmp"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,12 @@
|
|||
# Register (or re-register) the GitHub Actions self-hosted runner on igneum-build-1 from this Mac.
|
||||
# infra/build-server/runner/register.sh fetch a registration token with gh, run provision.sh on the box with it
|
||||
# infra/build-server/runner/register.sh --status list the repository's runners (name, status, labels) and the box's unit
|
||||
# infra/build-server/runner/register.sh --host <ip> another box (7 October 2026, igneum-build-2): the same, against that ip;
|
||||
# BOX_HOSTNAME, RUNNER_NAME, RUNNER_LABELS, RUNNER_CPUS and RUNNER_JOBS
|
||||
# from this shell's environment travel with it (provision.sh would
|
||||
# otherwise rename the box igneum-build-1), e.g.
|
||||
# BOX_HOSTNAME=igneum-build-2 RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 \
|
||||
# infra/build-server/runner/register.sh --host 142.132.249.238
|
||||
#
|
||||
# The token: `gh api -X POST repos/igneum-network/igneum/actions/runners/registration-token` as igneum-labs (the CLAUDE.md gh
|
||||
# rule: that account must be ACTIVE; any other active account fails here before anything is fetched). It is a one-hour
|
||||
|
|
@ -14,7 +20,20 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||
REPO_SLUG="${IGNEUM_GH_REPO:-igneum-network/igneum}"
|
||||
KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
|
||||
HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')"
|
||||
IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server (infra/build-server/run-from-mac.sh writes it)" >&2; exit 1; }
|
||||
IP="${HOST_LINE#*@}"
|
||||
if [ "${1:-}" = --host ]; then
|
||||
IP="${2:-}"; [ -n "$IP" ] || { echo "--host needs an ip" >&2; exit 1; }; shift 2
|
||||
[ -n "${BOX_HOSTNAME:-}" ] || { echo "--host: set BOX_HOSTNAME (provision.sh would otherwise rename the box igneum-build-1)" >&2; exit 1; }
|
||||
fi
|
||||
[ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server (infra/build-server/run-from-mac.sh writes it)" >&2; exit 1; }
|
||||
# the provisioning variables a second box needs, forwarded as assignments in front of the remote shell (values are plain
|
||||
# words: a hostname, a label list, a cpu range, a number; anything else is refused)
|
||||
FWD=""
|
||||
for v in BOX_HOSTNAME RUNNER_NAME RUNNER_LABELS RUNNER_CPUS RUNNER_JOBS; do
|
||||
val="${!v:-}"; [ -n "$val" ] || continue
|
||||
printf '%s' "$val" | grep -qE '^[A-Za-z0-9,._-]+$' || { echo "$v='$val' is not a plain word" >&2; exit 1; }
|
||||
FWD="$FWD $v=$val"
|
||||
done
|
||||
SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 "root@$IP")
|
||||
|
||||
gh_josh() {
|
||||
|
|
@ -43,7 +62,7 @@ echo "token received (not shown); running provision.sh on root@$IP with it (firs
|
|||
# dropped before it is shown (provision.sh never prints it; this is the belt)
|
||||
OUT=$(mktemp); trap 'rm -f "$OUT"' EXIT
|
||||
set +e
|
||||
{ printf '%s\n' "$TOKEN"; cat "$HERE/../provision.sh"; } | "${SSH[@]}" 'IFS= read -r RUNNER_TOKEN; export RUNNER_TOKEN; MODE=provision bash -s' > "$OUT" 2>&1
|
||||
{ printf '%s\n' "$TOKEN"; cat "$HERE/../provision.sh"; } | "${SSH[@]}" "IFS= read -r RUNNER_TOKEN; export RUNNER_TOKEN; MODE=provision$FWD bash -s" > "$OUT" 2>&1
|
||||
RC=$?
|
||||
set -e
|
||||
grep -v -F "$TOKEN" "$OUT" || true
|
||||
|
|
|
|||
43
tools/ci/docs-only-check.sh
Executable file
43
tools/ci/docs-only-check.sh
Executable file
|
|
@ -0,0 +1,43 @@
|
|||
#!/usr/bin/env bash
|
||||
# Did a push touch code, or only documents? The `changes` job of .github/workflows/ci.yml feeds the changed paths on
|
||||
# stdin; the answer is one line, `code=true` or `code=false`. A docs-only push (every path under docs/ or site/, or a
|
||||
# *.md anywhere) skips the two compile-or-compute jobs (igneum-pow tests, the simulators), which read none of those
|
||||
# paths, so the box's one runner queue carries only runs that can change their result (7 October 2026, 13:03Z to
|
||||
# 14:15Z: 31 runs queued on igneum-build-1, most of them status-document pushes, and no lane could read a conclusion).
|
||||
# The tree gate (site build, link check, identity grep, the unit tests) runs on ubuntu-latest for every push as before.
|
||||
#
|
||||
# An empty list (the compare API answered nothing, a new branch, a force push) answers code=true: when in doubt, run.
|
||||
#
|
||||
# printf 'docs/a.md\nsite/x.html\n' | tools/ci/docs-only-check.sh # code=false
|
||||
# printf 'docs/a.md\nigneum-pow/src/lib.rs\n' | tools/ci/docs-only-check.sh # code=true
|
||||
# tools/ci/docs-only-check.sh --self-test
|
||||
set -euo pipefail
|
||||
|
||||
classify() { # stdin: paths, one per line; prints code=true|false
|
||||
local p any=0 code=0
|
||||
while IFS= read -r p; do
|
||||
[ -n "$p" ] || continue
|
||||
any=1
|
||||
case "$p" in
|
||||
docs/*|site/*|*.md) ;;
|
||||
*) code=1 ;;
|
||||
esac
|
||||
done
|
||||
if [ "$any" = 0 ] || [ "$code" = 1 ]; then echo code=true; else echo code=false; fi
|
||||
}
|
||||
|
||||
if [ "${1:-}" = "--self-test" ]; then
|
||||
fails=0
|
||||
t() { local want="$1" got; shift; got="$(printf '%b' "$1" | classify)"; [ "$got" = "$want" ] || { echo "self-test failed: paths [$1] gave $got, expected $want"; fails=1; }; }
|
||||
t code=false 'docs/plans/x.md\nsite/index.html\nREADME.md\ndocs/plans/counter-asic-3-gate/a.json\n'
|
||||
t code=true 'docs/plans/x.md\nigneum-pow/src/lib.rs\n'
|
||||
t code=true 'tools/ci/pre-push.sh\n'
|
||||
t code=true 'site/build.mjs\n.github/workflows/ci.yml\n'
|
||||
t code=true 'sim/finality_v2.py\n'
|
||||
t code=true ''
|
||||
t code=true 'docs.rs/x.md\nproto-cuda/host.cu\n'
|
||||
t code=false 'CLAUDE.md\n'
|
||||
[ "$fails" = 0 ] && echo "self-test passed: docs/, site/ and *.md alone answer code=false; any other path, or no path at all, answers code=true"
|
||||
exit $fails
|
||||
fi
|
||||
classify
|
||||
|
|
@ -103,6 +103,7 @@ tree_checks() {
|
|||
run "income per tier: the public table equals its inputs, the schedule arithmetic" bash -c 'node tools/launch/income-tiers.mjs --check && node --test tools/launch/income-tiers.test.mjs'
|
||||
run "hash-origin report: a known-finished day and a known-failed day" node --test tools/observer/hash-origin.test.mjs
|
||||
run "harness summaries never carry a raw 64-hex key (the writer's own redaction and check)" node infra/fast-time/lib/redact-keys.mjs --self-test
|
||||
run "docs-only pushes skip the compile-or-compute CI jobs (the changes job's classifier)" bash tools/ci/docs-only-check.sh --self-test
|
||||
}
|
||||
|
||||
gated_refs() {
|
||||
|
|
|
|||
|
|
@ -4,10 +4,12 @@
|
|||
// opens the Actions page to learn a branch is red.
|
||||
// Node 22, standard library only.
|
||||
//
|
||||
// node tools/ci/red-watch.mjs record --file <red.jsonl> in the workflow's `red` job (runs on igneum-build-1 after a
|
||||
// failed run): reads the run from the GitHub environment and
|
||||
// node tools/ci/red-watch.mjs record --file <red.jsonl> in .github/workflows/ci-red.yml (a workflow_run job on
|
||||
// igneum-build-1 after a failed ci run on any branch): reads the
|
||||
// FAILED run from RED_WATCH_* (the workflow_run payload; the
|
||||
// GITHUB_* variables there describe the watcher's own run) and
|
||||
// the failed jobs and steps from the API with the job's own
|
||||
// token, appends ONE JSON line for this run id (idempotent)
|
||||
// token, appends ONE JSON line for that run id (idempotent)
|
||||
// node tools/ci/red-watch.mjs post --file <red.jsonl> [--live] on the box, every minute as `build` (igneum-ci-red.timer):
|
||||
// every recorded run not yet posted goes as one line to the
|
||||
// hidden updates channel (DISCORD_WEBHOOK_UPDATES in the
|
||||
|
|
@ -63,15 +65,22 @@ export function readLines(file) {
|
|||
return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
||||
}
|
||||
|
||||
// The run being recorded: the FAILED run from RED_WATCH_* when the watcher runs as a workflow_run job (ci-red.yml), else
|
||||
// the job's own run from GITHUB_* (the inline shape, kept for a branch whose ci.yml still carries the old `red` job).
|
||||
export const watchedRunId = (env = process.env) => env.RED_WATCH_RUN_ID || env.GITHUB_RUN_ID;
|
||||
export function runFromEnv(env = process.env) {
|
||||
const need = ['GITHUB_RUN_ID', 'GITHUB_REPOSITORY', 'GITHUB_REF_NAME', 'GITHUB_SHA', 'GITHUB_WORKFLOW'];
|
||||
const need = ['GITHUB_REPOSITORY', 'GITHUB_RUN_ID'];
|
||||
for (const k of need) if (!env[k]) throw new Error(`record: ${k} is not set (this command runs inside a GitHub Actions job)`);
|
||||
const pick = (own, fallback) => env[own] || env[fallback] || '';
|
||||
const server = env.GITHUB_SERVER_URL || 'https://github.com';
|
||||
const id = String(watchedRunId(env));
|
||||
const sha = pick('RED_WATCH_SHA', 'GITHUB_SHA');
|
||||
if (!sha) throw new Error('record: neither RED_WATCH_SHA nor GITHUB_SHA is set');
|
||||
return {
|
||||
run_id: String(env.GITHUB_RUN_ID), attempt: Number(env.GITHUB_RUN_ATTEMPT || 1), workflow: env.GITHUB_WORKFLOW,
|
||||
branch: env.GITHUB_REF_NAME, sha: env.GITHUB_SHA.slice(0, 7), event: env.GITHUB_EVENT_NAME || '',
|
||||
actor: env.GITHUB_ACTOR || '', author: env.RED_WATCH_AUTHOR || '', // who pushed (the GitHub login), who the head commit names
|
||||
url: `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`, at: new Date().toISOString(),
|
||||
run_id: id, attempt: Number(pick('RED_WATCH_ATTEMPT', 'GITHUB_RUN_ATTEMPT') || 1), workflow: pick('RED_WATCH_WORKFLOW', 'GITHUB_WORKFLOW'),
|
||||
branch: pick('RED_WATCH_BRANCH', 'GITHUB_REF_NAME'), sha: sha.slice(0, 7), event: pick('RED_WATCH_EVENT', 'GITHUB_EVENT_NAME'),
|
||||
actor: pick('RED_WATCH_ACTOR', 'GITHUB_ACTOR'), author: env.RED_WATCH_AUTHOR || '', // who pushed (the GitHub login), who the head commit names
|
||||
url: env.RED_WATCH_URL || `${server}/${env.GITHUB_REPOSITORY}/actions/runs/${id}`, at: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -79,7 +88,7 @@ export function runFromEnv(env = process.env) {
|
|||
// (the caller) is skipped by name. Any API trouble gives an empty list and a note, never a thrown error: the line is
|
||||
// the thing that must land.
|
||||
export async function failedJobs(env = process.env, fetchImpl = fetch) {
|
||||
const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; const id = env.GITHUB_RUN_ID;
|
||||
const token = env.GITHUB_TOKEN; const repo = env.GITHUB_REPOSITORY; const id = watchedRunId(env);
|
||||
const api = env.GITHUB_API_URL || 'https://api.github.com';
|
||||
if (!token) return { failed: [], note: 'no GITHUB_TOKEN; failed steps not read' };
|
||||
try {
|
||||
|
|
@ -238,6 +247,18 @@ async function selfTest() {
|
|||
await record(fileFeature, envFeature, fakeFetch);
|
||||
const textFeature = formatLine(readLines(fileFeature)[0]);
|
||||
if (!/^CI red: ci on ca3-v4-node @0f0abc6 "Merge box-work[^"]*": pushed by igneum-labs; site build at/.test(textFeature)) fails.push(`format on a feature branch: ${textFeature}`);
|
||||
// the workflow_run shape (ci-red.yml): GITHUB_* describe the watcher's own run, RED_WATCH_* the failed one; the line is the failed run's
|
||||
const envRun = { GITHUB_RUN_ID: '999', GITHUB_RUN_ATTEMPT: '1', GITHUB_REPOSITORY: 'igneum-network/igneum', GITHUB_REF_NAME: 'master', GITHUB_SHA: 'ffffffffffff', GITHUB_WORKFLOW: 'ci-red', GITHUB_ACTOR: 'igneum-labs', GITHUB_TOKEN: 'x',
|
||||
RED_WATCH_RUN_ID: '37620364667', RED_WATCH_ATTEMPT: '2', RED_WATCH_WORKFLOW: 'ci', RED_WATCH_BRANCH: 'ca3-v4-node', RED_WATCH_SHA: '26a4b0f1deadbeef', RED_WATCH_EVENT: 'push',
|
||||
RED_WATCH_URL: 'https://github.com/igneum-network/igneum/actions/runs/37620364667', RED_WATCH_ACTOR: 'igneum-labs', RED_WATCH_TITLE: 'Counter ASIC 3.0 node plan 6.7', RED_WATCH_AUTHOR: 'igneum-labs' };
|
||||
const fileRun = path.join(dir, 'workflow-run.jsonl'); const asked = [];
|
||||
const askingFetch = async (url) => { asked.push(url); return { ok: true, status: 200, json: async () => jobs }; };
|
||||
await record(fileRun, envRun, askingFetch);
|
||||
const lr = readLines(fileRun)[0];
|
||||
if (lr.run_id !== '37620364667' || lr.attempt !== 2 || lr.branch !== 'ca3-v4-node' || lr.sha !== '26a4b0f' || lr.workflow !== 'ci') fails.push(`workflow_run shape: recorded ${JSON.stringify({ run_id: lr.run_id, attempt: lr.attempt, branch: lr.branch, sha: lr.sha, workflow: lr.workflow })}, expected the failed run, not the watcher's`);
|
||||
if (!asked[0] || !asked[0].includes('/actions/runs/37620364667/jobs')) fails.push(`workflow_run shape: the jobs API was asked for ${asked[0]}, not the failed run`);
|
||||
const textRun = formatLine(lr);
|
||||
if (!/^CI red: ci on ca3-v4-node @26a4b0f "Counter ASIC 3.0 node plan 6.7": pushed by igneum-labs \(commit by igneum-labs\); site build at "identity grep of the public export list"; simulators at "\(job never started: runner or billing\)" https:\/\/github.com\/igneum-network\/igneum\/actions\/runs\/37620364667$/.test(textRun)) fails.push(`workflow_run line: ${textRun}`);
|
||||
// post, dry run: prints, sends nothing, marks nothing
|
||||
let printed = []; const log = (s) => printed.push(s);
|
||||
const sends = []; const hookFetch = async (url, init) => { sends.push({ url, body: JSON.parse(init.body) }); return { ok: true, status: 204 }; };
|
||||
|
|
@ -286,7 +307,7 @@ async function selfTest() {
|
|||
if (!d3.sent) fails.push('digest: not sent the next day');
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
if (fails.length) { for (const f of fails) console.error(`self-test failed: ${f}`); process.exit(1); }
|
||||
console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
|
||||
console.log('self-test passed: one line per run however often record runs, on any branch, naming the pushing author, the failed run and not the watcher\'s own under workflow_run; the dry run sends nothing; a missing key is named, never a URL; one live send per run; a webhook error keeps the run pending; a box row is counted, never posted alone; the digest goes once per London day, at or after 09:00');
|
||||
}
|
||||
|
||||
const cmd = args[0];
|
||||
|
|
|
|||
|
|
@ -33,5 +33,12 @@ BS_ROUTE_STATE_1="down" BS_ROUTE_STATE_2="free=3 slots=3 load
|
|||
BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a proving class with no box 3 prefers box 1" prove 1 0
|
||||
BS_ROUTE_STATE_1="free=0 slots=2 load1=5" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a proving class spills to box 2 when box 1 is held" prove 2 1
|
||||
BS_ROUTE_STATE_1="free=1 slots=2 load1=64" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "load exactly 64 is under the line" build 1 0
|
||||
# the ssh path itself under the hook's shell (/bin/bash is 3.2 on the Mac; the pool lane found `BS_SSH_OPTS[@]: unbound variable`
|
||||
# at the first unpinned route, 7 Oct 2026): a host file pointing at a port nothing answers on must read "down" in a few seconds,
|
||||
# not die on an unset array
|
||||
printf 'build@127.0.0.1\n' > "$t/hosts"
|
||||
out=$(IGNEUM_BUILD_KEY="$t/no-such-key" /bin/bash -c '. infra/build-server/lib.sh; bs_box_state 1' 2>&1 || true)
|
||||
if [ "$out" = down ]; then echo "route-spill: the ssh probe under /bin/bash $(/bin/bash -c 'echo $BASH_VERSION') reads an unreachable box as down"
|
||||
else echo "route-spill: the ssh probe under /bin/bash failed: '$out' (expected 'down')" >&2; fail=1; fi
|
||||
[ "$fail" = 0 ] && echo "route-spill: the class is a preference; a full or overloaded box hands the job to the other one"
|
||||
exit $fail
|
||||
|
|
|
|||
Loading…
Reference in a new issue