CI and boxes: the simulators job runs on master and release-* pushes and pull requests into them only (a feature-branch code push runs the igneum-pow tests alone; tools/ci/sims-branch-check.sh); the box lock self-tests run in parallel in one ssh; build-2 and build-3 join the runner pool bounded to 32 cores; the 0.3.20 first-wave script (seeds, hands, RPC-filter lift; dry run by default)
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
parent
739297d622
commit
71eb5b4ca9
6 changed files with 158 additions and 4 deletions
4
.github/workflows/ci.yml
vendored
4
.github/workflows/ci.yml
vendored
|
|
@ -77,7 +77,9 @@ jobs:
|
|||
sims:
|
||||
name: simulators, quick modes
|
||||
needs: changes
|
||||
if: ${{ needs.changes.outputs.code == 'true' }}
|
||||
# master and release-* pushes, and pull requests into them, only (main, 7 October 2026: every code push cost two box jobs and the
|
||||
# queue read 22); a feature-branch code push runs the igneum-pow tests alone. tools/ci/sims-branch-check.sh holds this rule.
|
||||
if: ${{ needs.changes.outputs.code == 'true' && ((github.event_name == 'push' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))) || (github.event_name == 'pull_request' && (github.base_ref == 'master' || startsWith(github.base_ref, 'release-')))) }}
|
||||
runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ RUNNER_LABELS="${RUNNER_LABELS:-igneum-build-1,ci-red}" # added to the defau
|
|||
# holds the red watcher's record file and poster. A second box: BOX_HOSTNAME=igneum-build-2
|
||||
# RUNNER_LABELS=igneum-build-1,igneum-build-2 RUNNER_CPUS=0-31 RUNNER_JOBS=32 (register.sh --host)
|
||||
RUNNER_CPUS="${RUNNER_CPUS:-}" # AllowedCPUs for the runner's service when set (a second box is bounded like a suite: 32 cores, nice 10)
|
||||
case "$BOX_HOSTNAME" in *-2|*-3) [ -n "$RUNNER_CPUS" ] || RUNNER_CPUS="0-31" ;; esac # build-2 and build-3 join the pool (label igneum-build-1) bounded to 32 cores at Nice 10 (main, 7 Oct 2026)
|
||||
RUNNER_JOBS="${RUNNER_JOBS:-48}" # cargo jobs for a CI job: half the box, the agents' builds keep the rest
|
||||
RUNNER_TOKEN="${RUNNER_TOKEN:-}" # a registration token (1 h), from infra/build-server/runner/register.sh over stdin; never logged
|
||||
RUNNER_SCCACHE_PORT="${RUNNER_SCCACHE_PORT:-4227}" # the runner's own sccache server; 4226 is the build user's
|
||||
|
|
|
|||
114
infra/build-server/wave1-0320.sh
Executable file
114
infra/build-server/wave1-0320.sh
Executable file
|
|
@ -0,0 +1,114 @@
|
|||
#!/usr/bin/env bash
|
||||
# The 0.3.20 FIRST wave (the shipper, 7 October 2026): the publish carries a floor-moved override file whose digest differs from
|
||||
# the live one, and a node on the old file refuses a node on the new one as a peer, so the three testnet seeds and the hands move
|
||||
# AT the publish minute, in parallel with the fleet's wave. Dry run by default (every line printed, no box touched); --go executes.
|
||||
#
|
||||
# infra/build-server/wave1-0320.sh seeds --override <file> --igneumd <seed-class igneumd> --sha256 <hex> [--miner <seed-class igneum-miner>] \
|
||||
# --digest <hex> [--commit c4459193] [--go]
|
||||
# the three seeds in PARALLEL (seed1/2/3.testnet, root over the ops key, infra/seed-nodes/seeds-testnet.tsv): the binary put
|
||||
# as /opt/igneum/bin/igneumd.new with its sha256 asserted on the box, the override written to /etc/igneum/override-params.json,
|
||||
# EXTRA_ARGS in /etc/igneum/seed.env set to --override-params-file=<that>, the unit igneumd stopped, the binary swapped (the old
|
||||
# one kept as igneumd.prev), the unit started on its kept datadir, then the read-back: commit string in the installed binary,
|
||||
# the "Consensus params digest" line of the new run against --digest, eth_syncing over the loopback EVM RPC, peers; one line
|
||||
# per seed, logs in the scratch directory
|
||||
# infra/build-server/wave1-0320.sh hands --node <fork worktree on the Mac> --override-json '<object>' --digest <hex> [--go]
|
||||
# the hands through infra/build-server/hands/move-hand.sh: binary (build + install + override), restart observer-node, restart
|
||||
# node1, each read back by that tool (first executing line, commit string, digest, powEngine)
|
||||
# infra/build-server/wave1-0320.sh lift-rpc-filter [--go]
|
||||
# on seed1 only, AFTER the three read-backs: BLOCKED_UNTIL_FIXED_NODE in /opt/igneum/bin/rpc-filter.py becomes the empty set
|
||||
# (the N7 class is fixed from 8097d600; backup kept as rpc-filter.py.bak-<stamp>), python3 -m py_compile, the unit
|
||||
# igneum-rpc-filter restarted, read back: unit active and eth_blockNumber answered through https://rpc.testnet.igneum.network
|
||||
set -euo pipefail
|
||||
HERE=$(cd "$(dirname "$0")" && pwd); ROOT=$(cd "$HERE/../.." && pwd)
|
||||
SEEDS_TSV="$ROOT/infra/seed-nodes/seeds-testnet.tsv"; KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"
|
||||
S="${WAVE_LOG_DIR:-/private/tmp/claude-501/-Users-joshm/cd75457f-4858-4f86-9634-7481ee056b7b/scratchpad/wave1-0320}"; mkdir -p "$S"
|
||||
SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15)
|
||||
now() { TZ=Europe/London date '+%H:%M:%S %Z'; }
|
||||
say() { echo "$(now) wave1: $*" >&2; }
|
||||
die() { say "ERROR: $*"; exit 1; }
|
||||
seed_ip() { awk -F'\t' -v n="$1" '$1==n {print $4}' "$SEEDS_TSV"; }
|
||||
mode="${1:-}"; [ -n "$mode" ] || { sed -n '2,24p' "$0" | sed 's/^# \{0,1\}//'; exit 2; }; shift
|
||||
GO=0; OVERRIDE=""; BIN=""; SHA=""; MINER=""; DIGEST=""; COMMIT="c4459193"; NODE=""; OVJSON=""
|
||||
while [ $# -gt 0 ]; do case "$1" in
|
||||
--go) GO=1; shift ;; --override) OVERRIDE="$2"; shift 2 ;; --igneumd) BIN="$2"; shift 2 ;; --sha256) SHA="$2"; shift 2 ;;
|
||||
--miner) MINER="$2"; shift 2 ;; --digest) DIGEST="$2"; shift 2 ;; --commit) COMMIT="$2"; shift 2 ;; --node) NODE="$2"; shift 2 ;;
|
||||
--override-json) OVJSON="$2"; shift 2 ;; *) die "unknown option $1" ;; esac; done
|
||||
|
||||
one_seed() { # <name>: runs in the background; prints one RESULT line at the end
|
||||
local name="$1" ip log t0 line
|
||||
ip=$(seed_ip "$name"); [ -n "$ip" ] || { echo "RESULT $name: no ip in $SEEDS_TSV"; return 1; }
|
||||
log="$S/$name.log"; : > "$log"; t0=$(date +%s)
|
||||
{
|
||||
if [ "$GO" = 0 ]; then
|
||||
echo "DRY: scp $BIN root@$ip:/opt/igneum/bin/igneumd.new; sha256 asserted = $SHA"
|
||||
[ -n "$MINER" ] && echo "DRY: scp $MINER root@$ip:/opt/igneum/bin/igneum-miner.new (swapped with the node)"
|
||||
echo "DRY: scp $OVERRIDE root@$ip:/etc/igneum/override-params.json; seed.env EXTRA_ARGS=--override-params-file=/etc/igneum/override-params.json"
|
||||
echo "DRY: systemctl stop igneumd; mv igneumd igneumd.prev; mv igneumd.new igneumd; systemctl start igneumd"
|
||||
echo "DRY: read back: grep -c $COMMIT in the binary; journal 'Consensus params digest' == $DIGEST; eth_syncing on 127.0.0.1:26890; peers"
|
||||
echo "DRY RUN, nothing touched; box $ip answers as $("${SSH[@]}" "root@$ip" 'hostname; systemctl is-active igneumd; sha256sum /opt/igneum/bin/igneumd | cut -c1-12' 2>/dev/null | tr '\n' ' ')"
|
||||
else
|
||||
scp -q -i "$KEY" -o BatchMode=yes "$BIN" "root@$ip:/opt/igneum/bin/igneumd.new"
|
||||
[ -n "$MINER" ] && scp -q -i "$KEY" -o BatchMode=yes "$MINER" "root@$ip:/opt/igneum/bin/igneum-miner.new"
|
||||
scp -q -i "$KEY" -o BatchMode=yes "$OVERRIDE" "root@$ip:/etc/igneum/override-params.json"
|
||||
"${SSH[@]}" "root@$ip" bash -s -- "$SHA" "$COMMIT" "$DIGEST" "$([ -n "$MINER" ] && echo 1 || echo 0)" <<'REMOTE'
|
||||
set -euo pipefail
|
||||
SHA="$1"; COMMIT="$2"; DIGEST="$3"; MINER="$4"; cd /opt/igneum/bin
|
||||
got=$(sha256sum igneumd.new | cut -d' ' -f1); [ "$got" = "$SHA" ] || { echo "sha256 MISMATCH on the box: $got"; exit 1; }
|
||||
python3 -c 'import json,sys; json.load(open("/etc/igneum/override-params.json"))' || { echo "override file does not parse"; exit 1; }
|
||||
chmod 755 igneumd.new; [ "$MINER" = 1 ] && chmod 755 igneum-miner.new
|
||||
grep -qE '^EXTRA_ARGS=' /etc/igneum/seed.env && sed -i -E 's#^EXTRA_ARGS=.*#EXTRA_ARGS="--override-params-file=/etc/igneum/override-params.json"#' /etc/igneum/seed.env || echo 'EXTRA_ARGS="--override-params-file=/etc/igneum/override-params.json"' >> /etc/igneum/seed.env
|
||||
t0=$(date +%s); systemctl stop igneumd
|
||||
mv -f igneumd igneumd.prev; mv -f igneumd.new igneumd; [ "$MINER" = 1 ] && { mv -f igneum-miner igneum-miner.prev 2>/dev/null || true; mv -f igneum-miner.new igneum-miner; }
|
||||
systemctl start igneumd; sleep 6
|
||||
down=$(( $(date +%s) - t0 ))
|
||||
commits=$(grep -a -c "$COMMIT" igneumd || true)
|
||||
first=$(journalctl -u igneumd --since "-40 s" --no-pager 2>/dev/null | grep -vE "Started|Stopped|Stopping|Deactivated|Consumed" | head -1 | cut -c1-120)
|
||||
dig=$(journalctl -u igneumd --since "-40 s" --no-pager 2>/dev/null | grep -oE "Consensus params digest: [0-9a-f]+" | tail -1 | awk '{print $4}')
|
||||
syncing=$(curl -s -m 5 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_syncing","params":[]}' http://127.0.0.1:26890 | cut -c1-80)
|
||||
peers=$(curl -s -m 5 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"net_peerCount","params":[]}' http://127.0.0.1:26890 | grep -oE '"result":"[^"]*"' | cut -d'"' -f4)
|
||||
dmatch=no; [ -n "$dig" ] && [ "$dig" = "$DIGEST" ] && dmatch=MATCH; [ -n "$dig" ] && [ "$dig" != "$DIGEST" ] && dmatch="MISMATCH($dig)"
|
||||
echo "unit $(systemctl is-active igneumd) down ${down}s; commit strings $commits; digest $dmatch; eth_syncing $syncing; peers $peers; first: $first"
|
||||
REMOTE
|
||||
fi
|
||||
} >> "$log" 2>&1; rc=$?
|
||||
line=$(tail -1 "$log" | cut -c1-300)
|
||||
echo "RESULT $name ($ip) rc=$rc after $(( $(date +%s) - t0 )) s: $line"
|
||||
}
|
||||
|
||||
case "$mode" in
|
||||
seeds)
|
||||
[ -n "$OVERRIDE" ] && [ -n "$BIN" ] && [ -n "$SHA" ] && [ -n "$DIGEST" ] || die "seeds needs --override --igneumd --sha256 --digest"
|
||||
[ -f "$OVERRIDE" ] && [ -f "$BIN" ] || die "override or binary file missing"
|
||||
local_sha=$(shasum -a 256 "$BIN" | cut -d' ' -f1); [ "$local_sha" = "$SHA" ] || die "the binary's sha256 is $local_sha, not $SHA"
|
||||
python3 -c 'import json,sys; json.load(open(sys.argv[1]))' "$OVERRIDE" || die "override file does not parse"
|
||||
say "seeds: $( [ "$GO" = 1 ] && echo GO || echo DRY RUN ); igneumd $SHA; override $OVERRIDE ($(python3 -c 'import json,sys; print(len(json.load(open(sys.argv[1]))))' "$OVERRIDE") fields); digest $DIGEST; commit $COMMIT"
|
||||
for n in seed1.testnet seed2.testnet seed3.testnet; do one_seed "$n" & done; wait
|
||||
say "seeds done; logs in $S" ;;
|
||||
hands)
|
||||
[ -n "$NODE" ] && [ -n "$OVJSON" ] && [ -n "$DIGEST" ] || die "hands needs --node --override-json --digest"
|
||||
g=""; [ "$GO" = 1 ] && g="--go"
|
||||
say "hands: $( [ "$GO" = 1 ] && echo GO || echo DRY RUN ) through move-hand.sh"
|
||||
"$HERE/hands/move-hand.sh" binary --node "$NODE" --override-json "$OVJSON" $g
|
||||
"$HERE/hands/move-hand.sh" restart observer-node --digest "$DIGEST" $g
|
||||
"$HERE/hands/move-hand.sh" restart node1 --digest "$DIGEST" $g ;;
|
||||
lift-rpc-filter)
|
||||
ip=$(seed_ip seed1.testnet)
|
||||
if [ "$GO" = 0 ]; then
|
||||
say "DRY RUN: on root@$ip: back up /opt/igneum/bin/rpc-filter.py, set BLOCKED_UNTIL_FIXED_NODE = set(), py_compile, systemctl restart igneum-rpc-filter, read back"
|
||||
"${SSH[@]}" "root@$ip" 'echo "current: $(grep -c "^BLOCKED_UNTIL_FIXED_NODE = {" /opt/igneum/bin/rpc-filter.py) block set(s), unit $(systemctl is-active igneum-rpc-filter.service)"'; exit 0; fi
|
||||
"${SSH[@]}" "root@$ip" bash -s <<'REMOTE'
|
||||
set -euo pipefail
|
||||
f=/opt/igneum/bin/rpc-filter.py; cp -a "$f" "$f.bak-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
python3 - <<'PY'
|
||||
import re
|
||||
p='/opt/igneum/bin/rpc-filter.py'; s=open(p).read()
|
||||
new, n = re.subn(r'BLOCKED_UNTIL_FIXED_NODE = \{.*?\n\}', 'BLOCKED_UNTIL_FIXED_NODE = set() # lifted 7 Oct 2026: every seed runs the fixed node (N7 class fixed from 8097d600)', s, count=1, flags=re.S)
|
||||
assert n == 1, "block set not found"
|
||||
open(p,'w').write(new)
|
||||
PY
|
||||
python3 -m py_compile "$f"; systemctl restart igneum-rpc-filter.service; sleep 2
|
||||
echo "unit $(systemctl is-active igneum-rpc-filter.service); blocked set now: $(grep -E '^BLOCKED_UNTIL_FIXED_NODE' "$f" | cut -c1-60)"
|
||||
REMOTE
|
||||
say "public read-back: $(curl -s -m 8 -H 'content-type: application/json' --data '{"jsonrpc":"2.0","id":1,"method":"eth_blockNumber","params":[]}' https://rpc.testnet.igneum.network | cut -c1-120)" ;;
|
||||
*) die "unknown mode $mode" ;;
|
||||
esac
|
||||
|
|
@ -14,8 +14,9 @@ stamp="ci-$$-$(date +%s)"
|
|||
scp -q "${BS_SSH_OPTS[@]}" infra/build-server/lease.sh "$BS_HOST:/tmp/lease-$stamp.sh"
|
||||
scp -q "${BS_SSH_OPTS[@]}" infra/build-server/remote-run.sh "$BS_HOST:/tmp/rr-$stamp.sh"
|
||||
rc=0
|
||||
bs_ssh "bash /tmp/lease-$stamp.sh --self-test 2>&1 | grep -E '^lease self-test'; exit \${PIPESTATUS[0]}" || rc=1
|
||||
bs_ssh "IGNEUM_REMOTE_RUN_UNDER_TEST=/tmp/rr-$stamp.sh bash /tmp/rr-$stamp.sh --self-test-slots 2>&1 | grep -E '^self-test-slots'; exit \${PIPESTATUS[0]}" || rc=1
|
||||
bs_ssh "rm -f /tmp/lease-$stamp.sh /tmp/rr-$stamp.sh" || true
|
||||
# the two self-tests run in parallel in one ssh session (each against its own scratch lock directory); their last lines are printed
|
||||
bs_ssh "set -o pipefail; (bash /tmp/lease-$stamp.sh --self-test 2>&1 | grep -E '^lease self-test'; echo lease=\${PIPESTATUS[0]} >> /tmp/locks-$stamp.rc) & (IGNEUM_REMOTE_RUN_UNDER_TEST=/tmp/rr-$stamp.sh bash /tmp/rr-$stamp.sh --self-test-slots 2>&1 | grep -E '^self-test-slots'; echo slots=\${PIPESTATUS[0]} >> /tmp/locks-$stamp.rc) & wait; cat /tmp/locks-$stamp.rc; rm -f /tmp/lease-$stamp.sh /tmp/rr-$stamp.sh /tmp/locks-$stamp.rc" | tee /tmp/box-locks-$$.out || rc=1
|
||||
grep -q '^lease=0$' /tmp/box-locks-$$.out && grep -q '^slots=0$' /tmp/box-locks-$$.out || rc=1
|
||||
rm -f /tmp/box-locks-$$.out
|
||||
[ "$rc" = 0 ] && echo "box-locks: the lease tool and the slot runner pass their self-tests on $BS_HOST"
|
||||
exit $rc
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ tree_checks() {
|
|||
run "build-remote without a priority flag bounds suites and benches (nice 10, 32 cores); a gate runs unbounded" bash tools/ci/build-kind-default-check.sh
|
||||
run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh
|
||||
run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh)" bash tools/ci/box-locks-check.sh
|
||||
run "the simulators job runs on master and release-* pushes and pull requests into them only" bash tools/ci/sims-branch-check.sh
|
||||
run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh'
|
||||
run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh'
|
||||
run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test
|
||||
|
|
|
|||
35
tools/ci/sims-branch-check.sh
Executable file
35
tools/ci/sims-branch-check.sh
Executable file
|
|
@ -0,0 +1,35 @@
|
|||
#!/usr/bin/env bash
|
||||
# The simulators job of .github/workflows/ci.yml runs on master and release-* pushes, and on pull requests into them, ONLY (main,
|
||||
# 7 October 2026: with build-2 in the runner pool the queue still read 22 because every code push cost two box jobs; a
|
||||
# feature-branch code push now runs the igneum-pow tests alone). This check reads the job's `if:` line and evaluates the rule
|
||||
# with the expression's own shape for the known cases, so a change that lets a feature-branch push run the simulators fails it.
|
||||
#
|
||||
# tools/ci/sims-branch-check.sh # exit 1 with the case that resolved wrongly (the check IS its self-test)
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/../.."
|
||||
line=$(awk '/^ sims:/{f=1} f && /^ if:/{print; exit}' .github/workflows/ci.yml)
|
||||
[ -n "$line" ] || { echo "sims-branch: no if: line under the sims job" >&2; exit 1; }
|
||||
for want in "github.event_name == 'push'" "github.ref == 'refs/heads/master'" "startsWith(github.ref, 'refs/heads/release-')" \
|
||||
"github.event_name == 'pull_request'" "github.base_ref == 'master'" "startsWith(github.base_ref, 'release-')" "needs.changes.outputs.code == 'true'"; do
|
||||
case "$line" in *"$want"*) ;; *) echo "sims-branch: the sims if: line lacks \"$want\": $line" >&2; exit 1 ;; esac
|
||||
done
|
||||
# the rule, as the expression reads: code and ((push and (master or release-*)) or (pull_request and base (master or release-*)))
|
||||
rule() { # <code> <event> <ref> <base_ref> -> run|skip
|
||||
local code="$1" ev="$2" ref="$3" base="$4"
|
||||
[ "$code" = true ] || { echo skip; return; }
|
||||
if [ "$ev" = push ] && { [ "$ref" = refs/heads/master ] || [[ "$ref" == refs/heads/release-* ]]; }; then echo run; return; fi
|
||||
if [ "$ev" = pull_request ] && { [ "$base" = master ] || [[ "$base" == release-* ]]; }; then echo run; return; fi
|
||||
echo skip
|
||||
}
|
||||
fail=0
|
||||
expect() { local want="$1"; shift; local got; got=$(rule "$@"); if [ "$got" = "$want" ]; then echo "sims-branch: [$*] -> $got"; else echo "sims-branch: [$*] -> $got, expected $want" >&2; fail=1; fi; }
|
||||
expect run true push refs/heads/master ""
|
||||
expect run true push refs/heads/release-0.3.20 ""
|
||||
expect skip true push refs/heads/build-server ""
|
||||
expect skip true push refs/heads/feature/x ""
|
||||
expect run true pull_request refs/pull/12/merge master
|
||||
expect run true pull_request refs/pull/12/merge release-0.3.21
|
||||
expect skip true pull_request refs/pull/12/merge build-server
|
||||
expect skip false push refs/heads/master ""
|
||||
[ "$fail" = 0 ] && echo "sims-branch: the simulators run on master and release-* pushes and on pull requests into them only"
|
||||
exit $fail
|
||||
Loading…
Reference in a new issue