From fd644f320fbce14d4c5647323ce35b6f4858d11f Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:40:39 +0000 Subject: [PATCH 01/10] Devnet 3 on build-1: ports reconciled with the fleet lane (its observer-node-dn3 on 26650/28650/26651/26850 is the observer instance; node1-dn3 moves to 26671/26670/28670/26870; ufw 26671 open 16:39Z); the units installed as root, disabled Co-Authored-By: Claude Fable 5.1 --- infra/build-server/devnet3/devnet3.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/infra/build-server/devnet3/devnet3.sh b/infra/build-server/devnet3/devnet3.sh index d8a97319..a3daa87f 100755 --- a/infra/build-server/devnet3/devnet3.sh +++ b/infra/build-server/devnet3/devnet3.sh @@ -35,7 +35,9 @@ REMOTE case "$mode" in seed) start_one dn3-seed "$DN3_SEED_APPDIR" "0.0.0.0:$DN3_SEED_P2P" "$DN3_SEED_RPC" "$DN3_SEED_JSON" "$DN3_SEED_EVM" "$DN3_SEED_LOG" --maxinpeers=128 --outpeers=8 ;; node1) start_one node1-dn3 "$DN3_NODE1_APPDIR" "0.0.0.0:$DN3_NODE1_P2P" "$DN3_NODE1_RPC" "$DN3_NODE1_JSON" "$DN3_NODE1_EVM" /srv/hands/node1-dn3.log --enable-unsynced-mining --addpeer=127.0.0.1:$DN3_SEED_P2P --maxinpeers=128 --outpeers=8 ;; - observer-node) start_one observer-dn3 "$DN3_OBS_APPDIR" "127.0.0.1:$DN3_OBS_P2P" "$DN3_OBS_RPC" "$DN3_OBS_JSON" "$DN3_OBS_EVM" /srv/hands/observer-dn3.log --addpeer=127.0.0.1:$DN3_NODE1_P2P --addpeer=127.0.0.1:$DN3_SEED_P2P ;; - status) "${SSH[@]}" "for l in $DN3_SEED_LOG /srv/hands/node1-dn3.log /srv/hands/observer-dn3.log; do [ -f \$l ] && { echo \"== \$l\"; head -1 \$l | cut -c1-100; grep -oE 'Consensus params digest: [0-9a-f]+|genesis [0-9a-f]+ executed' \$l | head -2; tail -1 \$l | cut -c1-120; }; done; ss -ltn | awk '{print \$4}' | grep -E ':(26631|26651|26661|27630|26650|26660)\$' | tr '\n' ' '; echo" ;; + observer-node) # the fleet lane's instance: its unit (installed 16:39Z, disabled); IGNEUMD_DN3 and DN3_PEERS come from /srv/hands/dn3/dn3.env + if [ "$GO" = 0 ]; then say "DRY observer-node: sudo systemctl enable --now igneum-observer-node-dn3 (reads $DN3_OBS_SCRIPT; dn3.env IGNEUMD_DN3 must name the 0.3.22 artefact)"; "${SSH[@]}" 'grep -E "^(IGNEUMD_DN3|DN3_PEERS|DN3_LISTEN)=" /srv/hands/dn3/dn3.env'; else + ssh -i "$KEY" -o BatchMode=yes "root@${HOST#*@}" 'systemctl enable --now igneum-observer-node-dn3 && sleep 8 && systemctl is-active igneum-observer-node-dn3 && journalctl -u igneum-observer-node-dn3 --since "-60 s" --no-pager | grep -oE "igneumd/[^ ]+|Consensus params digest: [0-9a-f]+|genesis [0-9a-f]+ executed|P2P Server starting on: [^ ]+" | head -4'; fi ;; + status) "${SSH[@]}" "for l in $DN3_SEED_LOG /srv/hands/node1-dn3.log; do [ -f \$l ] && { echo \"== \$l\"; head -1 \$l | cut -c1-100; grep -oE 'Consensus params digest: [0-9a-f]+|genesis [0-9a-f]+ executed' \$l | head -2; tail -1 \$l | cut -c1-120; }; done; ss -ltn | awk '{print \$4}' | grep -E ':(26631|26651|26671|27630|26650|26670)\$' | tr '\n' ' '; echo" ;; *) sed -n '2,12p' "$0" | sed 's/^# \{0,1\}//'; exit 2 ;; esac From 01275d2fb40c75366b930ededf5accb724c571e6 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:54:12 +0000 Subject: [PATCH 02/10] Devnet 3: devnet3.env names the 0.3.22 candidate 21d8f454 artefact (the file is tracked on purpose: ports and the binary path, no secret; a *.env ignore rule caught it) Co-Authored-By: Claude Fable 5.1 --- infra/build-server/devnet3/devnet3.env | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 infra/build-server/devnet3/devnet3.env diff --git a/infra/build-server/devnet3/devnet3.env b/infra/build-server/devnet3/devnet3.env new file mode 100644 index 00000000..7421c724 --- /dev/null +++ b/infra/build-server/devnet3/devnet3.env @@ -0,0 +1,17 @@ +# Devnet 3 on igneum-build-1 (0.3.22, main's order on the project lead's word, 7 October 2026): the network flag and the ports every script here +# reads. NET_FLAGS is a PLACEHOLDER until the node lane names the object's flag in its commit (igneum-devnet-3: own network id and +# p2p port, every activation at 0, no override file). Ports follow the box's pattern (devnet 266x1 p2p; Devnet 2 seed 27610/27612/27790): +NET_FLAGS="--devnet --devnet-suffix=3" # PLACEHOLDER: replace with the node lane's flag for igneum-devnet-3 +IGNEUMD="/srv/artefacts/0322-21d8f454/hands/igneumd" # the 0.3.22 candidate 21d8f454 (built 17:47 BST, 7 Oct 2026) +# the seed (a bare process beside the Devnet 2 one, run as build, empty datadir) +DN3_SEED_APPDIR=/home/build/dn3seed +DN3_SEED_P2P=26631 # ufw: opened 7 Oct 2026 (provision.sh P2P_PORTS carries it) +DN3_SEED_RPC=27630; DN3_SEED_JSON=27632; DN3_SEED_EVM=27810 +DN3_SEED_LOG=/home/build/dn3seed.log +# the hands' SECOND instances (the old devnet runs on for a day, so these run beside node1 and observer-node, not instead). +# Reconciled with the fleet lane's staging of 17:37 BST: ITS observer-node-dn3 (/srv/hands/bin/run-observer-node-dn3.sh, unit +# igneum-observer-node-dn3, rpc 26650, wrpc json 28650, p2p 26651, evm 26850, appdir /srv/hands/observer-node-dn3, peers from +# /srv/hands/dn3/dn3.env) is the observer instance; this lane runs the seed and node1-dn3 (p2p 26671, ufw open since 16:39Z). +DN3_NODE1_APPDIR=/srv/hands/node1-dn3; DN3_NODE1_P2P=26671; DN3_NODE1_RPC=26670; DN3_NODE1_JSON=28670; DN3_NODE1_EVM=26870 +DN3_OBS_APPDIR=/srv/hands/observer-node-dn3; DN3_OBS_P2P=26651; DN3_OBS_RPC=26650; DN3_OBS_JSON=28650; DN3_OBS_EVM=26850 +DN3_OBS_SCRIPT=/srv/hands/bin/run-observer-node-dn3.sh # the fleet lane's; devnet3.sh observer-node starts its unit, not a copy From ab364a10c8bfab9ff18acb09bcc5232d1cc8dee0 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:55:43 +0000 Subject: [PATCH 03/10] Build boxes: the engine gate (igneumd and igneum-miner fetched by build-remote must carry igneum-pow/src/ paths; a commit string alone does not prove the engine: the stub-engine 21d8f454 that rejected every block on the Devnet 3 hub, 7 Oct 2026); tools/ci/engine-check.sh with its self-test in the gate Co-Authored-By: Claude Fable 5.1 --- tools/build-remote.sh | 3 +++ tools/ci/engine-check.sh | 24 ++++++++++++++++++++++++ tools/ci/pre-push.sh | 1 + 3 files changed, 28 insertions(+) create mode 100755 tools/ci/engine-check.sh diff --git a/tools/build-remote.sh b/tools/build-remote.sh index ea8d27a4..365f4193 100755 --- a/tools/build-remote.sh +++ b/tools/build-remote.sh @@ -255,6 +255,9 @@ if [ "$FETCH" = 1 ] && [ -n "$ARTEFACTS" ]; then bs_log "artefact $dest: $(bs_size "$dest") bytes, sha256 $(bs_sha256 "$dest"), $(file -b "$dest" | cut -c1-60)" # the commit-string gate (rule of 6 October 2026): a node binary without its commit in its strings fails the run case "$BS_KIND:$(basename "$dest")" in node:igneumd) "$HERE/ci/commit-string-check.sh" "$dest" "$BS_SHA" || bs_die "commit-string gate failed for $a" ;; esac # only kaspad depends on kaspa-build-info + # the engine gate (7 Oct 2026, the Devnet 3 start): igneumd and igneum-miner must carry igneum-pow/src/ paths; a commit string + # alone does not prove the engine (a stub-engine 21d8f454 rejected every mined block on the fleet's hub) + case "$BS_KIND:$(basename "$dest")" in node:igneumd|node:igneum-miner) [ "${IGNEUM_ALLOW_STUB:-}" = 1 ] || "$HERE/ci/engine-check.sh" "$dest" || bs_die "engine gate failed for $a (IGNEUM_ALLOW_STUB=1 to fetch a stub-engine binary on purpose)" ;; esac # the glibc ceiling of anything that ships (main, 7 Oct 2026): a seed or a rig refuses a binary needing more than 2.36 if [ "$SHIP" = 1 ]; then "$HERE/ci/glibc-ceiling-check.sh" "$dest" "$GLIBC" || bs_die "glibc ceiling gate failed for $a"; fi done diff --git a/tools/ci/engine-check.sh b/tools/ci/engine-check.sh new file mode 100755 index 00000000..15340078 --- /dev/null +++ b/tools/ci/engine-check.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# The engine gate (7 October 2026, the Devnet 3 start): a 21d8f454 igneumd with its commit string twice but ZERO igneum-pow/src/ +# paths was placed in the artefact folder by a build outside the app tree; the fleet's hub ran it, the igneum-pow miner's blocks +# were every one rejected (Reject(BlockInvalid)), the go stopped. The commit-string gate cannot see this: the string comes from the +# fork's own git, the engine from the igneum-pow tree the build sat next to. So every igneumd and igneum-miner that +# tools/build-remote.sh fetches must carry igneum-pow source paths in its strings (rustc embeds the panic locations of the engine +# crate it compiled in: igneum-pow/src/...); none means the stub engine or a foreign igneum-pow, and the fetch refuses. +# +# tools/ci/engine-check.sh # exit 0 with the count, exit 1 "no igneum-pow/src/ path in " +# tools/ci/engine-check.sh --self-test # a fixture with the paths passes, one without fails +set -euo pipefail +if [ "${1:-}" = --self-test ]; then + t=$(mktemp -d); trap 'rm -rf "$t"' EXIT + printf 'binary junk\0/srv/builds/x/igneum-pow/src/lib.rs\0more junk\0igneum-pow/src/lottery.rs\0' > "$t/good" + printf 'binary junk\0/srv/builds/x/consensus/pow/src/stub.rs\0commit 21d8f454\0' > "$t/bad" + "$0" "$t/good" >/dev/null || { echo "engine-check self-test: a binary WITH igneum-pow paths was refused"; exit 1; } + if "$0" "$t/bad" >/dev/null 2>&1; then echo "engine-check self-test: a binary WITHOUT igneum-pow paths passed"; exit 1; fi + echo "engine-check self-test: a binary with igneum-pow/src/ paths passes, one without is refused"; exit 0 +fi +f="${1:-}"; [ -f "$f" ] || { echo "engine-check: no file '$f'" >&2; exit 2; } +n=$(LC_ALL=C grep -a -c 'igneum-pow/src/' "$f" || true) +if [ "${n:-0}" -gt 0 ]; then echo "engine-check: $(basename "$f") carries $n igneum-pow/src/ path line(s): the igneum-pow engine"; exit 0; fi +echo "engine-check: no igneum-pow/src/ path in $(basename "$f"): the stub engine or a foreign igneum-pow tree (the commit string alone does not prove the engine; 7 Oct 2026 Devnet 3)" >&2 +exit 1 diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh index 5699d52c..152303a5 100755 --- a/tools/ci/pre-push.sh +++ b/tools/ci/pre-push.sh @@ -95,6 +95,7 @@ tree_checks() { run "pinned guest programs match their manifest" bash tools/ci/pinned-guests-check.sh run "root prover playbooks kill the GPU server and unlink its socket" bash tools/ci/prover-socket-check.sh run "commit-string gate self-test" bash tools/ci/commit-string-check.sh --self-test + run "engine gate self-test (igneumd and igneum-miner must carry igneum-pow/src/ paths)" bash tools/ci/engine-check.sh --self-test run "build server remote checkout self-test" bash infra/build-server/remote-run.sh --self-test run "a slot holder keeps its own line for the whole run (the watcher-trust rule)" bash infra/build-server/remote-run.sh --self-test-keeper run "the remote checkout resets the mirror's tree before the branch checkout (the stale-overlay class)" bash -c 'bash tools/ci/mirror-reset-check.sh --self-test && bash tools/ci/mirror-reset-check.sh' From 719d5fc1333e773094dfd8e859212399a49fbcc2 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:59:28 +0000 Subject: [PATCH 04/10] Engine gate: any igneum-pow*/src/ directory pairs (the fork pairs through a paths override that may name the tree igneum-pow-amend, the archive of c3d32437; the shipper, 7 Oct 2026), the matched name printed in the line; self-test with both names Co-Authored-By: Claude Fable 5.1 --- tools/ci/engine-check.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/ci/engine-check.sh b/tools/ci/engine-check.sh index 15340078..9e74e6e6 100755 --- a/tools/ci/engine-check.sh +++ b/tools/ci/engine-check.sh @@ -4,7 +4,8 @@ # were every one rejected (Reject(BlockInvalid)), the go stopped. The commit-string gate cannot see this: the string comes from the # fork's own git, the engine from the igneum-pow tree the build sat next to. So every igneumd and igneum-miner that # tools/build-remote.sh fetches must carry igneum-pow source paths in its strings (rustc embeds the panic locations of the engine -# crate it compiled in: igneum-pow/src/...); none means the stub engine or a foreign igneum-pow, and the fetch refuses. +# crate it compiled in: igneum-pow/src/..., or igneum-pow-amend/src/... when the fork pairs through its paths override); none means +# the stub engine or no paired tree, and the fetch refuses; the matched directory name is printed so the pairing is visible. # # tools/ci/engine-check.sh # exit 0 with the count, exit 1 "no igneum-pow/src/ path in " # tools/ci/engine-check.sh --self-test # a fixture with the paths passes, one without fails @@ -12,13 +13,20 @@ set -euo pipefail if [ "${1:-}" = --self-test ]; then t=$(mktemp -d); trap 'rm -rf "$t"' EXIT printf 'binary junk\0/srv/builds/x/igneum-pow/src/lib.rs\0more junk\0igneum-pow/src/lottery.rs\0' > "$t/good" + printf 'binary junk\0/srv/builds/x/igneum-pow-amend/src/lib.rs\0igneum-pow-amend/src/lottery.rs\0' > "$t/amend" printf 'binary junk\0/srv/builds/x/consensus/pow/src/stub.rs\0commit 21d8f454\0' > "$t/bad" - "$0" "$t/good" >/dev/null || { echo "engine-check self-test: a binary WITH igneum-pow paths was refused"; exit 1; } + out=$("$0" "$t/good") || { echo "engine-check self-test: a binary WITH igneum-pow/src/ paths was refused"; exit 1; } + case "$out" in *"igneum-pow/src/ 2 paths"*) ;; *) echo "engine-check self-test: the matched directory is not printed: $out"; exit 1 ;; esac + out=$("$0" "$t/amend") || { echo "engine-check self-test: a binary paired through igneum-pow-amend/src/ was refused"; exit 1; } + case "$out" in *"igneum-pow-amend/src/ 2 paths"*) ;; *) echo "engine-check self-test: the amend directory is not printed: $out"; exit 1 ;; esac if "$0" "$t/bad" >/dev/null 2>&1; then echo "engine-check self-test: a binary WITHOUT igneum-pow paths passed"; exit 1; fi - echo "engine-check self-test: a binary with igneum-pow/src/ paths passes, one without is refused"; exit 0 + echo "engine-check self-test: igneum-pow/src/ and igneum-pow-amend/src/ pass with the directory named, a binary without is refused"; exit 0 fi f="${1:-}"; [ -f "$f" ] || { echo "engine-check: no file '$f'" >&2; exit 2; } -n=$(LC_ALL=C grep -a -c 'igneum-pow/src/' "$f" || true) -if [ "${n:-0}" -gt 0 ]; then echo "engine-check: $(basename "$f") carries $n igneum-pow/src/ path line(s): the igneum-pow engine"; exit 0; fi -echo "engine-check: no igneum-pow/src/ path in $(basename "$f"): the stub engine or a foreign igneum-pow tree (the commit string alone does not prove the engine; 7 Oct 2026 Devnet 3)" >&2 +# any directory name beginning igneum-pow and ending /src/ (the fork pairs its pow through a paths override that may name the tree +# igneum-pow-amend, the archive of 017e7037; the shipper, 7 Oct 2026); the matched name is printed so the pairing is visible +dirs=$(LC_ALL=C grep -a -oE 'igneum-pow[A-Za-z0-9._-]*/src/' "$f" | sort | uniq -c | awk '{printf "%s %s paths; ", $2, $1}') +n=$(LC_ALL=C grep -a -c -E 'igneum-pow[A-Za-z0-9._-]*/src/' "$f" || true) +if [ "${n:-0}" -gt 0 ]; then echo "engine-check: $(basename "$f") paired: ${dirs% }"; exit 0; fi +echo "engine-check: no igneum-pow*/src/ path in $(basename "$f"): the stub engine or no paired igneum-pow tree (the commit string alone does not prove the engine; 7 Oct 2026 Devnet 3)" >&2 exit 1 From 3e783b541031513a760a940fb4693ec734751d2d Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:59:56 +0000 Subject: [PATCH 05/10] Devnet 3: devnet3.env becomes devnet3.conf (the no-secrets gate refuses any tracked *.env by name; the file holds ports and a binary path, no secret) Co-Authored-By: Claude Fable 5.1 --- infra/build-server/devnet3/{devnet3.env => devnet3.conf} | 0 infra/build-server/devnet3/devnet3.sh | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename infra/build-server/devnet3/{devnet3.env => devnet3.conf} (100%) diff --git a/infra/build-server/devnet3/devnet3.env b/infra/build-server/devnet3/devnet3.conf similarity index 100% rename from infra/build-server/devnet3/devnet3.env rename to infra/build-server/devnet3/devnet3.conf diff --git a/infra/build-server/devnet3/devnet3.sh b/infra/build-server/devnet3/devnet3.sh index a3daa87f..cef4560f 100755 --- a/infra/build-server/devnet3/devnet3.sh +++ b/infra/build-server/devnet3/devnet3.sh @@ -9,12 +9,12 @@ # lines. No unit yet: the processes are detached (setsid nohup) under the build user like the Devnet 2 seed; units follow once the # network is green. The old devnet's node1 and observer-node are not touched. set -euo pipefail -HERE=$(cd "$(dirname "$0")" && pwd); . "$HERE/devnet3.env" +HERE=$(cd "$(dirname "$0")" && pwd); . "$HERE/devnet3.conf" KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}"; HOST=$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]') SSH=(ssh -i "$KEY" -o BatchMode=yes -o ConnectTimeout=15 "$HOST") say() { echo "$(TZ=Europe/London date '+%H:%M:%S %Z') devnet3: $*" >&2; } mode="${1:-}"; shift || true; GO=0; [ "${1:-}" = --go ] && GO=1 -case "$NET_FLAGS" in *suffix=3*) say "NET_FLAGS is still the placeholder ($NET_FLAGS): the node lane's flag goes into devnet3.env first" ;; esac +case "$NET_FLAGS" in *suffix=3*) say "NET_FLAGS is still the placeholder ($NET_FLAGS): the node lane's flag goes into devnet3.conf first" ;; esac case "$IGNEUMD" in *PLACEHOLDER*) [ "$mode" = status ] || { say "IGNEUMD is the placeholder: the 0.3.22 candidate's build fills it"; [ "$GO" = 0 ] || exit 1; } ;; esac start_one() { # [extra args...] local name="$1" appdir="$2" p2p="$3" rpc="$4" json="$5" evm="$6" log="$7"; shift 7 From 4df6a6e8371f9ac7b03c5ccd3f9d47ba30dc1423 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:06:20 +0000 Subject: [PATCH 06/10] Devnet 3: devnet3.conf names the 0.3.22 candidate 69d1b56e (genesis timestamp fix) Co-Authored-By: Claude Fable 5.1 --- infra/build-server/devnet3/devnet3.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/build-server/devnet3/devnet3.conf b/infra/build-server/devnet3/devnet3.conf index 7421c724..9236982d 100644 --- a/infra/build-server/devnet3/devnet3.conf +++ b/infra/build-server/devnet3/devnet3.conf @@ -2,7 +2,7 @@ # reads. NET_FLAGS is a PLACEHOLDER until the node lane names the object's flag in its commit (igneum-devnet-3: own network id and # p2p port, every activation at 0, no override file). Ports follow the box's pattern (devnet 266x1 p2p; Devnet 2 seed 27610/27612/27790): NET_FLAGS="--devnet --devnet-suffix=3" # PLACEHOLDER: replace with the node lane's flag for igneum-devnet-3 -IGNEUMD="/srv/artefacts/0322-21d8f454/hands/igneumd" # the 0.3.22 candidate 21d8f454 (built 17:47 BST, 7 Oct 2026) +IGNEUMD="/srv/artefacts/0322-69d1b56e/hands/igneumd" # the 0.3.22 candidate 69d1b56e (genesis timestamp fix; built 18:04 BST, 7 Oct 2026) # the seed (a bare process beside the Devnet 2 one, run as build, empty datadir) DN3_SEED_APPDIR=/home/build/dn3seed DN3_SEED_P2P=26631 # ufw: opened 7 Oct 2026 (provision.sh P2P_PORTS carries it) From cb04e2b8cabd4a8eb29946228a76f91eb513411a Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:26:14 +0000 Subject: [PATCH 07/10] Devnet 3: devnet3.sh passes the argument line base64-encoded over ssh (ssh flattens its argument list: the first --go started the seed on the OLD devnet, digest c562d70e, port 26611, 7 Oct 2026 18:22 BST) and refuses a line without --devnet-suffix=3 Co-Authored-By: Claude Fable 5.1 --- infra/build-server/devnet3/devnet3.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/infra/build-server/devnet3/devnet3.sh b/infra/build-server/devnet3/devnet3.sh index cef4560f..2facffc6 100755 --- a/infra/build-server/devnet3/devnet3.sh +++ b/infra/build-server/devnet3/devnet3.sh @@ -20,10 +20,13 @@ start_one() { # [extra arg local name="$1" appdir="$2" p2p="$3" rpc="$4" json="$5" evm="$6" log="$7"; shift 7 local args="$NET_FLAGS --appdir=$appdir --rpclisten=127.0.0.1:$rpc --rpclisten-json=127.0.0.1:$json --evm-rpclisten=127.0.0.1:$evm --listen=$p2p --nodnsseed --disable-upnp --nologfiles --yes $*" if [ "$GO" = 0 ]; then say "DRY $name: $IGNEUMD $args > $log"; return 0; fi - "${SSH[@]}" bash -s -- "$name" "$IGNEUMD" "$appdir" "$log" "$args" <<'REMOTE' -set -euo pipefail; name="$1"; bin="$2"; appdir="$3"; log="$4"; args="$5" + # ssh flattens its arguments into one remote command line, so the argument string travels base64-encoded (the first --go at + # 18:22 BST lost everything after the first flag: the seed started on the OLD devnet with digest c562d70e and port 26611) + "${SSH[@]}" bash -s -- "$name" "$IGNEUMD" "$appdir" "$log" "$(printf '%s' "$args" | base64 | tr -d '\n')" <<'REMOTE' +set -euo pipefail; name="$1"; bin="$2"; appdir="$3"; log="$4"; args="$(printf '%s' "$5" | base64 -d)" [ -x "$bin" ] || { echo "no binary $bin"; exit 1; } -mkdir -p "$appdir"; [ -z "$(ls -A "$appdir")" ] || echo "note: $appdir is not empty" +mkdir -p "$appdir"; [ -z "$(ls -A "$appdir")" ] || echo "note: $appdir is not empty: $(ls "$appdir" | tr '\n' ' ')" +case "$args" in *--devnet-suffix=3*) ;; *) echo "REFUSED: the argument line lacks --devnet-suffix=3: $args"; exit 1 ;; esac cd "$(dirname "$log")"; setsid nohup "$bin" $args > "$log" 2>&1 < /dev/null & pid=$!; sleep 8 echo "$name pid $pid alive=$(kill -0 $pid 2>/dev/null && echo yes || echo NO) commit-strings=$(grep -a -c "$(echo "$bin" | grep -oE '[0-9a-f]{8}' | tail -1)" /proc/$pid/exe 2>/dev/null || echo ?)" head -1 "$log" | cut -c1-120; grep -oE "Consensus params digest: [0-9a-f]+" "$log" | head -1 From f070417c7b7fd3ee540cbc2d9e4708ef0b76d512 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:56:30 +0000 Subject: [PATCH 08/10] HiveOS package: --kit (repeatable) puts program-pack kits under packs/ in the tar; h-run.sh seeds packs/devnet from a shipped pack only when the node's export left nothing (the first-start convenience, never the authority; SHIPPED_PACK, default v4-devnet3-epoch0); README line (0.3.22, 7 Oct 2026) Co-Authored-By: Claude Fable 5.1 --- packaging/hive/README.md | 10 ++++++++++ packaging/hive/h-run.sh | 10 ++++++++++ packaging/hive/make-hive-package.sh | 21 ++++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/packaging/hive/README.md b/packaging/hive/README.md index ed06aca8..1f7219f9 100644 --- a/packaging/hive/README.md +++ b/packaging/hive/README.md @@ -82,6 +82,16 @@ Stats JSON (what Hive reads from `$stats`): `hs` (kH/s per GPU), `hs_units` (`kh digest on the downloads page; a different one means the override is stale and the node is refused. - Ports: the bundled node listens on 26611 (p2p) and answers RPC on 127.0.0.1:26610 only. +## Shipped packs (0.3.22) + +`make-hive-package.sh --kit ` (repeatable) puts program-pack kits under `packs/` in the archive: the class v4 sub-version 3 +kit (eight packs, program_id `a785001687d8688a` for the shared devnet's epoch 0) and Devnet 3's epoch-0 pack +`v4-devnet3-epoch0` (program_id `fce15bf61030be57`, exported under igneum-pow 017e7037 over genesis `4020cb43` as epoch and +era seed, day bytes for 7 October UTC). They are the rig's FIRST-START convenience: `h-run.sh` seeds `packs/devnet` from the +shipped pack only when the node's own export left nothing, so a rig mines from its first start; the pack is dated, and a rig +starting after epoch 0 (3,600 DAA) re-exports from its own node as before. The shipped pack is never the authority; the +pack-id gate reads `program.json`'s `program_id`. + ## Building the package infra/cross/build-linux.sh # igneumd and igneum-miner for Linux (cargo-zigbuild), into infra/cross/out diff --git a/packaging/hive/h-run.sh b/packaging/hive/h-run.sh index 6622666b..a498e3b8 100755 --- a/packaging/hive/h-run.sh +++ b/packaging/hive/h-run.sh @@ -73,7 +73,17 @@ say "GPUs: $nv NVIDIA, $amd AMD (worker setting: $WORKER)" # 3. the hourly program pack from the node (the workers read it with --pack; the miner writes the next one to packs/prepare) export_pack() { [[ "$NODE_URL" == "none" ]] && return 0; rm -rf "$HERE/packs/devnet"; "$BIN/igneum-miner" export-pack "$NODE_URL" "$HERE/packs/devnet" >> "$MAIN" 2>&1; } +# a shipped pack (packs//program.json, from make-hive-package.sh --kit; 0.3.22) seeds packs/devnet ONLY when the node's own +# export left nothing: the rig's first-start convenience, never the authority (a rig starting after epoch 0 re-exports from its node; +# the miner's --prepare-packs and exit 42 keep it on the chain's program as before). SHIPPED_PACK names the directory under packs/ +# (h-config.sh or the Flight Sheet; default v4-devnet3-epoch0 when it exists). +seed_pack() { + [[ -d "$HERE/packs/devnet" && -f "$HERE/packs/devnet/program.json" ]] && return 0 + local sp="${SHIPPED_PACK:-v4-devnet3-epoch0}" + if [[ -f "$HERE/packs/$sp/program.json" ]]; then rm -rf "$HERE/packs/devnet"; cp -R "$HERE/packs/$sp" "$HERE/packs/devnet"; say "first start: packs/devnet seeded from the shipped pack $sp (program_id $(sed -n 's/.*"program_id" *: *"\{0,1\}\([0-9a-fx]*\)"\{0,1\}.*/\1/p' "$HERE/packs/$sp/program.json" | head -1)); the node's export replaces it"; fi +} export_pack || say "pack export failed; the miners retry" +seed_pack # 4. one miner per GPU, restarted on exit (exit 42 = the program changed and the worker cannot prepare: re-export the pack) run_gpu() { diff --git a/packaging/hive/make-hive-package.sh b/packaging/hive/make-hive-package.sh index 1278a768..af540b48 100755 --- a/packaging/hive/make-hive-package.sh +++ b/packaging/hive/make-hive-package.sh @@ -4,6 +4,8 @@ # and infra/cross/out-workers (the two GPU workers, build-workers-linux.sh) # NODE_OUT=... WORKERS_OUT=... VERSION=... OUT=... other inputs; VERSION defaults to the igneumd version in version.txt # --fake stub binaries instead (the self-test; never ship it) +# --kit (repeatable) program-pack kit(s) unpacked under packs/ in the tar (0.3.22: the sub-version 3 +# kit and Devnet 3's epoch-0 pack); the rig's first-start convenience, see h-run.sh # Output: packaging/hive/build/igneum-hive-.tar.gz with the directory igneum/ inside (what Hive expects: # the archive name carries the version, the directory does not), plus its sha256 and the Flight Sheet lines. set -euo pipefail @@ -12,7 +14,12 @@ REPO="$(cd "$HERE/../.." && pwd)" NODE_OUT="${NODE_OUT:-$REPO/infra/cross/out-v2}" WORKERS_OUT="${WORKERS_OUT:-$REPO/infra/cross/out-workers}" OUT="${OUT:-$HERE/build}" -FAKE=0; [[ "${1:-}" == "--fake" ]] && FAKE=1 +# --kit (repeatable; 0.3.22, 7 October 2026): a program-pack kit unpacked under packs/ in the tar (the class v4 sub-version 3 +# packs and Devnet 3's epoch-0 pack, from the hash lane), the rig's FIRST-START convenience: h-run.sh seeds packs/devnet from a +# shipped pack only when the node's own export leaves nothing, and a rig starting after epoch 0 re-exports from its own node as +# before; the shipped pack is never the authority. The pack-id gate reads program.json's program_id. +FAKE=0; KITS=() +while [[ $# -gt 0 ]]; do case "$1" in --fake) FAKE=1; shift ;; --kit) KITS+=("$2"); shift 2 ;; *) echo "unknown argument $1" >&2; exit 2 ;; esac; done log() { printf '%s %s\n' "$(date -u +%H:%M:%S)" "$*"; } die() { log "ERROR: $*" >&2; exit 1; } stage="$OUT/igneum"; rm -rf "$stage"; mkdir -p "$stage/bin" @@ -34,6 +41,18 @@ else VERSION="${VERSION:-$(head -1 "$NODE_OUT/version.txt" | awk '{print $2}')}" fi [[ -n "$VERSION" ]] || die "no version (VERSION=... or a version.txt with 'igneumd ')" +if [[ ${#KITS[@]} -gt 0 ]]; then + mkdir -p "$stage/packs" + for k in "${KITS[@]}"; do + [[ -f "$k" ]] || die "no kit zip at $k" + unzip -q -o "$k" -d "$stage/packs" || die "kit $k does not unzip" + log "kit $(basename "$k") sha256 $(shasum -a 256 "$k" 2>/dev/null | awk '{print $1}' || sha256sum "$k" | awk '{print $1}') unpacked under packs/" + done + # every pack directory holds program.json; its program_id is what the pack-id gate reads + n=0; while IFS= read -r pj; do d="$(dirname "$pj")"; id="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1])).get("program_id",""))' "$pj" 2>/dev/null || true)"; log "pack ${d#"$stage/packs/"}: program_id ${id:-?}"; n=$((n+1)); done < <(find "$stage/packs" -name program.json | sort) + [[ $n -gt 0 ]] || die "the kit(s) hold no pack (no program.json found)" + printf 'packs: %s pack(s) from %s (first-start convenience; the rig re-exports from its own node)\n' "$n" "$(for k in "${KITS[@]}"; do basename "$k"; done | tr '\n' ' ')" >> "$stage/version.txt" +fi cp "$HERE/h-config.sh" "$HERE/h-run.sh" "$HERE/h-stats.sh" "$HERE/README.md" "$stage/" sed "s/^CUSTOM_VERSION=.*/CUSTOM_VERSION=$VERSION/" "$HERE/h-manifest.conf" > "$stage/h-manifest.conf" chmod +x "$stage"/h-*.sh "$stage"/bin/* From 11888df9a76fcede68620a290a508490c3d6d6d3 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:10:18 +0000 Subject: [PATCH 09/10] Jobs publisher: a remove refuses a job the target's latest report shows running, and any job published with --installs-app, unless --force \"\" (7 Oct 2026 18:53 BST: a removal reached PC 2 one second after its job launched a silent installer over the running app; the runner's abort ended the process tree and the app went dark); tools/ci/publish-jobs-check.sh in the gate Co-Authored-By: Claude Fable 5.1 --- packaging/ota/publish-jobs.sh | 31 ++++++++++++++++++++++++++----- tools/ci/pre-push.sh | 1 + tools/ci/publish-jobs-check.sh | 30 ++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) create mode 100755 tools/ci/publish-jobs-check.sh diff --git a/packaging/ota/publish-jobs.sh b/packaging/ota/publish-jobs.sh index 38d2fdc2..055304c1 100755 --- a/packaging/ota/publish-jobs.sh +++ b/packaging/ota/publish-jobs.sh @@ -54,7 +54,7 @@ KIND="" TARGET="" PLATFORM="" REQUIRES="" REQUIRES_SET=0 ID="" TITLE="" EXPIRES_ SCRIPT="" SHELL_KIND="" ELEVATED=0 STOP_MINERS=0 TIMEOUT_MIN="" CARDS_OFF="" FILE="" URL="" SHA="" SIZE="" DIR="" TO="" EXTRACT=0 EXTRACT_DIR="" FRESH=0 GLOBS=() COMMAND="" WHAT="" -ZIP="" FIXTURES="" CAP_MIN="" DISTRO="" WSL_USER="" REMOVE_ID="" +ZIP="" FIXTURES="" CAP_MIN="" DISTRO="" WSL_USER="" REMOVE_ID="" FORCE="" INSTALLS_APP=0 TARGETS="" BUDGET_MIN="" STAGE_MIN="" MIN_FREE_GB="" TESTS=1 RELAY_URL="" NICE="" CARGO_JOBS="" case "$CMD" in remove) REMOVE_ID="${1:-}"; [ -n "$REMOVE_ID" ] || { echo "remove " >&2; exit 2; }; shift ;; @@ -67,6 +67,8 @@ while [ $# -gt 0 ]; do --requires) REQUIRES="$2"; REQUIRES_SET=1; [ "$REQUIRES" = none ] && REQUIRES=""; shift 2 ;; --id) ID="$2"; shift 2 ;; --title) TITLE="$2"; shift 2 ;; + --force) FORCE="$2"; shift 2 ;; # remove: override the running-job refusal, with the reason (7 Oct 2026) + --installs-app) INSTALLS_APP=1; shift ;; # add --kind run: the script installs over the app; never removed without --force --expires-hours) EXPIRES_H="$2"; shift 2 ;; --script) SCRIPT="$2"; shift 2 ;; --shell) SHELL_KIND="$2"; shift 2 ;; @@ -227,8 +229,8 @@ if [ "$CMD" = list ]; then [ -f "$JOBS" ] || { echo "no jobs file in $DEST"; exit 0; } "$SIGNER" verify-jobs "$PUB" "$JOBS" "$JOBS.sig" || { echo "the file in $DEST does not verify; run: $0 sign" >&2; exit 1; } if [ -f "$SIGNED" ]; then "$SIGNER" verify-signed-jobs "$PUB" "$SIGNED" >/dev/null || { echo "the envelope in $DEST does not verify; run: $0 sign" >&2; exit 1; }; else echo "(no igneum-jobs.signed.json yet; the next write makes one)"; fi - python3 - "$JOBS" <<'PY' -import json, sys, datetime + INSTALLS_APP="$INSTALLS_APP" python3 - "$JOBS" <<'PY' +import json, sys, datetime, os f = json.load(open(sys.argv[1])) now = datetime.datetime.now(datetime.timezone.utc) for j in f.get("jobs", []): @@ -353,17 +355,36 @@ except Exception: print("")' "${ZIP%.zip}.json" 2>/dev/null || true)" esac [ -n "$PLATFORM" ] || PLATFORM=any [ -n "$ID" ] || ID="$KIND-$(date -u +%Y%m%d-%H%M%S)" - NEW_JOB="$(python3 -c 'import json,sys,datetime + NEW_JOB="$(INSTALLS_APP="$INSTALLS_APP" python3 -c 'import json,sys,datetime,os a=sys.argv now=datetime.datetime.now(datetime.timezone.utc) t={"machine_ids": "all" if a[2]=="all" else [x.strip().lower() for x in a[2].split(",") if x.strip()], "platform": a[3]} +if os.environ.get("INSTALLS_APP")=="1" and a[5]=="run": t.setdefault("params",{})["installs_app"]=True if a[4]: t["requires"]=[x.strip() for x in a[4].split(",") if x.strip()] print(json.dumps({"id": a[1], "kind": a[5], "title": a[6], "created_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "expires_at": (now+datetime.timedelta(hours=float(a[7]))).strftime("%Y-%m-%dT%H:%M:%SZ"), "target": t, "params": json.loads(a[8]), "report": "log-intake"}))' "$ID" "$TARGET" "$PLATFORM" "$REQUIRES" "$KIND" "$TITLE" "$EXPIRES_H" "$PARAMS")" fi +# ---- the removal guard (7 October 2026, 18:53 BST: a removal reached PC 2 one second after its job had launched a silent +# installer over the running app; the runner's abort-on-removal ended the process tree and the app went dark). A remove refuses +# when any target's latest report for the id has started and carries no final line (the job is running), or when the job was +# published with --installs-app; `--force ""` overrides, and the reason is printed. The read is tools/jobs.mjs . +# REMOVE_GUARD_READ= replaces the read for the self-test (tools/ci/publish-jobs-check.sh). +if [ -n "$REMOVE_ID" ]; then + read_out="$( if [ -n "${REMOVE_GUARD_READ:-}" ]; then cat "$REMOVE_GUARD_READ"; else node "$ROOT/tools/jobs.mjs" "$REMOVE_ID" 2>/dev/null || true; fi )" + running="$(printf '%s\n' "$read_out" | grep -cE '^(job [^ ]+ \(.*\) on .* started|== running the)' || true)" + final="$(printf '%s\n' "$read_out" | grep -cE '^(SUMMARY: |job [^ ]+: (done|failed|aborted|timeout))' || true)" + jobs_now="$JOBS"; [ -f "$DEST/igneum-jobs.json" ] && jobs_now="$DEST/igneum-jobs.json" # the file at the destination this run writes + installs="$(python3 -c 'import json,sys; j=[x for x in json.load(open(sys.argv[1])).get("jobs",[]) if x.get("id")==sys.argv[2]]; print(1 if j and "\"installs_app\": true" in json.dumps(j[0]) else 0)' "$jobs_now" "$REMOVE_ID" 2>/dev/null || echo 0)" + if [ -z "$FORCE" ]; then + if [ "$running" -gt 0 ] && [ "$final" = 0 ]; then echo "remove refused: $REMOVE_ID has started on a machine and has no final line yet (a removal ends the running job's process tree); wait for its SUMMARY, or --force \"\"" >&2; exit 3; fi + if [ "$installs" = 1 ]; then echo "remove refused: $REMOVE_ID was published with --installs-app (it installs over the app); --force \"\" to remove it anyway" >&2; exit 3; fi + else + echo "remove: --force given ($FORCE); running=$running final=$final installs_app=$installs" + fi +fi # ---- merge: current jobs minus expired (minus a removed id), plus the new one; canonical JSON --------------------- NEW="$JOBS.new" -python3 - "$JOBS" "$NEW" "$NEW_JOB" "$REMOVE_ID" <<'PY' +INSTALLS_APP="$INSTALLS_APP" python3 - "$JOBS" "$NEW" "$NEW_JOB" "$REMOVE_ID" <<'PY' import json, sys, datetime, os cur, out, new_job, remove = sys.argv[1:5] now = datetime.datetime.now(datetime.timezone.utc) diff --git a/tools/ci/pre-push.sh b/tools/ci/pre-push.sh index 266cd712..3428f51b 100755 --- a/tools/ci/pre-push.sh +++ b/tools/ci/pre-push.sh @@ -106,6 +106,7 @@ tree_checks() { run "the class router is a preference with spill-over (a held or overloaded box hands the job to the other one)" bash tools/ci/route-spill-check.sh run "per-core leases, the quiet class and the reaper pass on the box (lease.sh and remote-run.sh self-tests over ssh)" bash tools/ci/box-locks-check.sh $( [ "$MODE" = ci ] && echo --ci ) run "the simulators job runs on master and release-* pushes and pull requests into them only" bash tools/ci/sims-branch-check.sh + run "publish-jobs.sh never removes a running or installs-app job without --force (the PC 2 abort class)" bash tools/ci/publish-jobs-check.sh run "no shell assignment hides behind a trailing comment (the swallowed-defaults class)" bash -c 'bash tools/ci/defaults-line-check.sh --self-test && bash tools/ci/defaults-line-check.sh' run "no script kills or finds a process by a plain name or a file name (pgrep/pkill -f literals, ps | grep)" bash -c 'bash tools/ci/kill-by-name-check.sh --self-test && bash tools/ci/kill-by-name-check.sh' run "the identity check's own self-test (excluded research path passes, exported leak fails)" bash tools/ci/identity-check.sh --self-test diff --git a/tools/ci/publish-jobs-check.sh b/tools/ci/publish-jobs-check.sh new file mode 100755 index 00000000..3334e94b --- /dev/null +++ b/tools/ci/publish-jobs-check.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# The removal guard of packaging/ota/publish-jobs.sh (7 October 2026, 18:53 BST: a removal reached PC 2 one second after its job had +# launched a silent installer over the running app; the runner's abort-on-removal ended the process tree and the app went dark). +# A remove must refuse while any target's report shows the job started with no final line, and refuse a job published with +# --installs-app; --force "" overrides. The check runs the script against a scratch destination (--dest) with the +# machine read replaced by a fixture (REMOVE_GUARD_READ), so nothing is published and no machine is read. +# +# tools/ci/publish-jobs-check.sh # exit 1 with the case that resolved wrongly (the check IS its self-test) +set -euo pipefail +cd "$(dirname "$0")/../.." +t=$(mktemp -d); trap 'rm -rf "$t"' EXIT +mkdir -p "$t/dest"; export IGNEUM_DLSITE="$t/site"; mkdir -p "$t/site/dl/testtoken" +P="packaging/ota/publish-jobs.sh" +# the real OTA key signs into the scratch --dest (as packaging/ota/test-publish-jobs.sh does; nothing is deployed, the downloads +# folder is untouched); a machine without the key or the signer skips the check as not applicable +[ -f "$HOME/.config/igneum/ota-signing-key" ] && [ -x app/igneum-app/target/release/igneum-ota-sign ] || { echo "publish-jobs-check: no OTA key or signer here; skipped as not applicable"; exit 0; } +printf 'echo hi\n' > "$t/s.ps1" +bash "$P" add --kind run --target 00000001 --script "$t/s.ps1" --id guard-run --title t --requires none --dest "$t/dest" >/dev/null 2>&1 || { echo "publish-jobs-check: add failed"; exit 1; } +bash "$P" add --kind run --target 00000001 --script "$t/s.ps1" --id guard-install --installs-app --title t --requires none --dest "$t/dest" >/dev/null 2>&1 || { echo "publish-jobs-check: add --installs-app failed"; exit 1; } +grep -q '"installs_app":true' "$t/dest/igneum-jobs.json" || { echo "publish-jobs-check: --installs-app did not write the param"; exit 1; } +fail=0 +printf 'job guard-run (run) on PC machine 0000000100000000 run job-guard-run-00000001, started 2026-10-07T18:00:00Z\n== running the powershell script (cap 40 min) ==\nRESULT start\n' > "$t/running.txt" +printf 'SUMMARY: done exit 0, started 2026-10-07T18:00:00Z, finished 2026-10-07T18:00:09Z, 9 s: script finished, exit 0\njob guard-run: done (exit 0) after 9 s: script finished\n' > "$t/done.txt" +if REMOVE_GUARD_READ="$t/running.txt" bash "$P" remove guard-run --dest "$t/dest" >/dev/null 2>&1; then echo "publish-jobs-check: a RUNNING job was removed"; fail=1; else echo "publish-jobs-check: a running job's removal is refused"; fi +if REMOVE_GUARD_READ="$t/running.txt" bash "$P" remove guard-run --dest "$t/dest" --force "test" >/dev/null 2>&1; then echo "publish-jobs-check: --force removes a running job (with the reason)"; else echo "publish-jobs-check: --force did not remove"; fail=1; fi +if REMOVE_GUARD_READ="$t/done.txt" bash "$P" remove guard-install --dest "$t/dest" >/dev/null 2>&1; then echo "publish-jobs-check: an --installs-app job was removed without --force"; fail=1; else echo "publish-jobs-check: an --installs-app job's removal is refused"; fi +bash "$P" add --kind run --target 00000001 --script "$t/s.ps1" --id guard-run2 --title t --requires none --dest "$t/dest" >/dev/null 2>&1 +if REMOVE_GUARD_READ="$t/done.txt" bash "$P" remove guard-run2 --dest "$t/dest" >/dev/null 2>&1; then echo "publish-jobs-check: a finished job's removal passes"; else echo "publish-jobs-check: a finished job's removal was refused"; fail=1; fi +[ "$fail" = 0 ] && echo "publish-jobs-check: a running or installs-app job is never removed without --force; a finished one is" +exit $fail From 65fdd39d5b6298ffca9f638730e739a4edfcb6c9 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Wed, 7 Oct 2026 18:23:14 +0000 Subject: [PATCH 10/10] Build boxes to near max (the project lead, 7 Oct 2026 19:4x BST): the bounded class is 88 of 96 cores with -j 88 (8 reserved for release builds, the seed and the observers), the jobs rule 88 alone / 44 shared, the spill line 80; checks updated. testnet-go.md: LG-2 waived by the owner, the new gate, the seeds held armed Co-Authored-By: Claude Fable 5.1 --- docs/plans/testnet-go.md | 12 ++++++++++++ infra/build-server/lib.sh | 4 ++-- infra/build-server/remote-run.sh | 13 ++++++++----- tools/build-remote.sh | 4 ++-- tools/ci/build-kind-default-check.sh | 6 +++--- tools/ci/route-spill-check.sh | 9 +++++---- 6 files changed, 32 insertions(+), 16 deletions(-) diff --git a/docs/plans/testnet-go.md b/docs/plans/testnet-go.md index 0668d6a8..85dbaa6a 100644 --- a/docs/plans/testnet-go.md +++ b/docs/plans/testnet-go.md @@ -135,3 +135,15 @@ Added by the launch-pack lane (branch `launch-pack`) from `docs/analysis/mission | LG-11 A signed proving customer | One customer paying for proofs at a published rate, or a signed letter of intent with a volume, before mainnet (the owner, 7 October 2026: a mainnet gate, not a testnet gate; the weight of the Devnet 2 gate: mainnet does not open without it) | the contract or the letter in the entity's records, a redacted copy linked from `docs/evidence.md`, the rate on the site; for the paying case the job market's payout contract shows a paid job for that customer; `grep -c "rollup signs for testnet" site/journey.json` is 0 after the handoff lands | M | NOT DONE: Taiko is named as the first customer and nothing is signed; the brief and the pilot progression are ledger X13 | the owner (the signature), the execution engineer (the paid job) | | LG-12 Hash origin daily for 90 days | The report posts every day for the first 90 days from the go, with no gap | `SELECT count(*) FROM hash_origin_reports WHERE day >= ''` reaches 90 with consecutive days; the timer's journal on the box shows a run per day | C | The job and its `--go ` flag exist; the timer is OWED (section 3 of the pack) | build-server lane (timer), the report | | LG-13 The disclosure prize | USD 50,000 for a reproduced break of the published hash class, paid in fiat by Igneum Labs LTD, announced only when escrowed and only when the entity's registered address exists on its documents and the owner gives the publish word | `docs/plans/funding.md` rule 3 (escrow before announcement); the staged text in docs/plans/cryptanalysis.md 3.3 on branch `cryptanalysis` (43d9700b, not on master yet); until the word, `grep -ci "50,000" site/*.html` is 0 | M (the announcement may come earlier, on the word) | APPROVED by the owner at 09:5x UK on 7 October 2026; STAGED; nothing public mentions it | the owner (escrow, the word), the cryptanalysis lane (the announcement) | + + +## LG-2 waived by the owner (7 October 2026, 19:5x BST) + +Launch gate LG-2 (seven daily hash-origin reports before the testnet) is waived by the owner, 7 October 2026 19:5x BST; the report +still runs daily from Devnet 3 (igneum-hash-origin-dn3.timer on build-1, 08:30 UTC, `--prefix dn3_`, DN3_GO_DATE 2026-10-07) and +from the testnet from its go. The testnet gate is now: the 24-hour proven window closed on Devnet 3 (about 19:00 BST on 8 October), +the 0.3.23 heights crossed on every Devnet 3 node, the launch text (LG-5) on the site, the seeds on the 0.3.22 object with the dry +run clean (done: fork 6ed56f63, digest 87d103b6, genesis 52a3e6a9, seed-class pair under /srv/artefacts/testnet-6ed56f63/seed/ on +build-1, three-seed dry run clean at height 0 at 18:44 BST), and the owner's word. The seeds stay armed for a go as early as the +evening of 8 October: `infra/build-server/wave1-0320.sh seeds --igneumd --sha256 80932b18… --miner +--digest 87d103b6… --commit 6ed56f63 --wipe-genesis --genesis 52a3e6a9… --go` on the word, nothing before. diff --git a/infra/build-server/lib.sh b/infra/build-server/lib.sh index 223d8956..b77d5e5b 100755 --- a/infra/build-server/lib.sh +++ b/infra/build-server/lib.sh @@ -33,13 +33,13 @@ bs_route() { # -> the } # Spill-over (the project lead, 7 October 2026, 15:02 UK: build-1 at load 139 with a queue of 1 h 40 min while build-2 read 4.5 with both # slots free). The class is a PREFERENCE, not a pin: a job goes to its class's box unless that box has no free slot or its 1-minute -# load is above BS_SPILL_LOAD (64), in which case it goes to the other box when THAT one has a free slot under the same load +# load is above BS_SPILL_LOAD (80 since 19:4x BST, was 64), in which case it goes to the other box when THAT one has a free slot under the same load # line; when neither qualifies it queues on its own box. The alternate of box 1 is box 2, of box 2 box 1, of box 3 box 1; a box # without a host file is never chosen. The decision is one line on the Mac (bs_log) and travels to the box in BR_ROUTE_* for the # JSONL row ("route": preferred, box, spilled, reason), so the dashboard shows it per job. A box is read with one ssh # (bs_box_state: free slots of the slot count, load1); BS_ROUTE_STATE_ in the environment replaces the ssh for the self-test # (tools/ci/route-spill-check.sh), "down" standing for an unreachable box. -BS_SPILL_LOAD="${BS_SPILL_LOAD:-64}" +BS_SPILL_LOAD="${BS_SPILL_LOAD:-80}" # the project lead, 7 Oct 2026 19:4x BST: both boxes to near max; was 64 bs_box_state() { # -> "free= slots= load1=" | "absent" | "down" local b="$1" v f h v=$(eval "printf '%s' \"\${BS_ROUTE_STATE_$b:-}\""); if [ -n "$v" ]; then printf '%s' "$v"; return 0; fi diff --git a/infra/build-server/remote-run.sh b/infra/build-server/remote-run.sh index 88b2b4d5..c0e0c196 100755 --- a/infra/build-server/remote-run.sh +++ b/infra/build-server/remote-run.sh @@ -190,10 +190,10 @@ if [ "${1:-}" = --self-test-slots ]; then after() { python3 -c "import sys; sys.exit(0 if float(open(sys.argv[1]).read()) >= float(open(sys.argv[2]).read()) else 1)" "$1" "$2"; } # 1. two concurrent builds: 45 jobs each fake a 6 & fake b 6 & wait - [ "$(cat "$t/a.jobs")" = JOBS=45 ] && [ "$(cat "$t/b.jobs")" = JOBS=45 ] || fail "two concurrent builds got $(cat "$t/a.jobs" "$t/b.jobs" | tr '\n' ' ') (want JOBS=45 JOBS=45)" + [ "$(cat "$t/a.jobs")" = JOBS=44 ] && [ "$(cat "$t/b.jobs")" = JOBS=44 ] || fail "two concurrent builds got $(cat "$t/a.jobs" "$t/b.jobs" | tr '\n' ' ') (want JOBS=44 JOBS=44)" # 2. one build alone: 90 fake c 1 - [ "$(cat "$t/c.jobs")" = JOBS=90 ] || fail "a lone build got $(cat "$t/c.jobs") (want JOBS=90)" + [ "$(cat "$t/c.jobs")" = JOBS=88 ] || fail "a lone build got $(cat "$t/c.jobs") (want JOBS=88)" # 3. a quiet measurement blocks an unbounded build (it starts only after the quiet ended) and lets a bounded suite run beside it fake m 9 1 & sleep 0.5; fake d 1 & BR_CORES=1 BR_NICE=10 fake s 1 & wait # the quiet holds 9 s: longer than a slot take plus the 3 s settle after "$t/d.start" "$t/m.end" || fail "an unbounded build started while a quiet measurement held the box (build start $(cat "$t/d.start"), quiet end $(cat "$t/m.end"))" @@ -214,8 +214,8 @@ if [ "${1:-}" = --self-test-slots ]; then grep -q 'self-test f' "$t/locks/build-0" || fail "the holder line of the busy slot build-0 was lost when another build probed it: '$(cat "$t/locks/build-0")'" wait # 6. the log carries the job count and the measure flag - grep -q '"jobs":45' "$t/log/builds.jsonl" && grep -q '"measure":true' "$t/log/builds.jsonl" || fail "builds.jsonl lacks jobs or measure fields" - echo "self-test-slots: two concurrent builds 45 each, a lone build 90, a quiet blocks an unbounded build and not a bounded suite, a quiet is refused beside a slot or a lease, a run keeps off leased cores, a probe keeps the holder line, the log carries jobs and measure"; exit 0 + grep -q '"jobs":44' "$t/log/builds.jsonl" && grep -q '"measure":true' "$t/log/builds.jsonl" || fail "builds.jsonl lacks jobs or measure fields" + echo "self-test-slots: two concurrent builds 44 each, a lone build 88, a quiet blocks an unbounded build and not a bounded suite, a quiet is refused beside a slot or a lease, a run keeps off leased cores, a probe keeps the holder line, the log carries jobs and measure"; exit 0 fi # One run per worktree directory at a time (6 October 2026, 19:51:09 UK: two runs of one worktree started in the same second; @@ -318,7 +318,7 @@ PY SLOTS_DIR="$IGNEUM_BUILD_SLOTS_DIR" slots=$(cat "$SLOTS_DIR/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1 -JOBS_ALONE="${JOBS_ALONE:-90}"; JOBS_SHARED="${JOBS_SHARED:-45}" +JOBS_ALONE="${JOBS_ALONE:-88}"; JOBS_SHARED="${JOBS_SHARED:-44}" # the project lead, 7 Oct 2026: 88 of 96 cores, 8 reserved for the release builds, the seed and the observers holder_line() { printf 'pid %s since %sZ waited %s s: %s\n' "$BR_PID" "$(date -u +%H:%M:%S)" "$1" "$BR_LABEL"; } give_up() { # echo "build-remote: gave up waiting for $1 after 2 h" >&2 @@ -469,6 +469,9 @@ BR_RUN_LOG="$RUN_LOG_DIR/$BR_HOST-$BR_T0-$BR_PID.log"; export BR_RUN_LOG # (since the third slot on build-2, 7 Oct 2026: a bounded run takes the band its SLOT owns, counted from the top: slot 0 the last N # cores, slot 1 the N below, slot 2 the N below that, so three bounded runs never share a core; a band below core 0 falls back to # the last N) +# (the project lead, 7 Oct 2026 19:4x BST, both boxes to near max: a bounded run takes the LAST N cores, N = 88 by default, leaving the first 8 +# to the release builds, the seed and the observer processes; with N above half the box the slots share the band, and nice 10 plus +# the per-slot jobs rule keep two suites fair; a smaller N (IGNEUM_BOUND_CORES) returns to disjoint bands per slot when it fits) ncpu=$(nproc); cores_str="0-$((ncpu - 1))" if [ "${BR_CORES:-0}" -gt 0 ] && [ "${BR_CORES}" -lt "$ncpu" ]; then band=0; case "${got:-}" in ''|measure) ;; *) band=$got ;; esac diff --git a/tools/build-remote.sh b/tools/build-remote.sh index 365f4193..b61254b3 100755 --- a/tools/build-remote.sh +++ b/tools/build-remote.sh @@ -110,7 +110,7 @@ case "${CARGO_ARGS[0]:-build}" in *) SCHED_CLASS=build ;; esac if [ "$PRIORITY" = gate ]; then BR_NICE=0; BR_CORES=0; BR_JOBS_CAP=0 -elif [ "$SCHED_CLASS" = suite ] || [ "$SCHED_CLASS" = bench ]; then BR_NICE=10; BR_CORES=32; BR_JOBS_CAP=32; fi +elif [ "$SCHED_CLASS" = suite ] || [ "$SCHED_CLASS" = bench ]; then BR_NICE=10; BR_CORES="${IGNEUM_BOUND_CORES:-88}"; BR_JOBS_CAP="${IGNEUM_BOUND_CORES:-88}"; fi # the project lead, 7 Oct 2026 19:4x BST: load both boxes to near max; 88 of 96, 8 reserved # an explicit --jobs above the bounded class's cap is clamped (main's rule: bounded unless a priority flag; 7 Oct 2026: two suites # ran at -j 90 on a box at load 190 because their callers passed --jobs 90) if [ "$BR_JOBS_CAP" -gt 0 ] && [ -n "$JOBS" ] && [ "$JOBS" -gt "$BR_JOBS_CAP" ]; then bs_log "--jobs $JOBS clamped to $BR_JOBS_CAP for a $SCHED_CLASS (pass --priority gate for the full set)"; JOBS=$BR_JOBS_CAP; fi @@ -132,7 +132,7 @@ bs_context if [ "$BS_CRATE_REL" = proving/igneum-prove ] && [ -z "${BOX_GIVEN:-}" ] && [ "$PRIORITY" != gate ]; then bs_route_spill prove; [ "$BS_ROUTE_BOX" != "$BOX" ] && { BOX=$BS_ROUTE_BOX; bs_host "$BOX"; }; fi # box 2 keeps the suites' numbers: everything that lands there runs at the bounded class (nice 10, a 32-core band, -j 32), builds # and gates included (main, 7 Oct 2026); a gate still takes its slot ahead of queued suites -if [ "$BOX" = 2 ] && [ "$BR_CORES" = 0 ]; then BR_NICE=10; BR_CORES=32; BR_JOBS_CAP=32; export BR_NICE BR_CORES BR_JOBS_CAP; bs_log "bounded on box 2 (nice 10, a 32-core band, -j 32) so a suite beside it keeps its number"; fi +if [ "$BOX" = 2 ] && [ "$BR_CORES" = 0 ]; then BR_NICE=10; BR_CORES="${IGNEUM_BOUND_CORES:-88}"; BR_JOBS_CAP="${IGNEUM_BOUND_CORES:-88}"; export BR_NICE BR_CORES BR_JOBS_CAP; bs_log "bounded on box 2 (nice 10, the ${BR_CORES}-core band, -j $BR_JOBS_CAP) so a suite beside it keeps its number"; fi if [ "$BR_JOBS_CAP" -gt 0 ] && [ -n "$JOBS" ] && [ "$JOBS" -gt "$BR_JOBS_CAP" ]; then bs_log "--jobs $JOBS clamped to $BR_JOBS_CAP on box $BOX"; JOBS=$BR_JOBS_CAP; fi bs_log "box $BOX ($BS_HOST) for class $SCHED_CLASS, priority $PRIORITY$( [ "${BR_ROUTE_SPILLED:-0}" = 1 ] && echo ", SPILLED from box $BR_ROUTE_PREF")" diff --git a/tools/ci/build-kind-default-check.sh b/tools/ci/build-kind-default-check.sh index 7a448c5a..d3688763 100755 --- a/tools/ci/build-kind-default-check.sh +++ b/tools/ci/build-kind-default-check.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # The scheduling classes of tools/build-remote.sh (main, 7 October 2026, the load-190 night): a build-remote call WITHOUT a priority -# flag must put every suite (cargo test) and bench (cargo bench) into the bounded class, nice 10 on 32 cores with -j 32, while a +# flag must put every suite (cargo test) and bench (cargo bench) into the bounded class, nice 10 on 88 cores with -j 88 (the project lead, 7 Oct 2026 19:4x BST: both boxes to near max, 8 cores reserved), while a # build keeps the box's own jobs rule and --priority gate gives nice 0 on the full set. This check runs the tool's --plan mode (no box, # no crate) for the four shapes and compares the resolved class; a change that lets a bare `cargo test` run unbounded again fails it. # @@ -14,8 +14,8 @@ expect() { # if [ "$got" = "$want" ]; then echo "build-kind: [$*] -> $got"; else echo "build-kind: [$*] resolved to '$got', expected '$want'" >&2; return 1; fi } fail=0 -expect 'kind=suite nice=10 cores=32 jobs=32 priority=normal' -- test -p kaspa-consensus-core --lib || fail=1 -expect 'kind=bench nice=10 cores=32 jobs=32 priority=normal' -- bench -p igneum-pow || fail=1 +expect 'kind=suite nice=10 cores=88 jobs=88 priority=normal' -- test -p kaspa-consensus-core --lib || fail=1 +expect 'kind=bench nice=10 cores=88 jobs=88 priority=normal' -- bench -p igneum-pow || fail=1 expect 'kind=build nice=0 cores=96 jobs=box priority=normal' -- build --release -p kaspad || fail=1 expect 'kind=gate nice=0 cores=96 jobs=box priority=gate' --priority gate -- test -p igneum-app || fail=1 expect 'kind=check nice=0 cores=96 jobs=box priority=normal' -- check || fail=1 diff --git a/tools/ci/route-spill-check.sh b/tools/ci/route-spill-check.sh index 0da8b3d0..ceadf82b 100755 --- a/tools/ci/route-spill-check.sh +++ b/tools/ci/route-spill-check.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # The spill-over router of tools/build-remote.sh (lib.sh bs_route_spill; the project lead, 7 October 2026, 15:02 UK: build-1 at load 139 with a # queue of 1 h 40 min while build-2 sat at 4.5 with free slots). The class is a preference: a job goes to its class's box unless -# that box has no free slot or its 1-minute load is above 64, in which case it goes to the other box when that one qualifies, else +# that box has no free slot or its 1-minute load is above 80 (was 64), in which case it goes to the other box when that one qualifies, else # it queues on its own box. This check feeds the router fixed box states through BS_ROUTE_STATE_ (no ssh) and host files in a # scratch directory, and compares the chosen box and the spilled flag for the known cases: a held build-1 selects build-2, a free # build-1 selects build-1, an overloaded build-1 spills, a held build-2 sends a suite to build-1, two full boxes queue on the @@ -22,17 +22,18 @@ expect() { # ; the states come from t } BS_ROUTE_STATE_1="free=0 slots=2 load1=30" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a held build-1 selects build-2" build 2 1 BS_ROUTE_STATE_1="free=1 slots=2 load1=20" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a free build-1 selects build-1" build 1 0 -BS_ROUTE_STATE_1="free=2 slots=2 load1=70" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "an overloaded build-1 spills" build 2 1 +BS_ROUTE_STATE_1="free=2 slots=2 load1=95" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "an overloaded build-1 spills" build 2 1 BS_ROUTE_STATE_1="free=0 slots=2 load1=30" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a gate spills like a build" gate 2 1 BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=0 slots=3 load1=10" expect "a held build-2 sends a suite to build-1" suite 1 1 -BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=1 slots=3 load1=90" expect "an overloaded build-2 sends a bench away" bench 1 1 +BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=1 slots=3 load1=120" expect "an overloaded build-2 sends a bench away" bench 1 1 BS_ROUTE_STATE_1="free=2 slots=2 load1=5" BS_ROUTE_STATE_2="free=2 slots=3 load1=10" expect "a free build-2 keeps its suite" suite 2 0 BS_ROUTE_STATE_1="free=0 slots=2 load1=80" BS_ROUTE_STATE_2="free=0 slots=3 load1=90" expect "two full boxes queue a build on build-1" build 1 0 BS_ROUTE_STATE_1="free=0 slots=2 load1=80" BS_ROUTE_STATE_2="free=0 slots=3 load1=90" expect "two full boxes queue a suite on build-2" suite 2 0 BS_ROUTE_STATE_1="down" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a build-1 that is down spills" build 2 1 BS_ROUTE_STATE_1="free=1 slots=2 load1=5" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a proving class with no box 3 prefers box 1" prove 1 0 BS_ROUTE_STATE_1="free=0 slots=2 load1=5" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "a proving class spills to box 2 when box 1 is held" prove 2 1 -BS_ROUTE_STATE_1="free=1 slots=2 load1=64" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "load exactly 64 is under the line" build 1 0 +BS_ROUTE_STATE_1="free=1 slots=2 load1=80" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "load exactly 80 is under the line" build 1 0 +BS_ROUTE_STATE_1="free=2 slots=2 load1=70" BS_ROUTE_STATE_2="free=3 slots=3 load1=5" expect "load 70 stays on build-1 (the line is 80)" build 1 0 # the ssh path itself under the hook's shell (/bin/bash is 3.2 on the Mac; the pool lane found `BS_SSH_OPTS[@]: unbound variable` # at the first unpinned route, 7 Oct 2026): a host file pointing at a port nothing answers on must read "down" in a few seconds, # not die on an unset array