From 697297dfceacfebc928abe04c69b4b9345be42c6 Mon Sep 17 00:00:00 2001 From: igneum-josh <337424239+igneum-josh@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:01:29 +0100 Subject: [PATCH] Block rate on Devnet 2: the plan file (runs A 10 bps and B 1 bps on the 38 wave cards, the readings, the per-tier payout arithmetic) and bps-collect.py; the remote-build tooling from master --- docs/analysis/block-rate-devnet2.md | 34 +++ infra/build-server/lib.sh | 181 +++++++++++++ infra/build-server/provision.sh | 398 ++++++++++++++++++++++++++++ infra/build-server/remote-run.sh | 110 ++++++++ infra/build-server/run-from-mac.sh | 62 +++++ tools/build-remote.sh | 112 ++++++++ tools/cross-remote.sh | 124 +++++++++ tools/fleet/bps-collect.py | 51 ++++ 8 files changed, 1072 insertions(+) create mode 100644 docs/analysis/block-rate-devnet2.md create mode 100755 infra/build-server/lib.sh create mode 100755 infra/build-server/provision.sh create mode 100755 infra/build-server/remote-run.sh create mode 100755 infra/build-server/run-from-mac.sh create mode 100755 tools/build-remote.sh create mode 100755 tools/cross-remote.sh create mode 100644 tools/fleet/bps-collect.py diff --git a/docs/analysis/block-rate-devnet2.md b/docs/analysis/block-rate-devnet2.md new file mode 100644 index 000000000..428fcb0e1 --- /dev/null +++ b/docs/analysis/block-rate-devnet2.md @@ -0,0 +1,34 @@ +# Block rate on Devnet 2: 10 blocks per second against 1, on the rented fleet + +6 October 2026, Josh's experiment (through the coordinator, 17:5xZ): "Devnet 2 at a higher block rate for solo miners +(Kaspa's answer)". Branch `gpu-fleet`; the node profile on the fork branch `devnet2-bps` (1279a1d6 on release-0.3.14-node +4c6b129d): a suffixed devnet started with `IGNEUMD_DEVNET_BPS=10` (or 5) runs `BlockrateParams::new::<10>()` with the +TenBps subsidy and target time, Kaspa's Crescendo-style constants for that rate (k 124, merge depth, sample rates, +mergeset limit, parents, coinbase maturity from `consensus/core/src/config/bps.rs`); the shared devnet never reads the +variable, so the live digest and rules are untouched. Built on igneum-build-1 (`tools/build-remote.sh`). Numbers land +below as they are measured; every figure carries its source file under `~/Desktop/fleet/bps/`. + +## The runs + +| Run | Chain | Miners | Length | What is read | +|---|---|---|---|---| +| A | igneum-devnet-2, fresh genesis, 10 bps | the 38 wave pods plus the 4 Devnet 2 boxes (42 cards, about 2.0 GH/s) | 60 min | blocks per second achieved, blue and red blocks per minute (orphan rate), blue score growth, max reorg depth, checkpoint lock delay and weight at this voter count, p2p bytes per node per minute, node CPU and RSS, exec lag (height minus executed tip), payout intervals per miner from the coinbase records | +| B | the same boxes, fresh genesis, 1 bps | the same | 30 min | the same (the control) | + +Payout interval per tier, from each run's measured block rate and the chain's hash: a 4070 at 28 MH/s, a 5090 at +128 MH/s, an 8x 4090 rig at 459 MH/s, at tonight's network hash and extrapolated to 1, 10 and 100 TH/s networks +(interval = blocks per second x miner share, the arithmetic in `tools/fleet/bps-collect.py`). + +## Measured + +RUN_A + +RUN_B + +## Per tier + +TIER_TABLE + +## The recommendation for mainnet's rate + +RECOMMENDATION diff --git a/infra/build-server/lib.sh b/infra/build-server/lib.sh new file mode 100755 index 000000000..b0dcfbd9e --- /dev/null +++ b/infra/build-server/lib.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Shared by infra/build-server/run-from-mac.sh, tools/build-remote.sh and tools/cross-remote.sh. Source it, do not run it. +# Everything that talks to igneum-build-1 from the Mac goes through here: the host line, the ssh options (the ops key +# ~/.ssh/igneum_ed25519, a shared control socket so one build is one ssh session), the mirror push, the remote checkout +# and the source overlay (rsync by checksum, changed files re-stamped: the copied-sources rule of 5 October 2026). +# +# Layout on the box (provision.sh): /srv/builds/ mirrors the Mac's igneum worktree ROOT (the directory that holds +# igneum-pow/, app/, proving/ and vendor/), so the fork's relative path dependency `../../../../igneum-pow` +# (vendor/igneum-node/consensus/pow/Cargo.toml) resolves on the box exactly as on the Mac: +# Mac /Users/joshm/Projects/igneum-wt-ship0311/vendor/igneum-node-0311 -> box /srv/builds/igneum-wt-ship0311/vendor/igneum-node-0311 +# Mac /Users/joshm/Projects/igneum-wt-ship0311/igneum-pow -> box /srv/builds/igneum-wt-ship0311/igneum-pow +# Mac /Users/joshm/Projects/igneum/app/igneum-app -> box /srv/builds/igneum/app/igneum-app +# The fork's kaspa-build-info reads `git rev-parse HEAD` at build time and the release plans check the commit in the binary's +# strings, so the fork tree on the box is a real clone of the bare mirror /srv/igneum-node.git checked out at the Mac's HEAD, +# with the Mac's uncommitted changes rsynced on top. The igneum repo's crates get the same from /srv/igneum.git. + +# shellcheck disable=SC2034 # shared with the scripts that source lib.sh +BS_KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}" +BS_HOST_FILE="${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" # one line: build@ +BS_ROOT_REMOTE=/srv/builds +BS_MIRROR_REPO=/srv/igneum.git +BS_MIRROR_NODE=/srv/igneum-node.git + +bs_log() { printf '%s %s: %s\n' "$(date -u +%H:%M:%S)" "${BS_TOOL:-build-server}" "$*" >&2; } +bs_die() { bs_log "ERROR: $*"; exit 1; } + +bs_host() { + BS_HOST="${BUILD_HOST:-}" + if [ -z "$BS_HOST" ]; then + [ -s "$BS_HOST_FILE" ] || bs_die "no build server: write build@ to $BS_HOST_FILE (infra/build-server/run-from-mac.sh does) or set BUILD_HOST" + BS_HOST="$(head -1 "$BS_HOST_FILE" | tr -d '[:space:]')" + fi + case "$BS_HOST" in *@*) ;; *) bs_die "BUILD_HOST must be user@host, got '$BS_HOST'" ;; esac + [ -r "$BS_KEY" ] || bs_die "no ssh key at $BS_KEY" + mkdir -p "$HOME/.ssh/cm" + BS_SSH_OPTS=(-i "$BS_KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=30 -o ServerAliveCountMax=6 + -o ControlMaster=auto -o ControlPath="$HOME/.ssh/cm/igneum-build-%r@%h:%p" -o ControlPersist=900) + BS_SSH_CMD="ssh"; local o; for o in "${BS_SSH_OPTS[@]}"; do BS_SSH_CMD="$BS_SSH_CMD $(printf '%q' "$o")"; done +} + +bs_ssh() { ssh "${BS_SSH_OPTS[@]}" "$BS_HOST" "$@"; } +bs_rsync() { rsync -e "$BS_SSH_CMD" "$@"; } + +# the two sides' rustc must agree (the box is pinned by provision.sh RUST_TOOLCHAIN; the Mac runs rustup's stable): +# a different compiler gives different bytes and, across a minor version, different lints and errors +bs_toolchain_check() { + local mac box + mac=$("${CARGO_HOME:-$HOME/.cargo}/bin/rustc" --version 2>/dev/null | awk '{ print $2 }') + box=$(bs_ssh '. /etc/profile.d/igneum-build.sh; rustc --version' 2>/dev/null | awk '{ print $2 }') + [ -n "$box" ] || bs_die "cannot read rustc on $BS_HOST (is it provisioned? infra/build-server/run-from-mac.sh)" + if [ "$mac" != "$box" ]; then + if [ "${IGNEUM_TOOLCHAIN_MISMATCH:-}" = ok ]; then bs_log "WARNING: rustc $mac on the Mac, $box on the box (IGNEUM_TOOLCHAIN_MISMATCH=ok)" + else bs_die "rustc $mac on the Mac, $box on the box; re-provision with RUST_TOOLCHAIN=$mac or set IGNEUM_TOOLCHAIN_MISMATCH=ok"; fi + else bs_log "rustc $box on both sides"; fi +} + +# Where am I? Sets BS_KIND (node = a worktree of the fork under vendor/; repo = a crate of the igneum repo), BS_TOP (the git +# top level of the crate's repo), BS_WT_ROOT (the igneum worktree root), BS_WT (its name = the directory on the box), +# BS_CRATE (the crate dir, = $PWD), BS_CRATE_REL (relative to BS_WT_ROOT), BS_MIRROR, BS_BRANCH, BS_SHA, BS_REMOTE_WT, +# BS_REMOTE_CRATE, and BS_LOCAL_DIRS (every directory of a path dependency, relative to BS_WT_ROOT, from cargo metadata). +bs_context() { + BS_CRATE="$PWD" + [ -f "$BS_CRATE/Cargo.toml" ] || bs_die "no Cargo.toml in $BS_CRATE: run from the crate directory (the fork worktree, igneum-pow, app/igneum-app, proving/igneum-prove)" + BS_TOP=$(git -C "$BS_CRATE" rev-parse --show-toplevel 2>/dev/null) || bs_die "$BS_CRATE is not inside a git worktree" + case "$BS_TOP" in + */vendor/*) + BS_KIND=node; BS_MIRROR=$BS_MIRROR_NODE + BS_WT_ROOT=$(cd "$BS_TOP/../.." && pwd) + [ -f "$BS_WT_ROOT/igneum-pow/Cargo.toml" ] || bs_die "$BS_TOP looks like a fork worktree but $BS_WT_ROOT/igneum-pow is missing" + ;; + *) + BS_KIND=repo; BS_MIRROR=$BS_MIRROR_REPO; BS_WT_ROOT="$BS_TOP" + ;; + esac + BS_WT=$(basename "$BS_WT_ROOT") + BS_CRATE_REL=$(python3 -c 'import os, sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))' "$BS_CRATE" "$BS_WT_ROOT") + BS_TOP_REL=$(python3 -c 'import os, sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))' "$BS_TOP" "$BS_WT_ROOT") + case "$BS_CRATE_REL" in ..*) bs_die "$BS_CRATE is outside the worktree root $BS_WT_ROOT" ;; esac + BS_BRANCH=$(git -C "$BS_TOP" branch --show-current 2>/dev/null || true) + BS_SHA=$(git -C "$BS_TOP" rev-parse HEAD) + [ -n "$BS_BRANCH" ] || BS_BRANCH="detached-$(git -C "$BS_TOP" rev-parse --short HEAD)" + BS_REMOTE_WT="$BS_ROOT_REMOTE/$BS_WT" + BS_REMOTE_CRATE="$BS_REMOTE_WT/$BS_CRATE_REL" + # every local (path) package of the crate's dependency graph, as directories relative to the worktree root; the ones inside + # BS_TOP are covered by the git checkout plus the overlay of BS_TOP itself (node kind) or synced one by one (repo kind) + BS_LOCAL_DIRS=$(cd "$BS_CRATE" && "${CARGO_HOME:-$HOME/.cargo}/bin/cargo" metadata --format-version 1 2>/dev/null | python3 -c ' +import json, os, sys +d = json.load(sys.stdin); root = sys.argv[1]; top = sys.argv[2]; kind = sys.argv[3] +dirs = set() +for p in d["packages"]: + if p["source"] is not None: continue + m = os.path.dirname(p["manifest_path"]) + if kind == "node" and (m == top or m.startswith(top + "/")): dirs.add(top); continue + dirs.add(m) +out = [] +for m in sorted(dirs): + r = os.path.relpath(m, root) + if r.startswith(".."): sys.exit("path dependency %s is outside the worktree root %s" % (m, root)) + out.append(r) +print("\n".join(out))' "$BS_WT_ROOT" "$BS_TOP" "$BS_KIND") || bs_die "cargo metadata failed in $BS_CRATE" + [ -n "$BS_LOCAL_DIRS" ] || bs_die "cargo metadata listed no local packages in $BS_CRATE" +} + +# push the crate repo's HEAD to its bare mirror on the box (fast after the first time), then check the remote tree out at that +# commit ON A BRANCH of that name: kaspa-build-info (build-info/build.rs try_git_head) embeds the commit only when .git is a +# directory AND HEAD is a symbolic ref to a loose branch file; a detached HEAD or a worktree's .git file gives an empty hash +# (which is why the Mac's worktree builds print "igneumd 2.1.0" with no commit, 6 Oct 2026). The mirror doubles as the CI +# runner's source later. +bs_push_and_checkout() { + local url="$BS_HOST:$BS_MIRROR" remote_top="$BS_REMOTE_WT/$BS_TOP_REL" + [ "$BS_TOP_REL" = . ] && remote_top="$BS_REMOTE_WT" + bs_log "push $BS_TOP HEAD $BS_SHA ($BS_BRANCH) -> $url" + GIT_SSH_COMMAND="$BS_SSH_CMD" git -C "$BS_TOP" push -q --force "$url" "HEAD:refs/heads/$BS_BRANCH" || bs_die "push to the mirror failed" + bs_ssh "set -e; mkdir -p '$BS_REMOTE_WT' + if [ ! -d '$remote_top/.git' ]; then rm -rf '$remote_top'; git clone -q --no-checkout '$BS_MIRROR' '$remote_top'; fi + cd '$remote_top'; git fetch -q origin '+refs/heads/*:refs/remotes/origin/*'; git checkout -q -B '$BS_BRANCH' '$BS_SHA'; git reset -q --hard '$BS_SHA' + git status --porcelain | head -3" || bs_die "remote checkout at $remote_top failed" + BS_REMOTE_TOP="$remote_top" +} + +# rsync one directory of the worktree to the same place on the box. By checksum and WITHOUT preserving times, so a file whose +# content changed is written with the box's clock and nothing older than the last build slips past cargo's mtime check (the +# stale-build class, 4 and 5 October 2026); the files rsync wrote are listed and re-stamped with touch as well, so the rule is +# visible here and tools/ci/copied-sources-check.sh sees it. target dirs and .git never travel; --delete keeps the box equal to +# the Mac inside the directory (excluded paths are protected). +bs_overlay_dir() { + local rel="$1" src="$BS_WT_ROOT/$1" dst="$BS_REMOTE_WT/$1" list nfiles ndirs + [ -d "$src" ] || bs_die "no $src" + list=$(mktemp) + bs_ssh "mkdir -p '$dst'" + bs_rsync -rlpgoD --checksum --delete --out-format='%n' \ + --exclude '/target' --exclude '/target-*' --exclude '/target/' --exclude 'target-*/' --exclude '.git' --exclude '.DS_Store' --exclude 'node_modules' \ + "$src/" "$BS_HOST:$dst/" > "$list" || { rm -f "$list"; bs_die "rsync of $rel failed"; } + # files only: directories are listed whenever an attribute differs (a fresh clone's ownership), and a tree whose files + # were all identical lists ONLY directories (first run of 6 October 2026: an empty file list failed the pipeline) + nfiles=$(grep -vc '/$' "$list" || true); ndirs=$(grep -c '/$' "$list" || true) + if [ "${nfiles:-0}" -gt 0 ]; then + if ! grep -v '/$' "$list" | tr '\n' '\0' | bs_ssh "cd '$dst' && xargs -0 -r touch --no-create"; then rm -f "$list"; bs_die "re-stamp of $rel failed"; fi + fi + bs_log "overlay $rel -> $dst: ${nfiles:-0} file(s) written and re-stamped, ${ndirs:-0} dir(s)" + rm -f "$list" +} + +bs_sync_sources() { + local d + bs_push_and_checkout + for d in $BS_LOCAL_DIRS; do bs_overlay_dir "$d"; done +} + +bs_sha256() { shasum -a 256 "$1" | awk '{ print $1 }'; } +bs_size() { stat -f %z "$1" 2>/dev/null || stat -c %s "$1"; } +bs_fmt_secs() { local s=$1; printf '%d min %02d s' $((s / 60)) $((s % 60)); } + +# kind of a run for the box's JSONL log (main's rule of 6 October 2026): +bs_kind() { + local tool="$1" word="$2" + case "$word" in test) echo suite; return ;; check|clippy) echo check; return ;; esac + if [ "$tool" = cross-remote ]; then + case "$BS_KIND:$BS_CRATE_REL" in node:*) echo node-windows ;; repo:app/igneum-app) echo app-windows ;; *) echo other ;; esac; return + fi + case "$BS_KIND:$BS_CRATE_REL" in node:*) echo node-linux ;; repo:app/igneum-app) echo app ;; repo:proving/igneum-prove) echo prove ;; *) echo other ;; esac +} + +# the remote runner (infra/build-server/remote-run.sh, piped to `bash -s` on the box behind the BR_* exports): takes one of the +# box's build slots (never the Mac's), runs the command in the crate dir with sccache, prints the RESULT line and appends one +# JSON line to /srv/builds/_log/builds.jsonl for the worker dashboard. +# bs_remote_run