diff --git a/docs/plans/build-server.md b/docs/plans/build-server.md index 9cdd3d385..b731b2bef 100644 --- a/docs/plans/build-server.md +++ b/docs/plans/build-server.md @@ -286,3 +286,44 @@ The box stays a build and test machine. If main wants a CPU prover anyway for co the shape is a systemd unit as `build` with `SP1_PROVER=cpu`, a throwaway devnet key (never the OTA key, never a hand's key), `--threads 48`, Nice 19 and the measure hold taken for the whole run, which would exclude builds for minutes at a time: that is why it is not written. + +## 8. The capacity layer (6 October 2026, Josh: "get the builder spun up to capacity") + +The box sits idle most of the minute between builds. `infra/build-server/capacity/` is a background workload layer that +uses the idle cores and yields to builds. It runs as `igneum-capacity.service` (user `build`, `Nice=19`, `chrt -i 0` +SCHED_IDLE, `IOSchedulingClass=idle`, `CPUQuota=8800%` so 8 of the 96 threads are always free) and is a controller +(`run.sh`) over a queue of jobs in `jobs/`. All work lives under `/srv/capacity`; the layer takes NO build slot and +writes NOTHING under `/srv/builds/`. + +### Rules (the layer obeys these; they are why a build never waits on it) + +| Rule | How | +|---|---| +| Background work never takes a build slot | the jobs run cargo directly in `/srv/capacity` and never call `remote-run.sh`; the controller's `cap_build_active` only READS `/srv/builds/_locks` with `flock -n` | +| It pauses whenever a build slot is taken or the measure hold exists | `run.sh` polls `/srv/builds/_locks` every 5 s; it `SIGSTOP`s the running job's whole process group the instant any `build-` or `measure` is held and `SIGCONT`s it when they clear; it does not even start a new slice while a build runs | +| It never touches `/srv/builds/` trees | its own checkout is `/srv/capacity/src` (repo) and `/srv/capacity/src/vendor/igneum-node` (fork), its targets are under `/srv/capacity`; the only `/srv/builds` access is a READ of an existing node binary and a READ of the lock files | +| It is killed by the night battery's start and restarted after | `igneum-night-battery.service` has `ExecStartPre=+-systemctl stop igneum-capacity.service` and `ExecStopPost=+-systemctl start igneum-capacity.service` (the `+` runs as root; the `-` never fails the battery) | + +### The jobs, in priority order (`CAP_SEQUENCE` gives the earlier ones more turns) + +| # | Job | What | Out | Dry-run / smoke | +|---|---|---|---|---| +| 1 | `pow-fuzz` | continuous `igneum-pow` mixer and scratch fuzz; `IGNEUM_FUZZ_SEED_BASE` advances from a cursor so every slice walks fresh programs; counts programs and units, saves any mismatch with its seed base | `/srv/capacity/out/pow-fuzz//` | `--dry-run` / `--smoke` (N 100, 10 min) | +| 2 | `sync-fuzz` | the sync-request gate for the 28 unwrap sites (`docs/analysis/horizon/consensus-security.md` s5): a throwaway pruned `igneumd` on a simnet datadir on the box (net `igneum-devnet-315`, loopback only, NEVER the live devnet), fed random, boundary and below-retention locator/header/antipast/IBD/pruning-point requests by `igneum-p2p-probe sync-fuzz`; a gRPC alive check every 25 requests; any panic saved with the trace | `/srv/capacity/out/sync-fuzz//` | `--dry-run` (builds one request of each kind) / `--smoke` (6 min of requests) | +| 3 | `sim-sweeps` | GHOSTDAG (`ghostdag_sim.py --seed-base`), finality (`finality_horizon.py`, `finality_v2.py --quick`) and difficulty attacks (`attacks.py --seed-base`) across seeds 1 to 1,000; CSV appended; a daily note of any bound that moved | `/srv/capacity/out/sim-sweeps//sweeps.csv` | `--dry-run` / `--smoke` (one seed, quick) | +| 4 | `model-sweeps` | the N-ladder and chip model (`sim/horizon/algorithm/model.py`) over every section, cached by the file's content hash | `/srv/capacity/out/model-sweeps//` | `--dry-run` / `--smoke` | +| 5 | `clippy-audit` | `cargo clippy` and `cargo audit` on every branch pushed to the box repo mirror in the last day, in its own checkout `/srv/capacity/clippy`, recorded per branch and commit so a slice only picks up new pushes | `/srv/capacity/out/clippy-audit///` | `--dry-run` / `--smoke` (one crate, newest branch) | + +Each job is a script in `jobs/` with a `--dry-run` mode (no build, no node, no run) and a `--smoke` mode (a ~10 minute +bounded run). Every job writes one line per slice into `/srv/workers/capacity.json` (`summary.mjs`, atomic), which the +worker dashboard's collector (`tools/workers/collect.mjs`) reads into `doc.background`; the page (`tools/workers/page`) +shows a "Background" lane from it. + +### Install + +`infra/build-server/capacity/install.sh` (idempotent): copies the scripts and the unit, pushes the `capacity-probe` +fork branch (the `sync-fuzz` subcommand) and the `box-capacity` repo branch to the box mirrors, and enables the +service. The layer tracks the repo branch `master`; until this work merges, the mirror's `master` lacks the capacity +tree, so install writes a drop-in pinning `CAP_REPO_BRANCH=box-capacity` and removes it once `master` carries +`infra/build-server/capacity/run.sh` (self-healing after the merge). `--no-start` enables without starting; +`--smoke ` installs then runs one job's 10-minute smoke and prints the summary. diff --git a/igneum-pow/tests/mixer.rs b/igneum-pow/tests/mixer.rs index b6b3b8bfc..eebda7f47 100644 --- a/igneum-pow/tests/mixer.rs +++ b/igneum-pow/tests/mixer.rs @@ -129,6 +129,9 @@ fn write_pack_with_bases(dir: &PathBuf, e: &Epoch, day: &str, bases: &[u32], sou #[test] fn fuzz_v3_programs_cpu() { let n: usize = std::env::var("IGNEUM_MIXER_FUZZ").ok().and_then(|s| s.parse().ok()).unwrap_or(200); + // IGNEUM_FUZZ_SEED_BASE (default 0) offsets the seed index so a continuous fuzzer (the box's capacity layer, + // infra/build-server/capacity) walks fresh programs round after round; the default run is unchanged + let base: usize = std::env::var("IGNEUM_FUZZ_SEED_BASE").ok().and_then(|s| s.parse().ok()).unwrap_or(0); let out = std::env::var("IGNEUM_MIXER_PACKS_OUT").ok().map(PathBuf::from); // IGNEUM_MIXER_CLASS=mx8 fuzzes the x8 candidate as a load class (generator 2 with the class in the id); the // default is V3_CLASS through the seam; IGNEUM_MIXER_ERA composes a test era over the class (class_under_test) @@ -142,7 +145,7 @@ fn fuzz_v3_programs_cpu() { let mut manifest = String::from("pack\tlog2\tprogram_id\tbases\n"); let mut units = 0usize; let mut wraps = 0usize; - for i in 0..n { + for i in base..base + n { let seed = format!("igneum-mixer-fuzz/{i}"); let p = program_of(&seed, class, era); contract(&p, &seed, class, era); @@ -173,7 +176,7 @@ fn fuzz_v3_programs_cpu() { } mh.insert(log2, e.dataset); } - println!("fuzz: {n} {} programs, {units} units on the CPU, {wraps} units in the top 256 nonces", class.name()); + println!("fuzz: {n} {} programs, {units} units on the CPU, {wraps} units in the top 256 nonces, seeds {base}..{}", class.name(), base + n); assert_eq!(units, 4 * n); assert_eq!(wraps, n); if let Some(dir) = &out { diff --git a/igneum-pow/tests/scratch.rs b/igneum-pow/tests/scratch.rs index 0fe68a9be..5ad1c873d 100644 --- a/igneum-pow/tests/scratch.rs +++ b/igneum-pow/tests/scratch.rs @@ -645,6 +645,8 @@ fn contract(p: &Program) { #[test] fn fuzz_scr_programs_cpu() { let n: usize = std::env::var("IGNEUM_SCRATCH_FUZZ").ok().and_then(|s| s.parse().ok()).unwrap_or(200); + // IGNEUM_FUZZ_SEED_BASE (default 0): the seed index offset for the continuous fuzzer (infra/build-server/capacity) + let base: usize = std::env::var("IGNEUM_FUZZ_SEED_BASE").ok().and_then(|s| s.parse().ok()).unwrap_or(0); let out = std::env::var("IGNEUM_SCRATCH_PACKS_OUT").ok().map(PathBuf::from); let mut rng = SplitMix64::new(0x6967_6e65_756d_2d73); // "igneum-s" let day = "2026-10-03"; @@ -676,7 +678,7 @@ fn fuzz_scr_programs_cpu() { } } } - for i in 0..n { + for i in base..base + n { let name = CLASSES[rng.below(CLASSES.len() as u64) as usize]; let c = class(name); let seed = format!("igneum-scratch-fuzz/{i}"); @@ -721,7 +723,7 @@ fn fuzz_scr_programs_cpu() { } let mut classes: Vec<_> = per_class.iter().collect(); classes.sort(); - println!("fuzz: {n} programs, {units} units on the CPU, {wraps} units in the top 256 nonces, classes {classes:?}"); + println!("fuzz: {n} programs, {units} units on the CPU, {wraps} units in the top 256 nonces, classes {classes:?}, seeds {base}..{}", base + n); assert_eq!(units, 4 * n); assert_eq!(wraps, n, "every program has a unit in the top 256 nonces"); if let Some(dir) = &out { diff --git a/infra/build-server/capacity/igneum-capacity.service b/infra/build-server/capacity/igneum-capacity.service new file mode 100644 index 000000000..0aa3ec9ca --- /dev/null +++ b/infra/build-server/capacity/igneum-capacity.service @@ -0,0 +1,38 @@ +# igneum-build-1: the background capacity layer (infra/build-server/capacity/run.sh). Uses idle cores and yields to +# builds. Installed by infra/build-server/capacity/install.sh. User build, Nice 19, SCHED_IDLE (chrt -i 0), idle IO, +# and a CPU quota that leaves 8 of the 96 threads free for ssh, rsync and the system. The controller itself pauses +# every job (SIGSTOP) the instant a build slot or the measure hold is taken, so a build never waits on this. +[Unit] +Description=Igneum background capacity layer (pow fuzz, sync fuzz, sim and model sweeps, clippy and audit; yields to builds) +After=network-online.target local-fs.target +Wants=network-online.target + +[Service] +Type=simple +User=build +Group=build +Environment=HOME=/home/build +Environment=PATH=/home/build/.cargo/bin:/usr/local/bin:/usr/bin:/bin +Environment=SCCACHE_DIR=/srv/sccache +Environment=IGNEUM_BUILD_SLOTS_DIR=/srv/builds/_locks +Environment=IGNEUM_BUILD_ROOT=/srv/builds +Environment=IGNEUM_CAPACITY_JSON=/srv/workers/capacity.json +WorkingDirectory=/srv/capacity +# SCHED_IDLE for the whole tree: a runnable build thread always preempts it +ExecStart=/usr/bin/chrt -i 0 /bin/bash /srv/capacity/bin/run.sh +Nice=19 +CPUSchedulingPolicy=idle +IOSchedulingClass=idle +# the hard ceiling: 88 of 96 threads, so 8 are always free even if the controller's pause lagged +CPUQuota=8800% +CPUWeight=1 +MemoryMax=96G +Restart=always +RestartSec=10 +# a stop (and the night battery's pre-stop) must reach the whole job tree, not just run.sh +KillMode=control-group +KillSignal=SIGTERM +TimeoutStopSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/infra/build-server/capacity/install.sh b/infra/build-server/capacity/install.sh new file mode 100755 index 000000000..d3efbc3a2 --- /dev/null +++ b/infra/build-server/capacity/install.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Install or refresh the background capacity layer on igneum-build-1 from this Mac. Idempotent. +# infra/build-server/capacity/install.sh copy the scripts and the unit, push the probe branch, enable the service +# infra/build-server/capacity/install.sh --no-start install but do not start (enable only) +# infra/build-server/capacity/install.sh --smoke install, then run one job's 10-minute smoke on the box and print its summary +# Needs root over ssh (root@ with ~/.ssh/igneum_ed25519); the host ip comes from ~/.config/igneum/build-server. +# The layer takes no build slot and writes only under /srv/capacity and /srv/workers/capacity.json. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; ROOT="$(cd "$HERE/../../.." && pwd)" +KEY="${IGNEUM_BUILD_KEY:-$HOME/.ssh/igneum_ed25519}" +HOST_LINE="$(head -1 "${IGNEUM_BUILD_HOST_FILE:-$HOME/.config/igneum/build-server}" | tr -d '[:space:]')" +IP="${HOST_LINE#*@}"; [ -n "$IP" ] || { echo "no build server in ~/.config/igneum/build-server" >&2; exit 1; } +SSH=(ssh -i "$KEY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10) +NOSTART=0; SMOKE_JOB="" +while [ $# -gt 0 ]; do case "$1" in --no-start) NOSTART=1;; --smoke) SMOKE_JOB="$2"; shift;; *) echo "unknown arg $1" >&2; exit 2;; esac; shift; done + +echo "capacity: installing on $IP" +# the probe branch (igneum-p2p-probe sync-fuzz) must exist on the node mirror so the box can check it out. The branch +# may live in this worktree's fork, or in the shared main checkout's fork (both share the same git); use whichever has it. +FORK="$ROOT/vendor/igneum-node-capacity" +[ -d "$FORK/.git" ] || git -C "$FORK" rev-parse -q --verify capacity-probe >/dev/null 2>&1 || FORK="$ROOT/vendor/igneum-node" +[ -d "$FORK" ] || FORK="$(cd "$ROOT/.." && pwd)/igneum/vendor/igneum-node" +if [ -d "$FORK" ] && git -C "$FORK" rev-parse -q --verify capacity-probe >/dev/null 2>&1; then + GIT_SSH_COMMAND="ssh -i $KEY -o BatchMode=yes" git -C "$FORK" push -q --force build capacity-probe && echo "capacity: pushed capacity-probe to the node mirror" || echo "capacity: WARNING push of capacity-probe failed (the box will fall back to the newest release-*-node)" >&2 +else + echo "capacity: note: no capacity-probe branch in $FORK; the box will fall back to the newest release-*-node for the sync-fuzz probe" >&2 +fi + +# directories owned by build +"${SSH[@]}" "root@$IP" 'install -d -o build -g build /srv/capacity /srv/capacity/bin /srv/capacity/bin/jobs /srv/capacity/out /srv/capacity/state /srv/capacity/log; install -d -o build -g build /srv/workers' +# the scripts +scp -q -i "$KEY" "$HERE/run.sh" "$HERE/lib.sh" "$HERE/summary.mjs" "build@$IP:/srv/capacity/bin/" +scp -q -i "$KEY" "$HERE/jobs/"*.sh "build@$IP:/srv/capacity/bin/jobs/" +"${SSH[@]}" "root@$IP" 'chown -R build:build /srv/capacity/bin; chmod +x /srv/capacity/bin/run.sh /srv/capacity/bin/jobs/*.sh' +# the unit, and the night battery unit (it now kills and restarts the layer) +scp -q -i "$KEY" "$HERE/igneum-capacity.service" "root@$IP:/etc/systemd/system/" +if [ -f "$HERE/../night/igneum-night-battery.service" ]; then scp -q -i "$KEY" "$HERE/../night/igneum-night-battery.service" "root@$IP:/etc/systemd/system/"; fi +# the layer tracks the repo branch master in production. Until this work merges, the box mirror's master lacks +# infra/build-server/capacity and the sim seed-base edits, so point the layer at box-capacity with a drop-in and push +# that branch. The drop-in removes itself once master carries the capacity run.sh (self-healing after the merge). +REPO_SRC="$ROOT"; [ -f "$REPO_SRC/infra/build-server/capacity/run.sh" ] || REPO_SRC="$(cd "$ROOT/.." && pwd)/igneum" +if git -C "$REPO_SRC" rev-parse -q --verify box-capacity >/dev/null 2>&1; then + GIT_SSH_COMMAND="ssh -i $KEY -o BatchMode=yes" git -C "$REPO_SRC" push -q --force build box-capacity 2>/dev/null \ + && echo "capacity: pushed box-capacity to the repo mirror" || echo "capacity: WARNING could not push box-capacity (is the build remote wired? run-from-mac.sh)" >&2 +fi +if "${SSH[@]}" "build@$IP" 'git -C /srv/igneum.git cat-file -e master:infra/build-server/capacity/run.sh 2>/dev/null'; then + "${SSH[@]}" "root@$IP" 'rm -f /etc/systemd/system/igneum-capacity.service.d/branch.conf; rmdir /etc/systemd/system/igneum-capacity.service.d 2>/dev/null || true' + echo "capacity: master carries the capacity layer; the layer tracks master" +else + "${SSH[@]}" "root@$IP" 'install -d /etc/systemd/system/igneum-capacity.service.d; printf "[Service]\nEnvironment=CAP_REPO_BRANCH=box-capacity\n" > /etc/systemd/system/igneum-capacity.service.d/branch.conf' + echo "capacity: master does not carry the capacity layer yet; drop-in pins CAP_REPO_BRANCH=box-capacity until the merge" +fi +"${SSH[@]}" "root@$IP" 'systemctl daemon-reload; systemctl enable --quiet igneum-capacity.service 2>/dev/null || true' + +if [ -n "$SMOKE_JOB" ]; then + echo "capacity: smoke $SMOKE_JOB (up to ~10 min)" + "${SSH[@]}" "build@$IP" "IGNEUM_CAPACITY_JSON=/srv/workers/capacity.json nice -n 19 chrt -i 0 bash /srv/capacity/bin/jobs/$SMOKE_JOB.sh --smoke; echo '--- capacity.json ---'; cat /srv/workers/capacity.json" + exit 0 +fi + +if [ "$NOSTART" = 1 ]; then echo "capacity: installed and enabled, NOT started (--no-start)"; exit 0; fi +"${SSH[@]}" "root@$IP" 'systemctl restart igneum-capacity.service; sleep 2; systemctl is-active igneum-capacity.service; systemctl --no-pager --lines=0 status igneum-capacity.service | sed -n 1,3p' +echo "capacity: running; journalctl -u igneum-capacity -n 30; summary at /srv/workers/capacity.json" diff --git a/infra/build-server/capacity/jobs/clippy-audit.sh b/infra/build-server/capacity/jobs/clippy-audit.sh new file mode 100755 index 000000000..236f4289e --- /dev/null +++ b/infra/build-server/capacity/jobs/clippy-audit.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Capacity job 5 (lowest priority): cargo clippy and cargo audit on every branch pushed to the box mirror within the +# last day, results per branch. Uses its own checkout (CAP_ROOT/clippy) so it never disturbs the other jobs' tree, and +# records which branches it has already done at a given commit so a slice only picks up new pushes. +# jobs/clippy-audit.sh --dry-run list the branches it would check; run nothing +# jobs/clippy-audit.sh --smoke clippy+audit one crate of the newest branch (~10 min) +# jobs/clippy-audit.sh --slice-s N check branches until about this long elapses +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; export CAP_JOB=clippy-audit; export CAP_PRIORITY=5 +. "$HERE/../lib.sh" + +DRY=0; SMOKE=0; SLICE_S="${CAP_SLICE_S:-2400}"; DAYS="${CAP_CLIPPY_DAYS:-1}" +while [ $# -gt 0 ]; do case "$1" in --dry-run) DRY=1;; --smoke) SMOKE=1; SLICE_S=600;; --slice-s) SLICE_S="$2"; shift;; *) cap_say "unknown arg $1";; esac; shift; done + +since=$(( $(date +%s) - DAYS * 86400 )) +# branches pushed to the repo mirror within the window, newest first +mapfile -t BRANCHES < <(git -C "$REPO_MIRROR" for-each-ref --sort=-committerdate --format='%(refname:short) %(committerdate:unix) %(objectname:short)' refs/heads 2>/dev/null | awk -v s="$since" '$2 >= s {print $1" "$3}') + +if [ "$DRY" = 1 ]; then + cap_say "DRY RUN: branches pushed in the last $DAYS day(s): ${#BRANCHES[@]}" + for b in "${BRANCHES[@]}"; do echo " $b"; done >&2 + printf '{"state":"idle","summary":%s,"counters":{"branches_due":%s,"checked":0},"dry_run":true}' "$(cap_json_str "dry run: ${#BRANCHES[@]} branch(es) in the last $DAYS day")" "${#BRANCHES[@]}" | cap_summary clippy-audit + exit 0 +fi + +CLIP="$CAP_ROOT/clippy"; mkdir -p "$CLIP" +[ -d "$CLIP/.git" ] || git clone -q "$REPO_MIRROR" "$CLIP" >/dev/null 2>&1 || { cap_say "clone failed"; exit 1; } +git -C "$CLIP" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' 2>/dev/null || true +OUT=$(cap_out_dir clippy-audit); done_file="$CAP_STATE/clippy-done.tsv"; touch "$done_file" +export CARGO_TARGET_DIR="$CLIP/target-capacity" +CRATES="${CAP_CLIPPY_CRATES:-igneum-pow igneum-census pool proto-vdf}" +[ "$SMOKE" = 1 ] && CRATES="igneum-pow" + +deadline=$(( $(date +%s) + SLICE_S )); checked=0; warn_total=0; err_total=0; audit_vulns=0 +for entry in "${BRANCHES[@]}"; do + [ "$(date +%s)" -lt "$deadline" ] || break + b="${entry% *}"; sha="${entry#* }" + grep -qF "$b $sha" "$done_file" && continue + git -C "$CLIP" checkout -q -- . 2>/dev/null || true + git -C "$CLIP" checkout -q -B "cap-$b" "origin/$b" 2>/dev/null || { cap_say "cannot check out $b"; continue; } + bdir="$OUT/$b"; mkdir -p "$bdir"; bw=0; be=0 + for c in $CRATES; do + [ -f "$CLIP/$c/Cargo.toml" ] || continue + [ "$(date +%s)" -lt "$deadline" ] || break + if timeout 1200 cap_cargo "$CLIP/$c" clippy --release --all-targets > "$bdir/clippy-$c.log" 2>&1; then :; else be=$((be + 1)); fi + bw=$((bw + $(grep -c '^warning: ' "$bdir/clippy-$c.log" 2>/dev/null || echo 0))) + if timeout 300 cap_cargo "$CLIP/$c" audit --color never > "$bdir/audit-$c.log" 2>&1; then :; else audit_vulns=$((audit_vulns + $(grep -c '^Crate:' "$bdir/audit-$c.log" 2>/dev/null || echo 0))); fi + done + echo -e "$b\t$sha\t$(date -u +%H:%M:%SZ)\twarnings=$bw\terrors=$be" >> "$done_file" + warn_total=$((warn_total + bw)); err_total=$((err_total + be)); checked=$((checked + 1)) + cap_say "$b @ $sha: $bw clippy warnings, $be clippy errors" +done +sum="checked $checked branch(es) this slice of ${#BRANCHES[@]} due; $warn_total clippy warnings, $err_total clippy errors, $audit_vulns audit advisories" +printf '{"state":"ran","summary":%s,"counters":{"branches_due":%s,"checked":%s,"warnings":%s,"errors":%s,"audit_advisories":%s},"out":%s}' \ + "$(cap_json_str "$sum")" "${#BRANCHES[@]}" "$checked" "$warn_total" "$err_total" "$audit_vulns" "$(cap_json_str "$OUT")" | cap_summary clippy-audit +cap_say "$sum" diff --git a/infra/build-server/capacity/jobs/model-sweeps.sh b/infra/build-server/capacity/jobs/model-sweeps.sh new file mode 100755 index 000000000..02b720fdb --- /dev/null +++ b/infra/build-server/capacity/jobs/model-sweeps.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Capacity job 4: the N-ladder and chip model sweeps (sim/horizon/algorithm/model.py) over the parameter grid, cached. +# model.py is pure arithmetic on cited inputs and prints every section as markdown; this job runs each section, caches +# the output under the out dir keyed by the model.py content hash, and only re-runs a section when model.py changed. +# jobs/model-sweeps.sh --dry-run print the plan; run nothing +# jobs/model-sweeps.sh --smoke run every section once (seconds) +# jobs/model-sweeps.sh --slice-s N same as a normal run (the model is fast); ignored beyond caching +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; export CAP_JOB=model-sweeps; export CAP_PRIORITY=4 +. "$HERE/../lib.sh" + +DRY=0; while [ $# -gt 0 ]; do case "$1" in --dry-run) DRY=1;; --smoke) ;; --slice-s) shift;; *) cap_say "unknown arg $1";; esac; shift; done +SECTIONS="${CAP_MODEL_SECTIONS:-chip fpga v5 ladder era verifier schedule}" + +if [ "$DRY" = 1 ]; then + cap_say "DRY RUN: would run model.py sections: $SECTIONS, cached by content hash" + printf '{"state":"idle","summary":%s,"counters":{"sections":0,"ran":0,"cached":0},"dry_run":true}' "$(cap_json_str "dry run: sections $SECTIONS")" | cap_summary model-sweeps + exit 0 +fi +[ -f "$CAP_SRC/sim/horizon/algorithm/model.py" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; } +MODEL="$CAP_SRC/sim/horizon/algorithm/model.py" +OUT=$(cap_out_dir model-sweeps) +h=$(sha256sum "$MODEL" | cut -c1-16) +ran=0; cached=0; total=0 +for sec in $SECTIONS; do + total=$((total + 1)) + cache="$CAP_OUT_ROOT/model-sweeps/cache/$sec-$h.md"; mkdir -p "$(dirname "$cache")" + if [ -f "$cache" ]; then cached=$((cached + 1)); cp "$cache" "$OUT/$sec.md"; continue; fi + if timeout 300 python3 "$MODEL" --section "$sec" > "$OUT/$sec.md" 2>>"$CAP_LOG/model.log"; then + cp "$OUT/$sec.md" "$cache"; ran=$((ran + 1)) + else cap_say "section $sec FAILED"; echo "FAILED" > "$OUT/$sec.md"; fi +done +sum="$total sections (model hash $h): $ran ran, $cached from cache" +printf '{"state":"ran","summary":%s,"counters":{"sections":%s,"ran":%s,"cached":%s},"model_hash":%s,"out":%s}' \ + "$(cap_json_str "$sum")" "$total" "$ran" "$cached" "$(cap_json_str "$h")" "$(cap_json_str "$OUT")" | cap_summary model-sweeps +cap_say "$sum" diff --git a/infra/build-server/capacity/jobs/pow-fuzz.sh b/infra/build-server/capacity/jobs/pow-fuzz.sh new file mode 100755 index 000000000..f1cc6ef07 --- /dev/null +++ b/infra/build-server/capacity/jobs/pow-fuzz.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Capacity job 1 (highest priority): continuous igneum-pow fuzz. Builds the two fuzz test binaries once into the +# layer's own target dir, then runs them with IGNEUM_FUZZ_SEED_BASE advancing from a cursor so every slice walks fresh +# programs. Reads igneum-pow/tests (the mixer and scratch fuzz harnesses the night battery also calls). Counts programs +# generated and units (vectors) checked; any mismatch or panic is saved with its seed under the out dir and counted. +# jobs/pow-fuzz.sh --dry-run build nothing, run nothing; print the plan and the cursor; exit 0 +# jobs/pow-fuzz.sh --smoke a ~10 minute bounded run (small N, one build) +# jobs/pow-fuzz.sh --slice-s 3600 run fuzz for about this long, then write the summary and exit (the controller's slice) +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; export CAP_JOB=pow-fuzz; export CAP_PRIORITY=1 +. "$HERE/../lib.sh" + +DRY=0; SMOKE=0; SLICE_S="${CAP_SLICE_S:-3600}"; N_PER="${CAP_POW_N:-250}" +while [ $# -gt 0 ]; do case "$1" in --dry-run) DRY=1;; --smoke) SMOKE=1; SLICE_S=600; N_PER=100;; --slice-s) SLICE_S="$2"; shift;; *) cap_say "unknown arg $1";; esac; shift; done + +base=$(cap_cursor_get pow-fuzz 0) +if [ "$DRY" = 1 ]; then + cap_say "DRY RUN: would build igneum-pow mixer+scratch fuzz binaries in $CAP_SRC/igneum-pow, then run IGNEUM_FUZZ_SEED_BASE=$base IGNEUM_MIXER_FUZZ=$N_PER IGNEUM_SCRATCH_FUZZ=$N_PER" + printf '{"state":"idle","summary":%s,"counters":{"programs":0,"vectors":0,"panics":0},"dry_run":true,"next_seed_base":%s}' "$(cap_json_str "dry run: next seed base $base, N $N_PER")" "$base" | cap_summary pow-fuzz + exit 0 +fi + +[ -d "$CAP_SRC/igneum-pow" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; } +OUT=$(cap_out_dir pow-fuzz) +POW="$CAP_SRC/igneum-pow" +export CARGO_TARGET_DIR="$CAP_ROOT/target/pow" + +# build the two fuzz test binaries once (cargo test --no-run); the layer's SCHED_IDLE cargo yields to real builds +cap_say "building igneum-pow fuzz binaries (seed base $base, N $N_PER, slice ${SLICE_S}s)" +if ! cap_cargo "$POW" test --release --no-run --test mixer --test scratch > "$CAP_LOG/pow-build.log" 2>&1; then + cap_say "build FAILED (see $CAP_LOG/pow-build.log)" + printf '{"state":"error","summary":%s,"counters":{"panics":0}}' "$(cap_json_str "build failed: $(grep -m1 '^error' "$CAP_LOG/pow-build.log" | cut -c1-120)")" | cap_summary pow-fuzz + exit 1 +fi + +deadline=$(( $(date +%s) + SLICE_S )) +total_programs=$(cap_cursor_get pow-fuzz-programs 0) +total_vectors=$(cap_cursor_get pow-fuzz-vectors 0) +panics=$(cap_cursor_get pow-fuzz-panics 0) +slice_programs=0; slice_vectors=0; rounds=0 +mani="$OUT/fuzz.tsv"; [ -f "$mani" ] || echo -e "utc\tseed_base\tn\ttest\tprograms\tunits\tresult" > "$mani" + +while [ "$(date +%s)" -lt "$deadline" ]; do + log="$CAP_LOG/pow-slice.log" + if IGNEUM_FUZZ_SEED_BASE=$base IGNEUM_MIXER_FUZZ=$N_PER IGNEUM_SCRATCH_FUZZ=$N_PER \ + cap_cargo "$POW" test --release --test mixer --test scratch -- fuzz --nocapture > "$log" 2>&1; then + # the fuzz lines: "fuzz: ... programs, units ... seeds .." + while IFS= read -r line; do + n=$(echo "$line" | grep -oE '^fuzz: [0-9]+' | grep -oE '[0-9]+' | head -1) + units=$(echo "$line" | grep -oE '[0-9]+ units' | grep -oE '[0-9]+' | head -1) + [ -n "$n" ] || continue + which=mixer; echo "$line" | grep -q 'classes' && which=scratch + slice_programs=$((slice_programs + n)); slice_vectors=$((slice_vectors + ${units:-0})) + echo -e "$(date -u +%H:%M:%S)\t$base\t$N_PER\t$which\t$n\t${units:-0}\tpass" >> "$mani" + done < <(grep '^fuzz:' "$log") + else + # a mismatch or a panic: save the whole log with the seed base, count it, stop the slice + panics=$((panics + 1)) + save="$OUT/mismatch-seedbase-$base-$(date -u +%H%M%S).log"; cp "$log" "$save" + echo -e "$(date -u +%H:%M:%S)\t$base\t$N_PER\tboth\t0\t0\tMISMATCH:$save" >> "$mani" + cap_say "MISMATCH at seed base $base, saved $save" + break + fi + base=$((base + N_PER)); rounds=$((rounds + 1)) + cap_cursor_set pow-fuzz "$base" + total_programs=$((total_programs + 2 * N_PER)); total_vectors=$((total_vectors + slice_vectors - (total_vectors - total_vectors))) +done +total_programs=$(cap_cursor_get pow-fuzz-programs 0); total_programs=$((total_programs + slice_programs)) +total_vectors=$(cap_cursor_get pow-fuzz-vectors 0); total_vectors=$((total_vectors + slice_vectors)) +cap_cursor_set pow-fuzz-programs "$total_programs"; cap_cursor_set pow-fuzz-vectors "$total_vectors"; cap_cursor_set pow-fuzz-panics "$panics" + +saved_json=$(ls "$OUT"/mismatch-*.log 2>/dev/null | sed 's|.*/||' | "$NODE_BIN" -e 'const l=require("fs").readFileSync(0,"utf8").split("\n").filter(Boolean);process.stdout.write(JSON.stringify(l))' 2>/dev/null || echo '[]') +sum="rounds $rounds, this slice $slice_programs programs and $slice_vectors units, total $total_programs programs; $panics mismatch(es); next seed base $base" +printf '{"state":"ran","summary":%s,"counters":{"programs":%s,"vectors":%s,"panics":%s,"slice_programs":%s,"slice_vectors":%s},"next_seed_base":%s,"out":%s,"saved":%s}' \ + "$(cap_json_str "$sum")" "$total_programs" "$total_vectors" "$panics" "$slice_programs" "$slice_vectors" "$base" "$(cap_json_str "$OUT")" "${saved_json:-[]}" | cap_summary pow-fuzz +cap_say "$sum" +[ "$panics" = 0 ] diff --git a/infra/build-server/capacity/jobs/sim-sweeps.sh b/infra/build-server/capacity/jobs/sim-sweeps.sh new file mode 100755 index 000000000..5ef72a1dc --- /dev/null +++ b/infra/build-server/capacity/jobs/sim-sweeps.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Capacity job 3: GHOSTDAG and finality simulator sweeps across the adversary grid the Horizon lanes used, seeds walking +# 1 to 1,000 from a cursor. Runs sim/horizon/consensus-security/ghostdag_sim.py and finality_horizon.py, sim/finality_v2.py +# and the difficulty attack sims (sim/difficulty/attacks/attacks.py). Results appended as CSV under the out dir; a daily +# summary notes any bound that moved against the previous day. Pure Python (numpy on the box), no build, no node. +# jobs/sim-sweeps.sh --dry-run print the plan and the next seed; run nothing +# jobs/sim-sweeps.sh --smoke one seed through each sim, quick mode (~10 min) +# jobs/sim-sweeps.sh --slice-s 3600 sweep seeds until about this long elapses +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; export CAP_JOB=sim-sweeps; export CAP_PRIORITY=3 +. "$HERE/../lib.sh" + +DRY=0; SMOKE=0; SLICE_S="${CAP_SLICE_S:-3600}"; SEEDS_PER="${CAP_SIM_SEEDS:-5}"; QUICK="" +while [ $# -gt 0 ]; do case "$1" in --dry-run) DRY=1;; --smoke) SMOKE=1; SLICE_S=600; SEEDS_PER=1; QUICK="--quick";; --slice-s) SLICE_S="$2"; shift;; *) cap_say "unknown arg $1";; esac; shift; done + +seed=$(cap_cursor_get sim-sweeps 1); [ "$seed" -ge 1 ] 2>/dev/null || seed=1 +if [ "$DRY" = 1 ]; then + cap_say "DRY RUN: would sweep ghostdag_sim, finality_horizon, finality_v2 and difficulty attacks from seed $seed, $SEEDS_PER per slice" + printf '{"state":"idle","summary":%s,"counters":{"seeds":0,"rows":0,"moved":0},"dry_run":true,"next_seed":%s}' "$(cap_json_str "dry run: next seed $seed")" "$seed" | cap_summary sim-sweeps + exit 0 +fi +[ -d "$CAP_SRC/sim" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; } +OUT=$(cap_out_dir sim-sweeps); CSV="$OUT/sweeps.csv"; [ -f "$CSV" ] || echo "utc,sim,seed,metric,value" > "$CSV" +cd "$CAP_SRC" +PY="python3" +deadline=$(( $(date +%s) + SLICE_S )); did=0; rows0=$(wc -l < "$CSV") + +emit() { echo "$(date -u +%H:%M:%S),$1,$2,$3,$4" >> "$CSV"; } + +while [ "$(date +%s)" -lt "$deadline" ] && [ "$seed" -le 1000 ]; do + s_end=$((seed + SEEDS_PER - 1)); [ "$s_end" -gt 1000 ] && s_end=1000; n=$((s_end - seed + 1)) + # 1. GHOSTDAG withholding: --seed-base offsets the episode seeds; capture the worst reorg depth and win rate + gd="$OUT/ghostdag-seed$seed.json" + if timeout 1200 $PY sim/horizon/consensus-security/ghostdag_sim.py --seeds "$n" --seed-base "$seed" --holds 30,60 --delays 0.67,2 --shares 0.34,0.51 --json "$gd" > "$OUT/ghostdag-seed$seed.md" 2>>"$CAP_LOG/sim.log"; then + mx=$($PY -c 'import json,sys;d=json.load(open(sys.argv[1]));print(max((c.get("reorg_max",0) for c in d["cells"]),default=0))' "$gd" 2>/dev/null || echo 0) + won=$($PY -c 'import json,sys;d=json.load(open(sys.argv[1]));print(round(max((c.get("won",0) for c in d["cells"]),default=0),3))' "$gd" 2>/dev/null || echo 0) + emit ghostdag "$seed-$s_end" reorg_max "$mx"; emit ghostdag "$seed-$s_end" won_max "$won" + else emit ghostdag "$seed-$s_end" error timeout_or_fail; fi + # 2. finality horizon (v2 and v3 over the share grid): the renter day-10 and day-20 share reaching the veto + if timeout 1800 $PY sim/horizon/consensus-security/finality_horizon.py --seeds "$seed" --sweeps R $QUICK --out "$OUT/finality-horizon-seed$seed.md" >>"$CAP_LOG/sim.log" 2>&1; then + emit finality_horizon "$seed" ran 1 + else emit finality_horizon "$seed" error timeout_or_fail; fi + # 3. difficulty attacks across the controllers (seed-base walks) + if timeout 1200 $PY sim/difficulty/attacks/attacks.py --seeds "$n" --seed-base "$seed" --scenario ts,hop --rules igneum > "$OUT/diff-attacks-seed$seed.md" 2>>"$CAP_LOG/sim.log"; then + emit difficulty "$seed-$s_end" ran 1 + else emit difficulty "$seed-$s_end" error timeout_or_fail; fi + did=$((did + n)); seed=$((s_end + 1)); cap_cursor_set sim-sweeps "$seed" + [ "$SMOKE" = 1 ] && break +done + +# 4. finality_v2 quick once per slice (the full scenario set, not seed-swept): a liveness and lock sanity check +timeout 900 $PY sim/finality_v2.py --quick --scenarios A,B,C,D,E,F,G > "$OUT/finality-v2-quick.md" 2>>"$CAP_LOG/sim.log" && emit finality_v2 quick ran 1 || emit finality_v2 quick error timeout_or_fail + +# daily summary: any metric that moved against yesterday's csv +prev=$(ls -d "$CAP_OUT_ROOT/sim-sweeps"/*/ 2>/dev/null | grep -v "/$(UTC_DATE)/" | sort | tail -1) +moved=0; movelog="$OUT/moved.txt"; : > "$movelog" +if [ -n "$prev" ] && [ -f "${prev}sweeps.csv" ]; then + moved=$($PY - "$CSV" "${prev}sweeps.csv" <<'PYEOF' 2>/dev/null || echo 0 +import sys,csv +def worst(p): + m={} + for r in csv.DictReader(open(p)): + try:v=float(r["value"]) + except:continue + k=(r["sim"],r["metric"]); m[k]=max(m.get(k,v),v) + return m +a=worst(sys.argv[1]); b=worst(sys.argv[2]); n=0 +for k in a: + if k in b and abs(a[k]-b[k])>1e-9: + print("%s %s: %.3f -> %.3f"%(k[0],k[1],b[k],a[k]),file=sys.stderr); n+=1 +print(n) +PYEOF +) + ls -d "$CAP_OUT_ROOT/sim-sweeps"/*/ >/dev/null 2>&1 && grep -h . "$movelog" 2>/dev/null || true +fi +rows=$(( $(wc -l < "$CSV") - rows0 )) +sum="swept $did seeds this slice (next $seed/1000), $rows CSV rows added; bounds moved vs yesterday: $moved" +printf '{"state":"ran","summary":%s,"counters":{"seeds_swept":%s,"rows":%s,"moved":%s},"next_seed":%s,"out":%s}' \ + "$(cap_json_str "$sum")" "$did" "$rows" "${moved:-0}" "$seed" "$(cap_json_str "$OUT")" | cap_summary sim-sweeps +cap_say "$sum" diff --git a/infra/build-server/capacity/jobs/sync-fuzz.sh b/infra/build-server/capacity/jobs/sync-fuzz.sh new file mode 100755 index 000000000..199e6e7c5 --- /dev/null +++ b/infra/build-server/capacity/jobs/sync-fuzz.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Capacity job 2: the sync-request fuzz against a pruned node (the Horizon security lane's gate for the 28 sync-manager +# unwrap sites, docs/analysis/horizon/consensus-security.md section 5). Starts a throwaway pruned igneumd on a simnet +# datadir on the box (the 0.3.15 node line; NEVER the live devnet), on loopback ports far from every other network, and +# drives igneum-p2p-probe sync-fuzz at it: random, boundary and below-retention locator, header, antipast, IBD and +# pruning-point requests. The node is the subject; a gRPC alive check every N requests catches any panic, which is saved +# with the request. Builds igneumd and igneum-p2p-probe from the capacity fork checkout. +# jobs/sync-fuzz.sh --dry-run plan only; also runs the probe's own --dry-run (builds one request of each kind) +# jobs/sync-fuzz.sh --smoke build, start a node, 10 minutes of requests, stop the node +# jobs/sync-fuzz.sh --slice-s 1800 fuzz for about this long (the controller's slice) +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; export CAP_JOB=sync-fuzz; export CAP_PRIORITY=2 +. "$HERE/../lib.sh" + +DRY=0; SMOKE=0; SLICE_S="${CAP_SLICE_S:-1800}" +while [ $# -gt 0 ]; do case "$1" in --dry-run) DRY=1;; --smoke) SMOKE=1; SLICE_S=360;; --slice-s) SLICE_S="$2"; shift;; *) cap_say "unknown arg $1";; esac; shift; done + +BASE_PORT="${CAP_SYNC_PORT:-31500}" # grpc BASE+0, p2p BASE+1, json BASE+2: far from live (26610/40), harness (27xxx), fast-time (295xx) +SUFFIX="${CAP_SYNC_SUFFIX:-315}" +DATADIR="$CAP_ROOT/sync-node" +NET="igneum-devnet-$SUFFIX" + +if [ "$DRY" = 1 ]; then + cap_say "DRY RUN: would start a pruned igneumd (net $NET, grpc 127.0.0.1:$BASE_PORT, p2p 127.0.0.1:$((BASE_PORT+1)), data $DATADIR) and fuzz it" + PROBE="$CAP_FORK/target-capacity/release/igneum-p2p-probe" + if [ -x "$PROBE" ]; then "$PROBE" sync-fuzz 127.0.0.1:$((BASE_PORT+1)) grpc://127.0.0.1:$BASE_PORT --dry-run --seed 1 || true; fi + printf '{"state":"idle","summary":%s,"counters":{"requests":0,"panics":0},"dry_run":true}' "$(cap_json_str "dry run: net $NET, loopback only, live devnet untouched")" | cap_summary sync-fuzz + exit 0 +fi + +[ -d "$CAP_FORK/.git" ] || cap_sync_checkout >/dev/null 2>&1 || { cap_say "no checkout"; exit 1; } +OUT=$(cap_out_dir sync-fuzz) +export CARGO_TARGET_DIR="$CAP_FORK/target-capacity" +IGNEUMD="$CARGO_TARGET_DIR/release/igneumd" +PROBE="$CARGO_TARGET_DIR/release/igneum-p2p-probe" +OVERRIDE="$CAP_SRC/infra/fast-time/override-60x.json" + +cap_say "building igneumd and igneum-p2p-probe (fork $CAP_NODE_BRANCH)" +if ! cap_cargo "$CAP_FORK" build --release -p kaspad -p igneum-p2p-probe --features kaspad/igneum-pow > "$CAP_LOG/sync-build.log" 2>&1; then + cap_say "build FAILED (see $CAP_LOG/sync-build.log)" + printf '{"state":"error","summary":%s,"counters":{"panics":0}}' "$(cap_json_str "build failed: $(grep -m1 '^error' "$CAP_LOG/sync-build.log" | cut -c1-120)")" | cap_summary sync-fuzz + exit 1 +fi +[ -x "$IGNEUMD" ] || IGNEUMD="$CARGO_TARGET_DIR/release/igneumd" +# a merged override with a tiny retention so the node prunes (pruned-node semantics) +MERGED="$CAP_ROOT/sync-override.json" +"$NODE_BIN" -e 'const fs=require("fs");const o=JSON.parse(fs.readFileSync(process.argv[1]));o.skip_proof_of_work=true;fs.writeFileSync(process.argv[2],JSON.stringify(o))' "$OVERRIDE" "$MERGED" 2>/dev/null || cp "$OVERRIDE" "$MERGED" + +rm -rf "$DATADIR"; mkdir -p "$DATADIR" +cap_say "starting pruned igneumd ($NET) on loopback" +nice -n 19 ionice -c3 "$IGNEUMD" --devnet --devnet-suffix="$SUFFIX" --nodnsseed --disable-upnp --nologfiles \ + --enable-unsynced-mining --utxoindex --unsaferpc --archival=false --retention-period-days=0.02 \ + --appdir="$DATADIR" --rpclisten=127.0.0.1:$BASE_PORT --rpclisten-json=127.0.0.1:$((BASE_PORT+2)) \ + --listen=127.0.0.1:$((BASE_PORT+1)) --outpeers=0 --override-params-file="$MERGED" --loglevel=info --yes \ + > "$DATADIR/node.log" 2>&1 & +NODE_PID=$! +cleanup() { kill -INT "$NODE_PID" 2>/dev/null; for _ in $(seq 1 50); do kill -0 "$NODE_PID" 2>/dev/null || break; sleep 0.1; done; kill -KILL "$NODE_PID" 2>/dev/null; } +trap cleanup EXIT +# wait for the node's gRPC port to accept a connection (bash /dev/tcp; no log-line guessing) +up=0; for _ in $(seq 1 90); do + kill -0 "$NODE_PID" 2>/dev/null || break + if (exec 3<>"/dev/tcp/127.0.0.1/$BASE_PORT") 2>/dev/null; then exec 3>&- 3<&-; up=1; break; fi + sleep 1 +done +sleep 2 +if [ "$up" != 1 ]; then + cap_say "node did not start (see $DATADIR/node.log)" + printf '{"state":"error","summary":%s,"counters":{"panics":0}}' "$(cap_json_str "node failed to start: $(tail -2 "$DATADIR/node.log" | tr '\n' ' ' | cut -c1-120)")" | cap_summary sync-fuzz + exit 1 +fi + +trace="$OUT/trace-$(date -u +%H%M%S).jsonl" +cap_say "fuzzing for ${SLICE_S}s (net $NET, trace $trace)" +"$PROBE" sync-fuzz 127.0.0.1:$((BASE_PORT+1)) grpc://127.0.0.1:$BASE_PORT --seconds "$SLICE_S" --network "$NET" \ + --seed "$(( $(date +%s) % 100000 ))" --report-every 200 --alive-every 25 | tee "$trace" | tail -1 > "$OUT/.last" || true +last=$(cat "$OUT/.last" 2>/dev/null) +reqs=$(echo "$last" | "$NODE_BIN" -e 'try{const o=JSON.parse(require("fs").readFileSync(0,"utf8"));process.stdout.write(String(o.requests||0))}catch{process.stdout.write("0")}' 2>/dev/null || echo 0) +alive=$(echo "$last" | grep -c '"node_alive":true' || true) +down=$(grep -c '"sync-fuzz":"node-down"' "$trace" 2>/dev/null || echo 0) + +panics=$(cap_cursor_get sync-fuzz-panics 0) +if [ "$down" != 0 ] || { [ "$alive" = 0 ] && echo "$last" | grep -q 'node-down'; }; then + panics=$((panics + 1)); cap_cursor_set sync-fuzz-panics "$panics" + save="$OUT/node-down-$(date -u +%H%M%S).jsonl"; grep -A2 -B2 'node-down' "$trace" > "$save" 2>/dev/null; cp "$DATADIR/node.log" "$OUT/node-$(date -u +%H%M%S).log" + cap_say "NODE WENT DOWN under fuzz: saved $save and the node log" +fi +total=$(cap_cursor_get sync-fuzz-requests 0); total=$((total + reqs)); cap_cursor_set sync-fuzz-requests "$total" +sum="this slice $reqs requests, total $total; node alive $([ "$down" = 0 ] && echo yes || echo NO); $panics panic(s); net $NET (loopback, live devnet untouched)" +printf '{"state":"ran","summary":%s,"counters":{"requests":%s,"slice_requests":%s,"panics":%s},"out":%s,"node_alive":%s}' \ + "$(cap_json_str "$sum")" "$total" "$reqs" "$panics" "$(cap_json_str "$OUT")" "$([ "$down" = 0 ] && echo true || echo false)" | cap_summary sync-fuzz +cap_say "$sum" +[ "$down" = 0 ] diff --git a/infra/build-server/capacity/lib.sh b/infra/build-server/capacity/lib.sh new file mode 100755 index 000000000..e7990eddd --- /dev/null +++ b/infra/build-server/capacity/lib.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# Shared helpers for the capacity layer on igneum-build-1 (infra/build-server/capacity). Sourced by run.sh and every +# job in jobs/. Nothing here takes a build slot (the slots are /srv/builds/_locks/build-, owned by remote-run.sh) +# and nothing writes under /srv/builds/. All work lives under /srv/capacity. +# +# Paths (overridable by environment for the smoke test and the dry run): +# CAP_ROOT /srv/capacity the layer's own tree +# CAP_SRC $CAP_ROOT/src a clone of /srv/igneum.git (master), the repo the jobs build and run from +# CAP_FORK $CAP_SRC/vendor/igneum-node a clone of /srv/igneum-node.git, the node fork +# CAP_OUT $CAP_ROOT/out// results, one directory per job per UTC day +# CAP_STATE $CAP_ROOT/state per-job cursors (the continuous jobs advance their seed base here) +# CAP_LOG $CAP_ROOT/log per-job slice logs +# LOCKS /srv/builds/_locks read only: the build slots and the measure hold the layer yields to +# CAP_JSON /srv/workers/capacity.json the one-line-per-job summary the dashboard reads +set -uo pipefail + +CAP_ROOT="${CAP_ROOT:-/srv/capacity}" +CAP_SRC="${CAP_SRC:-$CAP_ROOT/src}" +CAP_FORK="${CAP_FORK:-$CAP_SRC/vendor/igneum-node}" +CAP_OUT_ROOT="${CAP_OUT_ROOT:-$CAP_ROOT/out}" +CAP_STATE="${CAP_STATE:-$CAP_ROOT/state}" +CAP_LOG="${CAP_LOG:-$CAP_ROOT/log}" +LOCKS="${IGNEUM_BUILD_SLOTS_DIR:-/srv/builds/_locks}" +BUILDS_ROOT="${IGNEUM_BUILD_ROOT:-/srv/builds}" +export IGNEUM_CAPACITY_JSON="${IGNEUM_CAPACITY_JSON:-/srv/workers/capacity.json}" +REPO_MIRROR="${CAP_REPO_MIRROR:-/srv/igneum.git}" +NODE_MIRROR="${CAP_NODE_MIRROR:-/srv/igneum-node.git}" +NODE_BRANCH_DEFAULT="capacity-probe" # the sync-fuzz branch; falls back to the newest release-*-node on the mirror +REPO_BRANCH="${CAP_REPO_BRANCH:-master}" # the repo branch the layer builds and runs from; master in production, box-capacity until it merges (install.sh writes a drop-in) +HERE_LIB="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SUMMARY="$HERE_LIB/summary.mjs" +NODE_BIN="${NODE_BIN:-/usr/local/bin/node}" +UTC_DATE() { date -u +%Y-%m-%d; } +cap_say() { printf '%s capacity/%s: %s\n' "$(date -u +%H:%M:%S)" "${CAP_JOB:-run}" "$*" >&2; } + +# ---- the build-slot and measure hold the layer yields to -------------------------------------------------------------- +# a slot or the measure file is "held" when flock -n cannot take it (another process holds the fd). The same probe the +# collector uses (tools/workers/collect.mjs flockHeld). Returns 0 (true) when ANY build slot or the measure hold is taken. +cap_build_active() { + local slots k f + slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1 + if [ -e "$LOCKS/measure" ] && ! flock -n "$LOCKS/measure" true 2>/dev/null; then return 0; fi + for k in $(seq 0 $((slots - 1))); do + f="$LOCKS/build-$k" + [ -e "$f" ] || continue + if ! flock -n "$f" true 2>/dev/null; then return 0; fi + done + return 1 +} +cap_hold_reason() { + local slots k + if [ -e "$LOCKS/measure" ] && ! flock -n "$LOCKS/measure" true 2>/dev/null; then echo "measure hold"; return; fi + slots=$(cat "$LOCKS/slots" 2>/dev/null || echo 1); [ "$slots" -ge 1 ] 2>/dev/null || slots=1 + for k in $(seq 0 $((slots - 1))); do + [ -e "$LOCKS/build-$k" ] || continue + if ! flock -n "$LOCKS/build-$k" true 2>/dev/null; then echo "build slot build-$k held"; return; fi + done + echo "" +} + +# ---- the capacity checkout (never a /srv/builds worktree) ------------------------------------------------------------- +# clone-or-fetch the repo at master and the fork at its branch into $CAP_SRC. Idempotent. The jobs build into target +# directories under this tree, never /srv/builds. +cap_sync_checkout() { + local node_branch="${1:-$NODE_BRANCH_DEFAULT}" + mkdir -p "$CAP_ROOT" "$CAP_STATE" "$CAP_LOG" + if [ ! -d "$CAP_SRC/.git" ]; then git clone -q "$REPO_MIRROR" "$CAP_SRC" || { cap_say "repo clone failed"; return 1; }; fi + git -C "$CAP_SRC" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' || cap_say "repo fetch failed (using the last checkout)" + git -C "$CAP_SRC" checkout -q -- . 2>/dev/null || true + git -C "$CAP_SRC" clean -qfd -e target -e 'target-*' -e vendor -e out 2>/dev/null || true + local rb="$REPO_BRANCH" + git -C "$CAP_SRC" rev-parse -q --verify "origin/$rb" >/dev/null 2>&1 || rb=master + git -C "$CAP_SRC" checkout -q -B capacity-run "origin/$rb" 2>/dev/null || git -C "$CAP_SRC" reset -q --hard "origin/$rb" + CAP_REPO_BRANCH_USED="$rb"; export CAP_REPO_BRANCH_USED + mkdir -p "$CAP_SRC/vendor" + if [ ! -d "$CAP_FORK/.git" ]; then git clone -q --no-checkout "$NODE_MIRROR" "$CAP_FORK" || { cap_say "fork clone failed"; return 1; }; fi + git -C "$CAP_FORK" fetch -q origin '+refs/heads/*:refs/remotes/origin/*' || cap_say "fork fetch failed" + local b="$node_branch" + git -C "$CAP_FORK" rev-parse -q --verify "origin/$b" >/dev/null 2>&1 || b=$(git -C "$CAP_FORK" branch -r --list 'origin/release-*-node' | sed 's|.*/||' | sort -V | tail -1) + [ -n "$b" ] || b=master + git -C "$CAP_FORK" checkout -q -- . 2>/dev/null || true + git -C "$CAP_FORK" clean -qfd -e target -e 'target-*' 2>/dev/null || true + git -C "$CAP_FORK" checkout -q -B "$b" "origin/$b" 2>/dev/null || git -C "$CAP_FORK" reset -q --hard "origin/$b" + CAP_NODE_BRANCH="$b" + CAP_REPO_SHA=$(git -C "$CAP_SRC" rev-parse --short HEAD 2>/dev/null) + CAP_FORK_SHA=$(git -C "$CAP_FORK" rev-parse --short HEAD 2>/dev/null) + export CAP_NODE_BRANCH CAP_REPO_SHA CAP_FORK_SHA + cap_say "checkout: repo master $CAP_REPO_SHA, fork $CAP_NODE_BRANCH $CAP_FORK_SHA" +} + +# cargo under the layer's discipline: no build slot, idle IO, SCHED_IDLE, nice 19, a bounded job count (the controller's +# SIGSTOP pauses it the instant a real build takes a slot; this keeps it gentle even while it runs). +cap_cargo() { + ( cd "$1" && shift && CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS="${CAP_CARGO_JOBS:-16}" \ + nice -n 19 ionice -c3 chrt -i 0 "$HOME/.cargo/bin/cargo" "$@" ) +} + +cap_out_dir() { # : make and echo today's out dir + local d="$CAP_OUT_ROOT/$1/$(UTC_DATE)"; mkdir -p "$d"; echo "$d" +} + +# cap_summary : read a JSON fragment on stdin and merge it into capacity.json under that job +cap_summary() { CAP_PRIORITY="${CAP_PRIORITY:-}" "$NODE_BIN" "$SUMMARY" job "$1"; } +cap_controller_summary() { "$NODE_BIN" "$SUMMARY" controller; } + +# a cursor is a plain number per job (the continuous jobs' seed base); cap_cursor_get / cap_cursor_set +cap_cursor_get() { cat "$CAP_STATE/$1.cursor" 2>/dev/null || echo "${2:-0}"; } +cap_cursor_set() { mkdir -p "$CAP_STATE"; echo "$2" > "$CAP_STATE/$1.cursor"; } + +# json_escape a string for a one-line summary (python, always present) +cap_json_str() { "$NODE_BIN" -e 'process.stdout.write(JSON.stringify(process.argv[1]||""))' "$1"; } diff --git a/infra/build-server/capacity/run.sh b/infra/build-server/capacity/run.sh new file mode 100755 index 000000000..d522c59b6 --- /dev/null +++ b/infra/build-server/capacity/run.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# The capacity controller on igneum-build-1 (infra/build-server/capacity). Runs the job queue in priority order, one job +# at a time, each for a bounded slice; a 5 s poll of /srv/builds/_locks SIGSTOPs the running job the instant any build +# slot or the measure hold is taken and SIGCONTs it when they clear. The layer never takes a build slot and never writes +# under /srv/builds/. Started as user build by igneum-capacity.service (Nice 19, chrt -i 0 wrapper, CPUQuota +# leaving 8 threads free). Killed by the night battery's start and restarted after (the service's ExecStartPre/StopPost). +# +# run.sh the controller loop (systemd runs this) +# run.sh --once one pass through the weighted sequence, then exit (for a manual check) +# run.sh --dry-run call every job's --dry-run in priority order and exit +# +# The weighted sequence (CAP_SEQUENCE) gives the earlier, higher-priority jobs more turns. Default: +# pow-fuzz pow-fuzz pow-fuzz sync-fuzz sync-fuzz sim-sweeps model-sweeps clippy-audit +# Each turn runs one job for CAP_SLICE_S (default 1800 s) through run_slice, which backgrounds the job in its own +# process group and pauses/resumes it with the lock poll. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"; export CAP_JOB=run +. "$HERE/../capacity/lib.sh" 2>/dev/null || . "$HERE/lib.sh" + +SLICE_S="${CAP_SLICE_S:-1800}" +POLL_S="${CAP_POLL_S:-5}" +SEQUENCE="${CAP_SEQUENCE:-pow-fuzz pow-fuzz pow-fuzz sync-fuzz sync-fuzz sim-sweeps model-sweeps clippy-audit}" +JOBS_DIR="$HERE/jobs" + +"$NODE_BIN" "$SUMMARY" init 2>/dev/null || true + +if [ "${1:-}" = --dry-run ]; then + for j in pow-fuzz sync-fuzz sim-sweeps model-sweeps clippy-audit; do + cap_say "dry-run $j"; bash "$JOBS_DIR/$j.sh" --dry-run || cap_say "$j dry-run returned $?" + done + exit 0 +fi + +# run_slice : background the job in its own process group, poll the locks, STOP while a build or measure holds, +# CONT when clear, enforce the slice as a wall clock, reap at the end. The job writes its own summary; the controller +# updates the top-level state and current_job. +run_slice() { + local job="$1" log="$CAP_LOG/$job.slice.log"; mkdir -p "$CAP_LOG" + cap_say "slice start: $job (${SLICE_S}s)" + # own process group via setsid so STOP/CONT reach the whole cargo/python tree + setsid bash "$JOBS_DIR/$job.sh" --slice-s "$SLICE_S" > "$log" 2>&1 & + local pid=$! pgid; pgid=$(ps -o pgid= -p "$pid" 2>/dev/null | tr -d ' '); [ -n "$pgid" ] || pgid="$pid" + local paused=0 end=$(( $(date +%s) + SLICE_S + 120 )) # a 2 min grace over the slice for the job's own teardown + while kill -0 "$pid" 2>/dev/null; do + if cap_build_active; then + if [ "$paused" = 0 ]; then kill -STOP -"$pgid" 2>/dev/null && paused=1; local why; why=$(cap_hold_reason); cap_say "pause $job ($why)"; printf '{"state":"paused","current_job":%s,"paused_reason":%s,"slice_s":%s}' "$(cap_json_str "$job")" "$(cap_json_str "$why")" "$SLICE_S" | cap_controller_summary; fi + else + if [ "$paused" = 1 ]; then kill -CONT -"$pgid" 2>/dev/null; paused=0; cap_say "resume $job"; printf '{"state":"running","current_job":%s,"paused_reason":null,"slice_s":%s}' "$(cap_json_str "$job")" "$SLICE_S" | cap_controller_summary; fi + fi + # a safety stop so a wedged job cannot hold the turn forever; a paused job's clock does not advance it past end+grace + if [ "$paused" = 0 ] && [ "$(date +%s)" -ge "$end" ]; then cap_say "slice over time, stopping $job"; kill -INT -"$pgid" 2>/dev/null; sleep 3; kill -KILL -"$pgid" 2>/dev/null; break; fi + sleep "$POLL_S" + done + [ "$paused" = 1 ] && kill -CONT -"$pgid" 2>/dev/null + wait "$pid" 2>/dev/null; local rc=$? + cap_say "slice end: $job rc $rc" +} + +once() { + for job in $SEQUENCE; do + [ -f "$JOBS_DIR/$job.sh" ] || { cap_say "no job $job"; continue; } + # wait out a running build before starting a slice (do not even launch during a build) + while cap_build_active; do + printf '{"state":"waiting","current_job":null,"paused_reason":%s,"slice_s":%s}' "$(cap_json_str "$(cap_hold_reason)")" "$SLICE_S" | cap_controller_summary + sleep "$POLL_S" + done + printf '{"state":"running","current_job":%s,"paused_reason":null,"slice_s":%s,"host":%s}' "$(cap_json_str "$job")" "$SLICE_S" "$(cap_json_str "$(hostname)")" | cap_controller_summary + run_slice "$job" + done +} + +cap_sync_checkout >/dev/null 2>&1 || cap_say "initial checkout had trouble (jobs will retry)" +if [ "${1:-}" = --once ]; then once; printf '{"state":"idle","current_job":null,"paused_reason":null}' | cap_controller_summary; exit 0; fi + +trap 'cap_say "controller stopping"; printf "{\"state\":\"stopped\",\"current_job\":null}" | cap_controller_summary; exit 0' INT TERM +cap_say "controller start (sequence: $SEQUENCE; slice ${SLICE_S}s; poll ${POLL_S}s)" +cycle=0 +while true; do + cycle=$((cycle + 1)) + # refresh the checkout at the top of each cycle (cheap when nothing moved); yields if a build is active + while cap_build_active; do sleep "$POLL_S"; done + cap_sync_checkout >/dev/null 2>&1 || true + once +done diff --git a/infra/build-server/capacity/summary.mjs b/infra/build-server/capacity/summary.mjs new file mode 100755 index 000000000..27b991a90 --- /dev/null +++ b/infra/build-server/capacity/summary.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// The capacity layer's summary writer. Atomically merges one JSON fragment into /srv/workers/capacity.json, the file +// the worker dashboard's collector (tools/workers/collect.mjs) reads for the "Background" lane. Never takes a build +// slot, never writes anywhere but IGNEUM_CAPACITY_JSON. Node 22, no deps. +// +// echo '{"state":"running","summary":"...","counters":{...}}' | summary.mjs job pow-fuzz +// merge the fragment into doc.jobs["pow-fuzz"], stamping updated_at; set doc.jobs[job].priority from CAP_PRIORITY if given +// echo '{"state":"running","current_job":"pow-fuzz","paused_reason":null}' | summary.mjs controller +// merge the fragment into the top-level controller fields (state, current_job, paused_reason, slice, host) +// summary.mjs init write an empty document if none exists +// +// The document: +// { v, generated_at, host, state, current_job, paused_reason, slice_s, +// jobs: { : { priority, state, updated_at, summary, counters, panics, saved, ... } } } +import { readFileSync, writeFileSync, renameSync, mkdirSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { hostname } from 'node:os'; + +const OUT = process.env.IGNEUM_CAPACITY_JSON || '/srv/workers/capacity.json'; +const iso = () => new Date().toISOString().replace(/\.\d{3}Z$/, 'Z'); + +function load() { + try { + const d = JSON.parse(readFileSync(OUT, 'utf8')); + if (d && typeof d === 'object') { d.jobs = d.jobs || {}; return d; } + } catch { /* fall through to a fresh document */ } + return { v: 1, host: hostname(), state: 'starting', current_job: null, paused_reason: null, slice_s: null, jobs: {} }; +} + +function save(d) { + d.v = 1; d.host = d.host || hostname(); d.generated_at = iso(); + mkdirSync(dirname(OUT), { recursive: true }); + const tmp = OUT + '.tmp'; + writeFileSync(tmp, JSON.stringify(d)); + renameSync(tmp, OUT); +} + +function readStdin() { + try { + const t = readFileSync(0, 'utf8').trim(); + if (!t) return {}; + const o = JSON.parse(t); + return o && typeof o === 'object' ? o : {}; + } catch (e) { process.stderr.write('summary.mjs: bad JSON on stdin: ' + (e.message || e) + '\n'); process.exit(1); } +} + +const mode = process.argv[2]; +const d = load(); + +if (mode === 'init') { save(d); process.exit(0); } + +if (mode === 'controller') { + const frag = readStdin(); + for (const k of ['state', 'current_job', 'paused_reason', 'slice_s', 'host']) if (k in frag) d[k] = frag[k]; + save(d); + process.exit(0); +} + +if (mode === 'job') { + const job = process.argv[3]; + if (!job) { process.stderr.write('summary.mjs job \n'); process.exit(1); } + const frag = readStdin(); + const prev = d.jobs[job] || {}; + const merged = { ...prev, ...frag, updated_at: iso() }; + if (process.env.CAP_PRIORITY) merged.priority = Number(process.env.CAP_PRIORITY); + d.jobs[job] = merged; + save(d); + process.exit(0); +} + +process.stderr.write('summary.mjs: mode is one of init | controller | job \n'); +process.exit(1); diff --git a/infra/build-server/night/igneum-night-battery.service b/infra/build-server/night/igneum-night-battery.service index 0ff06cb56..7085a7356 100644 --- a/infra/build-server/night/igneum-night-battery.service +++ b/infra/build-server/night/igneum-night-battery.service @@ -25,6 +25,11 @@ Environment=BR_AGENT=night Environment="BR_COMMAND=night-battery.sh (suites, fuzz, sims, harness, clippy, audit)" Environment=BR_TARGET=x86_64-unknown-linux-gnu Environment=IGNEUM_AGENT=night +# the background capacity layer is killed while the battery runs and started again after (capacity rule, section 8 of +# docs/plans/build-server.md); the + prefix runs these as root regardless of User=build. A failed stop/start never +# fails the battery (-). +ExecStartPre=+-/usr/bin/systemctl stop igneum-capacity.service +ExecStopPost=+-/usr/bin/systemctl start igneum-capacity.service ExecStart=/bin/bash /srv/builds/_bin/remote-run.sh TimeoutStartSec=8h StandardOutput=append:/srv/builds/_log/night-battery.log diff --git a/sim/difficulty/attacks/attacks.py b/sim/difficulty/attacks/attacks.py index 2d8fa8ee0..8f0e0c668 100644 --- a/sim/difficulty/attacks/attacks.py +++ b/sim/difficulty/attacks/attacks.py @@ -584,13 +584,14 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--scenario", default="all") ap.add_argument("--seeds", type=int, default=3) + ap.add_argument("--seed-base", type=int, default=7, help="first seed (default 7, the lane's runs; the box's capacity sweep passes 1 to 1,000)") ap.add_argument("--rules", default="igneum,kaspa") ap.add_argument("--base", action="store_true", help="also run the base simulator's profiles through each rule") ap.add_argument("--jobs", type=int, default=4) ap.add_argument("--hop-hours", type=int, default=24) ap.add_argument("--ts-rules", default="kaspa", help="timestamp rules the forger obeys: kaspa or tight (README.md)") args = ap.parse_args() - seeds = list(range(7, 7 + args.seeds)) + seeds = list(range(args.seed_base, args.seed_base + args.seeds)) rules = args.rules.split(",") want = args.scenario.split(",") if args.scenario != "all" else ["hop", "pulse", "ts", "osc", "epoch", "pollute"] jobs = [] diff --git a/sim/horizon/consensus-security/ghostdag_sim.py b/sim/horizon/consensus-security/ghostdag_sim.py index 66f27de06..a6bd81b15 100644 --- a/sim/horizon/consensus-security/ghostdag_sim.py +++ b/sim/horizon/consensus-security/ghostdag_sim.py @@ -355,6 +355,7 @@ def main(argv=None): ap.add_argument("--selfish-run", type=float, default=600.0, help="seconds of the selfish-lead strategy per episode") ap.add_argument("--out", default="") ap.add_argument("--json", default="") + ap.add_argument("--seed-base", type=int, default=0, help="offset added to every episode seed (the box's capacity sweep walks seeds 1 to 1,000 with --seeds 1)") args = ap.parse_args(argv) delays = [float(x) for x in args.delays.split(",")] shares = [float(x) for x in args.shares.split(",")] @@ -364,14 +365,14 @@ def main(argv=None): ("%g" % args.bps), args.k, args.honest, args.seeds), ""] out.append("Generated by `sim/horizon/consensus-security/ghostdag_sim.py` on %s. Every number is this simulator's; the model and its limits are in the file header." % time.strftime("%Y-%m-%d %H:%M UTC", time.gmtime())) out.append("") - results = {"bps": args.bps, "k": args.k, "cells": []} + results = {"bps": args.bps, "k": args.k, "seed_base": args.seed_base, "seeds": args.seeds, "cells": []} # 1. honest-only baseline per delay out.append("## 1. Honest parallelism alone: natural selected-chain reorg depth at honest miner 0 (chain blocks), %g s per episode" % (args.warm + args.post)) out.append("") rows = [] for d in delays: - eps = [episode(args.bps, args.k, d, 0.0, "hold", 0.0, 0, args.honest, args.warm + args.post, 0.0, 1000 + s) for s in range(args.seeds)] + eps = [episode(args.bps, args.k, d, 0.0, "hold", 0.0, 0, args.honest, args.warm + args.post, 0.0, 1000 + args.seed_base + s) for s in range(args.seeds)] p99 = max(e["natural_p99"] for e in eps) mx = max(e["natural_max"] for e in eps) rows.append(["%g" % d, "%.2f" % (args.bps * d), "%.0f" % p99, mx]) @@ -388,7 +389,7 @@ def main(argv=None): for d in delays: for H in shares: for T in holds: - eps = [episode(args.bps, args.k, d, H, "hold", T, 0, args.honest, args.warm, args.post, 2000 + s) for s in range(args.seeds)] + eps = [episode(args.bps, args.k, d, H, "hold", T, 0, args.honest, args.warm, args.post, 2000 + args.seed_base + s) for s in range(args.seeds)] won = np.mean([e["won"] for e in eps]) reorg = [e["reorg"] for e in eps] age = [e["fork_age"] for e in eps if e["won"]] @@ -412,7 +413,7 @@ def main(argv=None): d = delays[1] if len(delays) > 1 else delays[0] rows = [] for H in shares: - eps = [episode(args.bps, args.k, d, H, "selfish", 0, 6, args.honest, 60.0, args.selfish_run, 3000 + s) for s in range(max(3, args.seeds // 4))] + eps = [episode(args.bps, args.k, d, H, "selfish", 0, 6, args.honest, 60.0, args.selfish_run, 3000 + args.seed_base + s) for s in range(max(3, args.seeds // 4))] ab = sum(e["att_blue"] for e in eps) at = sum(e["att_blocks"] for e in eps) hb = sum(e["hon_blue"] for e in eps) diff --git a/tools/workers/collect.mjs b/tools/workers/collect.mjs index 03cede18d..42131a8c8 100644 --- a/tools/workers/collect.mjs +++ b/tools/workers/collect.mjs @@ -126,6 +126,14 @@ export function collect() { const recent = parseBuildsJsonl(logText, { limit: 200 }); const mac = source('mac', now), pcs = source('pcs', now); let headline = null; try { headline = JSON.parse(readFileSync(join(OUT_DIR, 'headline.json'), 'utf8')); } catch { headline = null; } + // the background capacity layer (infra/build-server/capacity) writes capacity.json beside this file; show it when + // present and under 10 min old (the layer's controller stamps it every slice and on every pause/resume) + let background = null; + try { + const p = join(OUT_DIR, 'capacity.json'); const st = statSync(p); const age = Math.round((now - st.mtimeMs) / 1000); + const j = JSON.parse(readFileSync(p, 'utf8')); + background = { ...j, meta: { at: iso(st.mtimeMs), age_s: age, stale: age > 600, ok: true } }; + } catch (e) { background = { meta: { ok: false, error: e.code === 'ENOENT' ? 'the capacity layer has not written yet' : String(e.message || e) } }; } const doc = { v: 1, generated_at: iso(now), @@ -143,7 +151,8 @@ export function collect() { }, mac: mac.data, pcs: pcs.data, headline, baselines: BASELINES, - sources: { box: { ok: true, at: iso(now) }, mac: mac.meta, pcs: pcs.meta }, + background, + sources: { box: { ok: true, at: iso(now) }, mac: mac.meta, pcs: pcs.meta, background: background && background.meta }, }; return doc; } diff --git a/tools/workers/page/workers.html b/tools/workers/page/workers.html index 8fc00d916..9a8709567 100644 --- a/tools/workers/page/workers.html +++ b/tools/workers/page/workers.html @@ -168,6 +168,9 @@

Queue

+

Background idle cores, yields to builds

+
+

Recently done click a row for the closing lines

@@ -261,6 +264,32 @@ function renderNow() { } function tick() { for (const el of document.querySelectorAll('.job.running[data-start]')) { const s = el.dataset.start; if (!s) continue; const e = el.querySelector('.elapsed'); if (e) e.textContent = fmtDurShort((Date.now() - Date.parse(s)) / 1000); } } +const BG_PRIORITY = { 'pow-fuzz': 1, 'sync-fuzz': 2, 'sim-sweeps': 3, 'model-sweeps': 4, 'clippy-audit': 5 }; +const BG_LABEL = { 'pow-fuzz': 'igneum-pow fuzz', 'sync-fuzz': 'sync-request fuzz', 'sim-sweeps': 'GHOSTDAG + finality sweeps', 'model-sweeps': 'N-ladder + chip model', 'clippy-audit': 'clippy + audit per branch' }; +function renderBackground() { + const bg = D.background; + const el = $('background'); const sub = $('bg-sub'); + if (!bg || (bg.meta && bg.meta.ok === false)) { + sub.textContent = 'idle cores, yields to builds'; + el.innerHTML = `
${esc((bg && bg.meta && bg.meta.error) || 'The capacity layer (infra/build-server/capacity) has not written capacity.json yet. It starts with igneum-capacity.service and writes within one slice.')}
`; + return; + } + const stale = bg.meta && bg.meta.stale; + const ctl = bg.state || 'unknown'; + sub.innerHTML = `${esc(bg.host || 'igneum-build-1')} · controller ${esc(ctl)}${bg.paused_reason ? ' · ' + esc(bg.paused_reason) : ''}${bg.current_job ? ' · on ' + esc(bg.current_job) : ''}${stale ? ' · stale' : ''}`; + const jobs = bg.jobs || {}; + const names = Object.keys(jobs).sort((a, b) => (jobs[a].priority || BG_PRIORITY[a] || 9) - (jobs[b].priority || BG_PRIORITY[b] || 9)); + if (!names.length) { el.innerHTML = `
No jobs reported yet.
`; return; } + el.innerHTML = names.map(n => { + const j = jobs[n]; const pr = j.priority || BG_PRIORITY[n] || ''; + const running = j.state === 'running' || j.state === 'ran'; + const panics = j.counters && j.counters.panics; + const kindCls = panics ? 'error' : running ? 'running' : ''; + const c = j.counters || {}; + const nums = Object.entries(c).filter(([k]) => k !== 'panics').map(([k, v]) => `${esc(k)} ${esc(String(v))}`).join(' · '); + return `
${esc(BG_LABEL[n] || n)}
priority ${esc(String(pr))} · ${esc(n)}${n === D.background.current_job ? ' · now' : ''}
${pill(j.state || 'idle', kindCls)}
${esc(j.summary || '')}
${nums ? `
${nums}
` : ''}
${panics ? `${esc(String(panics))} saved` : '0 panics'}${j.out ? ' · ' + esc(String(j.out).replace(/^.*\/out\//, 'out/')) : ''}${esc(ago(j.updated_at))}
`; + }).join(''); +} function renderQueue() { const items = []; for (const w of (D.box && D.box.queue) || []) items.push({ where: D.box.name, title: w.label ? kindLabel(w.kind) : 'a build', sub: w.label || 'label unknown: the waiter writes none until its slot is taken', since: w.since, wait: w.waiting_s }); @@ -345,7 +374,7 @@ function apply(d) { const live = $('live'); live.className = 'live' + (age > 900 ? ' down' : age > 420 ? ' stale' : ''); const u = ukTime(d.generated_at); $('stamp').innerHTML = `${age > 420 ? 'stale: ' : ''}written ${t(d.generated_at)} (${esc(ago(d.generated_at))})${d._feed ? ' · ' + esc(d._feed) : ''}`; - renderServer(d.box, d.sources && d.sources.box); renderCrew(); renderNow(); renderQueue(); renderDone(); renderHeadline(); renderAnalytics(); renderSources(); + renderServer(d.box, d.sources && d.sources.box); renderCrew(); renderNow(); renderQueue(); renderBackground(); renderDone(); renderHeadline(); renderAnalytics(); renderSources(); } const LIVE_URL = 'https://build.igneum.network/workers.json'; // the box itself, 30 s fresh (Caddy, read-only); the folder copy is up to 5 min behind let skipLive = 0, feed = '';