diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90af33b10..2f0805968 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,10 +1,19 @@ # CI on every push and pull request (private repository, free runner minutes). # # What runs: the lottery-hash crate's tests (igneum-pow, release profile), the census tool's build, the two Python -# simulators' --quick modes (each under two minutes), the site build with an internal link check, the gh-free -# identity grep of the public export list (tools/ci/forbidden-strings.txt), and the no-secrets check of the tree -# (tools/ci/no-secrets-check.sh: no file named like a key of ~/.config/igneum, no 64-hex value assigned to a -# token/key/secret name outside tests and the allowlist; docs/security/keys.md). +# simulators' --quick modes (each under two minutes), and the tree gate: every fast check in ONE script, +# tools/ci/pre-push.sh (site build and link check, the ledger sentences, the identity grep of the public export list +# and the served site, Windows-valid paths, workflow shell syntax, the copied-sources and playbook classes, the unit +# tests, the no-secrets check). The pre-push hook runs the SAME script before a push to master or release-*, so the +# local gate and CI cannot drift (6 October 2026: 131 red `ci` runs in three days, 92 of them on master, every one a +# tree check that would have failed on the pushing machine in under 25 s; docs/analysis/ci-failures-2026-10-06.md). +# +# Where it runs: `pow` and `sims` go to the box's runner (igneum-build-1, rustc pinned, sccache read-only, 48 jobs) +# when the repository variable IGNEUM_CI_RUNNER is `box`, else to ubuntu-latest (docs/plans/ci-self-hosted.md; GitHub +# has no fallback in runs-on, the variable is the switch). The `site` job stays on GitHub's machines. The `red` job +# runs on the box after any failed master or release-* run and records the failure for the watcher +# (tools/ci/red-watch.mjs; infra/build-server/ci-red): one line per run to the hidden updates channel and to +# /srv/ci-red/red.jsonl, so nobody opens the Actions page to learn master is red. # # What does not run, on purpose: the node fork (vendor/igneum-node*, a rusty-kaspa fork of about 500 crates with # rocksdb, blst and the execution layer) is gitignored here and too big for the free runners today (a cold build is @@ -17,7 +26,7 @@ on: jobs: pow: name: igneum-pow tests, igneum-census build - runs-on: ubuntu-latest + runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 - name: toolchain @@ -32,13 +41,15 @@ jobs: run: cargo build --release sims: name: simulators, quick modes - runs-on: ubuntu-latest + runs-on: ${{ vars.IGNEUM_CI_RUNNER == 'box' && fromJSON('["self-hosted", "linux", "x64", "igneum-build-1"]') || 'ubuntu-latest' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 + if: vars.IGNEUM_CI_RUNNER != 'box' # the box has python3 and numpy from provision.sh with: python-version: '3.12' - run: python3 -m pip install --quiet numpy + if: vars.IGNEUM_CI_RUNNER != 'box' - name: finality_v2.py --quick (under two minutes) working-directory: sim run: time timeout 120 python3 finality_v2.py --quick > finality_quick.md @@ -59,52 +70,32 @@ jobs: - uses: actions/setup-node@v4 with: node-version: '22' - - name: site build - run: node site/build.mjs - - name: internal link check of site/*.html - run: node tools/ci/link-check.mjs - - name: identity grep of the public export list - run: bash tools/ci/identity-check.sh - - name: no conflict markers in tracked files - run: bash tools/ci/no-conflict-markers.sh - - name: PowerShell drive-reference check (a `$name:` inside a double-quoted string is a 5.1 parse error) - run: bash tools/ci/ps-drive-ref-check.sh - - name: copied sources are re-stamped before a build - run: bash tools/ci/copied-sources-check.sh - - name: override params files parse with no duplicate key (the duplicate-field class, 6 October 2026) - run: bash tools/ci/override-json-check.sh - - name: second-engine playbooks log to a file and end their tree (C35) - run: bash tools/ci/second-engine-check.sh - - name: no playbook quits, pauses or resumes the installed app (self-test first, then the tree) - run: bash tools/ci/playbook-quit-check.sh --self-test && bash tools/ci/playbook-quit-check.sh - - name: the signer is never piped into head - run: bash tools/ci/signer-pipe-check.sh - - name: bash bodies in PowerShell job scripts pass bash -n, the lost-quote class (self-test first, then the tree) - run: bash tools/ci/bash-body-check.sh --self-test && bash tools/ci/bash-body-check.sh - - name: run jobs test their fetched kit before use, the wiped-jobs-folder class (self-test first, then the tree) - run: bash tools/ci/kit-path-check.sh --self-test && bash tools/ci/kit-path-check.sh - - name: every Windows spawn of the app runs with a hidden console (self-test first, then the tree) - run: node tools/ci/windows-spawn-check.mjs --self-test && node tools/ci/windows-spawn-check.mjs - - name: pinned guest programs match their manifest and are built only by pin-guests.sh - run: bash tools/ci/pinned-guests-check.sh - - name: root prover playbooks kill the GPU server and unlink its socket (the root-socket class, 5 October 2026) - run: bash tools/ci/prover-socket-check.sh - - name: commit-string gate self-test (the empty-commit class of 6 October 2026; the gate itself runs in build-remote.sh, cross-remote.sh and cross-build.sh on every node binary) - run: bash tools/ci/commit-string-check.sh --self-test - - name: build server remote checkout self-test (the stale-overlay class of 6 October 2026) - run: bash infra/build-server/remote-run.sh --self-test - - name: no secret file names and no 64-hex secrets in the tree (self-test first, then the tree) - run: bash tools/ci/no-secrets-check.sh --self-test && bash tools/ci/no-secrets-check.sh - - name: faucet unit tests (validation, the daily limits, the signed transaction; keccak, RLP and secp256k1 vectors) - run: node --test site/api/faucet.test.mjs - - name: explorer and public stats unit tests (search router, formatters, emission rule against the node's own test values, the documented API fields from a fixture) - run: node --test site/lib/explorer.test.mjs site/lib/emission.test.mjs site/api/public-stats.test.mjs + - name: the tree gate, tools/ci/pre-push.sh --ci (the same script the pre-push hook runs; one line per check, a red check prints its output) + run: bash tools/ci/pre-push.sh --ci - name: public stats API answers with the documented fields (the live site; master only, the endpoints exist there after the merge) if: github.ref == 'refs/heads/master' run: node tools/ci/public-api-check.mjs https://igneum.network - - name: ship tool self-test (version bump, the dl-both and public manifest helpers) - run: node tools/ship-app.mjs --self-test - - name: relay unit tests (parsers, secret compare, the wake endpoint) - run: node --test relay/test/parse.test.mjs relay/test/auth.test.mjs relay/test/wake.test.mjs relay/test/ember.test.mjs - - name: miner app notice strip and update card (ordering, keys, wording, timers, when the card shows) - run: node --test app/igneum-app/ui/notices.test.mjs app/igneum-app/ui/update-card.test.mjs app/igneum-app/ui/view.test.mjs app/igneum-app/ui/tune-line.test.mjs + + red: + # Runs only when a master or release-* run has a failed job, on the box's own runner (not a GitHub-hosted machine: + # the billing block of 6 October 2026, 18:37Z to 20:10Z, failed every hosted job at start and nobody was told). + # tools/ci/red-watch.mjs record appends ONE line for this run to /srv/ci-red/red.jsonl (idempotent per run attempt); + # the box's igneum-ci-red.timer posts each new line once to the hidden updates channel. Never blocks a release: + # it reads the run, writes one line, and ends. + name: red watcher (master and release-* only; one line per failed run to the updates channel and the box file) + needs: [pow, sims, site] + if: ${{ failure() && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) }} + runs-on: [self-hosted, linux, x64, igneum-build-1] + timeout-minutes: 5 + permissions: + actions: read # the run's jobs API (the first real red run, 21:19Z: the default token answered 403 and the line carried no step) + contents: read + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: tools/ci + - name: record this run (one line, the failed jobs and their first failed step, from the run's own API) + env: + GITHUB_TOKEN: ${{ github.token }} + RED_WATCH_TITLE: ${{ github.event.head_commit.message }} + run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 377b452e9..82aa2f60c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -293,3 +293,24 @@ jobs: path: build/inputs-artifact/ retention-days: 90 if-no-files-found: error + + red: + # The red watcher for the Windows pipeline (see ci.yml `red`): one line per failed master or release-* run to the + # hidden updates channel and /srv/ci-red/red.jsonl on the box, recorded by the box's own runner. + name: red watcher (master and release-* only; one line per failed run to the updates channel and the box file) + needs: [parse, build] + if: ${{ failure() && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-')) }} + runs-on: [self-hosted, linux, x64, igneum-build-1] + timeout-minutes: 5 + permissions: + actions: read # the run's jobs API (the first real red run, 21:19Z: the default token answered 403 and the line carried no step) + contents: read + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: tools/ci + - name: record this run (one line, the failed jobs and their first failed step, from the run's own API) + env: + GITHUB_TOKEN: ${{ github.token }} + RED_WATCH_TITLE: ${{ github.event.head_commit.message }} + run: node tools/ci/red-watch.mjs record --file /srv/ci-red/red.jsonl diff --git a/.gitignore b/.gitignore index e60bf5fb6..d5959dccb 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,8 @@ vendor/igneum-node-ship/ # Trademark instruction packs name the director and the applicant company; never in the repository brand/trademark/pbip-pack/ brand/trademark/*.zip + +# the Discord bot and reddit bot dry-run renders +tools/community/out/ +# the GPU workers built on igneum-build-1 (tools/workers-remote.sh); binaries, never committed +infra/cross/out-workers-box/ diff --git a/CLAUDE.md b/CLAUDE.md index 9a9e7a624..70e2feea3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,7 @@ launch, SP1 behind a swappable proving interface, Taiko as first proving custome ## The design in one paragraph Leader election by a random-program GPU hash (RandomX idea rebuilt for GPUs, new kernel each ~1h -epoch, era parameters drawn automatically every 6 months from chain state inside rules fixed at genesis, NO scheduled human releases; dataset grows on a genesis-fixed schedule and instruction families unlock by height from a genesis reserve (automatic anti-ASIC escalators); only writing new code is human, via 90% miner signalling, never required; warp-unit CPU-verifiable). Ordering by a GHOSTDAG +epoch, era parameters drawn automatically every 6 months from chain state inside rules fixed at genesis, NO scheduled human releases; dataset grows on a genesis-fixed schedule and instruction families unlock by height from a genesis reserve (automatic schedule changes against fixed datapaths and human forks; against a chip that stores the dataset every drawn parameter is firmware, and the defence is the latency-shadow work of class v4 and the price per joule, per the Horizon algorithm lane, 6 October 2026); only writing new code is human, via miner signalling (the three thresholds, one sentence everywhere: 60% for a parameter, 90% for an upgrade, 95% with a floor height for a class change, the P2 mechanism), never required; warp-unit CPU-verifiable). Ordering by a GHOSTDAG BlockDAG forked from rusty-kaspa. Execution by a zkEVM. Every block ZK-proven by miners in chunks, aggregated 20 to 60 s behind the tip at launch (target under 10 s as provers improve). Finality is miner-only and self-contained: FINALITY RULE V2 (review round 2, 3 Oct 2026): vote weight = blue blocks per BLS vote key (in header) over a flat 30-day DAA window, no damping (the 2x cap was Sybil-void), dust threshold 100 blocks; every voter signs every 30-s checkpoint (VRF picks 8 aggregators only); lock = 2/3 of ALL 30-day weight (DECIDED 4 Oct 2026, O-3.15: the floor was raised from 56.7% of total to 2/3 of total, which makes the 2/3-of-active test implied; finality pauses whenever under 2/3 of the window is connected and signing, and the node reports it; the old floor 0.85 x 2/3 had been added after sim v2 showed the bare active denominator locks both sides of a 50/50 partition after 60 min; with the floor: 0 conflicting locks in every partition and eclipse scenario); equivocation evidence strips weight 30 days; fork choice = GHOSTDAG among tips through all certified checkpoints under Kaspa merge-depth 3,600 s; NO hidden-block n^2 penalty (removed, breaks DAG determinism); epoch seed = 10-min class-group VDF of a certified checkpoint, era draw = 1-h VDF; dataset = 256 MB RandomX-style cache, 8 dependent reads per item (not closed-form, not 64 MB); fees: base fee burned in full on both gas dimensions, priority fee 80/20 (miners and provers / app, attributed per call frame, unregistered share burned), external jobs 90/10 (provers / burn), no dev fund and no fee to any team. Headline: 10 days of 100% hashrate to reach 1/3 of weight, 20 days for 2/3; 51% never reaches 2/3 while honest miners stay. NO stake and NO other chain anywhere in consensus (Bitcoin anchoring was @@ -69,8 +69,12 @@ Every Igneum task in Claude in Chrome runs in the Chrome profile "josh (igneum.n binaries). Every run writes one line to `/srv/builds/_log/builds.jsonl` (the worker dashboard reads it); `IGNEUM_AGENT=` tags it. The PCs keep only jobs that need their GPUs or the Windows runtime (measurements, Windows test suites, installer smoke runs: `node tools/build-job.mjs run --target ae432dc7|1ccfe586 ...`, PC 1 = ae432dc7, PC 2 = 1ccfe586). The Mac - keeps macOS binaries, the DMG and Metal tests, under the build lock. The box never hosts a live-devnet node and never - holds a secret; its Devnet 2 seed, when it starts, runs its own unit on 26611 (ufw open). Setup and re-provision: + keeps macOS binaries, the DMG and Metal tests, under the build lock. THE MAC RUNS NOTHING THE NETWORK DEPENDS ON (Josh, + 6 October 2026, evening): after the 0.3.15 cut the two devnet hands, node 1 and the observer (its node and + tools/observer), run on the box as systemd units (`igneum-node1`, `igneum-observer-node`, `igneum-observer`; + docs/plans/hands-on-build-1.md; `infra/build-server/hands/`), node 1's p2p on 26611 with `--externalip`, every RPC on + loopback, the observer's Neon string in `/srv/observer/env` (mode 600, copied from the Mac, never in the repo); that file + is the only secret the box holds. The Devnet 2 seed, when it starts, gets its own unit the same way. Setup and re-provision: `infra/build-server/run-from-mac.sh ` (idempotent); the rustc pin is `RUST_TOOLCHAIN` in `infra/build-server/provision.sh` (1.99.0; there is no rust-toolchain file, add one) and build-remote.sh refuses a version mismatch. Nobody deletes another worktree's target dir on the box. Windows exes are reproducible (`-Wl,--no-insert-timestamp` in cross-remote.sh, the Mac's @@ -108,6 +112,27 @@ Josh's ruling after the DAA 198,000 incident (a fixed-height activation crossed - No "clean day" waits (Josh, 6 October 2026, 19:3x UK: "we dont need a clean day for anything this is just delaying things"): a cut's only gate is the Devnet 2 crossing (the class v4 cut's gate is its P1 rehearsal on the fleet chain); digest moves are bundled into one cut (0.3.15 = class v4 + the fourteenth field), miners first, hands last. - A deep reorg never resets execution; a snapshot a node cannot read fails loudly; the p2p snapshot path refuses a snapshot below the node's tip or the restart; a hands script never pgreps its own command line. - Main checks an agent's number against the log or the chain before relaying it to Josh, or labels it unverified. +- Never remove more than 10 percent of the live devnet's 30-day vote weight in any hour (Horizon finality lane, 6 October 2026: the class v4 rehearsal took 13 fleet keys off the live chain at 18:27Z; with seven earlier leavers that was 42.7 percent of the voter table frozen at the last lock, and rule v3 holds the pause for a full window; under v3 a sudden departure of a third of weight is a 30-day pause on mainnet). Experiments that borrow live miners do it in slices with an hour between. The protocol fix is the signed LEAVE item (0.3.16). +- Every NODE cut's Devnet 2 gate includes a MINING new node beside an old node on the live file for ten minutes (the old node accepting the new node's blocks, the hub's reject count unchanged), and a new node restarted mid-window re-syncing from an old peer. Found 6 October 2026, 20:5x UK: 0.3.15's node stamped the class v4 signal bit into the block version (1026) on the thirteen-field file; every 0.3.14 node rejected it; the digest-compat test passed because the handshake peers while the block version splits; the canary caught it before any live box moved. + +- Standing fleet (Josh, 6 October 2026, 20:0x UK: "cant we keep rented cards up longer"): 16 live-devnet boxes and 6 Devnet 2 boxes are kept up permanently and re-rented on host death; only benchmark and wave boxes are one-shot; a standing box never leaves the live devnet for an experiment (the class v4 rehearsal took the 15 prover boxes and paused live finality at 18:42Z; experiments use wave boxes only). The Mac runs nothing the network depends on: node 1 and the observer move to igneum-build-1 as systemd units (docs/plans/hands-on-build-1.md). +- Secrets on igneum-build-1: none that sign releases, move funds or reach the hands. Exception recorded 6 October 2026: Discord webhook URLs at /srv/discord-hooks/env (mode 600, rotatable in one click) for tools/community/discord-hooks.mjs. + +## Shared scratchpad and lane ownership (standing rules, 7 October 2026, 08:2x UK) + +- A lane never removes a directory under the shared scratchpad; it removes only files it wrote inside one. Each lane keeps its scratch under its own prefix (`bs-` for the build-server lane, `r03xx-ship` for the shipper, and so on). At 07:23 and 07:56 UK the build-server lane removed `r0317` and `r0318` as superseded and the shipper's running 0.3.18 chain lost its log and staging copy mid-build. +- Every wait keys on its own run's marker, never on a generic line in a shared log (the ten-member pool window mis-fired at 07:44 UK on the 0.3.17 cases' end marker). A presence check never shares a shell with the command it checks (the fleet's installer answered "already running" to its own command line and left pool-1's node down seven minutes). +- A supervised node restart read back by a new lock line from the hub is not a weight removal under the 10 percent rule; a key that stops voting is. The fleet's table gate reads the folded certificate weight over the frozen table (mean of the last five checkpoints), not the lock-moment share. + +## Nothing heavy on the Mac (standing rule, 7 October 2026, 10:5x UK) +The Mac crashed and rebooted under agent load (about twenty lanes, local cargo tests and guest builds, several headless chromiums for captures and Playwright suites at once). Josh: "stop building on it unless its 100% necessary to build on a mac". Rule: the only Mac builds are the shipper's macOS binaries and the DMG, one at a time under the build lock. No cargo build or test on the Mac for any other lane; no guest builds, z3 or census runs; no Playwright suites; captures use one headless chromium at a time and never from several lanes at once. Every Linux and Windows build, every test suite and every benchmark runs on igneum-build-1 (`tools/build-remote.sh`) or as a PC job. The pre-push gate script stays (checks, not a build). Main keeps the number of concurrently running lanes near ten, Josh's waiting items first. + +## CI red is stop-the-line (standing rule, 6 October 2026, 22:0x UK) +- Whoever's merge turns master or a release-* branch red owns the fix inside 15 minutes or reverts the merge; the red watcher posts every failed run to the hidden updates channel and to /srv/ci-red/red.jsonl on the box (tools/ci/red-watch.mjs); the box's own red builds land in the same file with a class (remote-run.sh pre-flight, kept run logs) and the 09:00 UK digest counts them per class with each class's guard. +- The pre-push gate is the same script CI runs: `tools/ci/pre-push.sh` (installed by `tools/ci/install-hooks.sh`; `--hook` before a push to master or release-*, `--ci` in the workflow). A check is added there, never only in ci.yml. +- Research and operations documents live outside the public export list: name them in `tools/ci/export-exclude.txt` (read by the identity check and by the mirror's sync.sh). What stays in the list is read by the public and must pass the identity grep. +- A path Windows cannot hold (colon, trailing dot or space, reserved name, over 240 characters) never enters a commit: the pre-commit hook runs `tools/ci/windows-paths-check.sh --staged`. +- Record: docs/analysis/ci-failures-2026-10-06.md (168 non-green runs in three days classified; 126 on master; all but two classes were tree checks that the gate now runs locally first). ## A rule row closes only with its check (standing rule, 6 October 2026, 18:4x UK) @@ -115,4 +140,4 @@ Josh, after the pgrep self-match hit twice in one day (the shipper's hands scrip - A bug class found today gets its `tools/ci` check merged on master the same hour, or the rule row stays OPEN and main says so. A rule row without a check is not closed. - Box operations (ssh to a rented box, install a payload, start a node, wait for sync, read height, peers and exec tip, start a miner or prover) live in ONE shared, tested library (`tools/fleet/lib/`), exercised against a Devnet 2 box by a test; agents call it and never write their own copy in bash or PowerShell. - A watcher or collector verifies the chain-side fact (height, peers, exec tip, paid records), never a reported rate or a process name. -- `pgrep -f` / `ps | grep` with a literal pattern is banned in scripts; use the bracket form `[i]gneumd` or `pgrep -x` on the binary name (CI check: pgrep-self-match-check). +- `pgrep -f` / `pkill -f` / `ps | grep` with a literal pattern is banned in scripts; use the bracket form `[i]gneumd`, `-x` on the binary name, or a pid file (`pkill -F`, `tools/fleet/fleet-bg.sh`). Kill by exact command line or pid file, never by a name: at 22:09 UK on 6 October a Mac-side `pkill -f ` matched nothing (a redirect is not on the command line) and the roll-everything script wiped a box it had been told to hold (CI check and gate: `tools/ci/kill-by-name-check.sh`, which also flags a file-name shape under pgrep/pkill). diff --git a/app/igneum-app/Cargo.lock b/app/igneum-app/Cargo.lock index 557c75108..a1d5e9db0 100644 --- a/app/igneum-app/Cargo.lock +++ b/app/igneum-app/Cargo.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "igneum-app" -version = "0.3.13" +version = "0.3.14" dependencies = [ "ed25519-dalek", "getrandom", diff --git a/app/igneum-app/Cargo.toml b/app/igneum-app/Cargo.toml index 28d4ac4f8..82a8803ba 100644 --- a/app/igneum-app/Cargo.toml +++ b/app/igneum-app/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "igneum-app" -version = "0.3.13" +version = "0.3.14" edition = "2021" description = "Igneum Miner engine: supervises the node, the miner and the GPU workers, and serves the dashboard on 127.0.0.1" license = "MIT" diff --git a/app/igneum-app/resources/igneum-app.rc b/app/igneum-app/resources/igneum-app.rc index f581c416d..23b97b1ef 100644 --- a/app/igneum-app/resources/igneum-app.rc +++ b/app/igneum-app/resources/igneum-app.rc @@ -6,8 +6,8 @@ 1 ICON "igneum.ico" 1 VERSIONINFO -FILEVERSION 0,3,13,0 -PRODUCTVERSION 0,3,13,0 +FILEVERSION 0,3,14,0 +PRODUCTVERSION 0,3,14,0 FILEFLAGSMASK 0x3fL FILEFLAGS 0x0L FILEOS VOS_NT_WINDOWS32 @@ -20,12 +20,12 @@ BEGIN BEGIN VALUE "CompanyName", "Igneum" VALUE "FileDescription", "Igneum Miner engine" - VALUE "FileVersion", "0.3.13" + VALUE "FileVersion", "0.3.14" VALUE "InternalName", "igneum-app" VALUE "LegalCopyright", "Igneum contributors" VALUE "OriginalFilename", "igneum-app.exe" VALUE "ProductName", "Igneum Miner" - VALUE "ProductVersion", "0.3.13" + VALUE "ProductVersion", "0.3.14" END END BLOCK "VarFileInfo" diff --git a/app/igneum-app/src/config.rs b/app/igneum-app/src/config.rs index 214ced9e5..abd308b89 100644 --- a/app/igneum-app/src/config.rs +++ b/app/igneum-app/src/config.rs @@ -39,6 +39,9 @@ pub struct CardPref { pub sweep_class: String, #[serde(default)] pub sweep_source: String, + /// Ember 2: the memory clock the last tune chose (0 = the driver's default) + #[serde(default)] + pub sweep_mem_mhz: u32, } #[derive(Clone, Serialize, Deserialize)] @@ -90,6 +93,15 @@ pub struct Settings { /// unanswered prompt switches it back off with a notice, no retries. #[serde(default)] pub power_control: bool, + /// Ember 2 (6 October 2026): the tune's goal ("efficiency" = most MH per watt within 10% of the top rate, + /// "balanced" = within 1%, "rate" = the fastest point), the electricity price in pence per kWh for the £/day + /// reading on each card, and the hill-climb switch (memory up, core down from the fleet prior; off = the ladders). + #[serde(default = "balanced")] + pub tune_goal: String, + #[serde(default)] + pub power_price_pence: f64, + #[serde(default)] + pub tune_climb: bool, /// When this install first ran (unix s), for the "first hour after install" sweep. #[serde(default)] pub installed_at: u64, @@ -113,13 +125,16 @@ pub struct Settings { fn one() -> u32 { 1 } +fn balanced() -> String { + "balanced".into() +} fn yes() -> bool { true } impl Default for Settings { fn default() -> Settings { - Settings { setup_done: false, address: String::new(), address_source: String::new(), key_saved: false, identities: 1, cards: HashMap::new(), display_name: String::new(), vote: true, paused: false, accepted_total: 0, auto_update: true, remote_jobs: true, prove: false, sweep: true, power_control: false, installed_at: 0, dev_fee: true, fee_total: 0, proof_verify_trust: false, prove_default_applied: false } + Settings { setup_done: false, address: String::new(), address_source: String::new(), key_saved: false, identities: 1, cards: HashMap::new(), display_name: String::new(), vote: true, paused: false, accepted_total: 0, auto_update: true, remote_jobs: true, prove: false, sweep: true, power_control: false, tune_goal: "balanced".into(), power_price_pence: 0.0, tune_climb: false, installed_at: 0, dev_fee: true, fee_total: 0, proof_verify_trust: false, prove_default_applied: false } } } diff --git a/app/igneum-app/src/ember.rs b/app/igneum-app/src/ember.rs index d839135e5..9a9f5a354 100644 --- a/app/igneum-app/src/ember.rs +++ b/app/igneum-app/src/ember.rs @@ -20,9 +20,11 @@ use std::time::{Duration, Instant}; /// The power steps, percent of the card's default limit, highest first (the same ladder as src/sweep.rs). pub const POWER_STEPS_PCT: [u32; 6] = [100, 90, 80, 70, 60, 50]; /// The clock steps, percent of the card's maximum core clock, highest first; 100 = unlocked (the card's own boost). -pub const CLOCK_STEPS_PCT: [u32; 5] = [100, 90, 80, 70, 60]; +/// 6 October 2026, run 6: the 5090's best MH/W sat on the 60% floor (1,854 MHz: 0.563 MH/W, the rate within 0.15%), +/// so the ladder and the floor go to 45% of the maximum; the 1% rate tolerance is the guard below that +pub const CLOCK_STEPS_PCT: [u32; 7] = [100, 90, 80, 70, 60, 50, 45]; /// A card's clock floor when the vendor reports none: this share of its maximum core clock. -pub const CLOCK_FLOOR_PCT: u32 = 60; +pub const CLOCK_FLOOR_PCT: u32 = 45; /// A point may lose this much rate against the fastest point and still win on MH per watt (the manifest can change it). pub const RATE_TOLERANCE_PCT: f64 = 1.0; /// A step whose hottest GPU reading reaches this is marked hot and cannot win (the engine aborts at 90). @@ -49,6 +51,10 @@ pub struct Limits { pub clock_max_mhz: u32, /// the lowest cap the vendor allows (the ADLX gmax_range floor); 0 = CLOCK_FLOOR_PCT of the maximum pub clock_min_mhz: u32, + /// Ember 2: the memory clock the card runs at by default and the vendor's maximum (nvidia-smi clocks.mem and + /// clocks.max.mem); 0 = no memory knob (AMD through ADLX on RDNA 4 exposes none) + pub mem_default_mhz: u32, + pub mem_max_mhz: u32, } impl Limits { @@ -66,6 +72,13 @@ impl Limits { } mhz.clamp(self.clock_floor(), self.clock_max_mhz) } + /// A memory clock inside the vendor's range; 0 stays 0 (the default). + pub fn clamp_mem(&self, mhz: u32) -> u32 { + if mhz == 0 || self.mem_max_mhz == 0 || self.mem_default_mhz == 0 { + return 0; + } + mhz.clamp(self.mem_default_mhz, self.mem_max_mhz) + } /// The watts a power percent asks for, inside the card's min and max, rounded to a watt. pub fn watts_for(&self, pct: u32) -> f64 { let mut w = self.power_default_w * pct as f64 / 100.0; @@ -79,13 +92,15 @@ impl Limits { } } -/// One setting of the two knobs. +/// One setting of the knobs (Ember 2 adds the memory clock: 0 = the driver's default). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Point { /// the core clock cap in MHz; 0 = unlocked pub clock_mhz: u32, /// the power limit, percent of the default pub power_pct: u32, + /// the memory clock in MHz (NVIDIA `-lmc`, locked to one value); 0 = the driver's default + pub mem_mhz: u32, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -96,6 +111,8 @@ pub enum Kind { Clock, /// the fleet prior and one neighbour Confirm, + /// Ember 2: a hill-climb probe + Climb, } #[derive(Clone, Debug, PartialEq)] @@ -106,11 +123,53 @@ pub struct Step { pub kind: Kind, } +/// What the tune is for (Settings > Ember Tune > goal). The rate floor is the share of the best rate seen a point +/// must keep to win on MH per watt: efficiency keeps 90%, balanced 99% (the 1% rule of lever 3), maximum rate +/// takes the fastest point and uses MH per watt only to break ties. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Goal { + Efficiency, + Balanced, + MaxRate, +} + +impl Goal { + pub fn parse(s: &str) -> Goal { + match s { + "efficiency" | "eff" => Goal::Efficiency, + "rate" | "max_rate" | "maximum" => Goal::MaxRate, + _ => Goal::Balanced, + } + } + pub fn name(&self) -> &'static str { + match self { + Goal::Efficiency => "efficiency", + Goal::Balanced => "balanced", + Goal::MaxRate => "rate", + } + } + /// The rate tolerance the choice rule uses, percent under the best rate. + pub fn tolerance_pct(&self, manifest_default: f64) -> f64 { + match self { + Goal::Efficiency => 10.0, + Goal::Balanced => manifest_default, + Goal::MaxRate => 0.0, + } + } +} + +/// Pounds a day for a draw at a price in pence per kWh: watts × 24 h / 1000 × price / 100. +pub fn pounds_per_day(watts: f64, pence_per_kwh: f64) -> f64 { + watts * 24.0 / 1000.0 * pence_per_kwh / 100.0 +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum PlanKind { Full, Confirm, Baseline, + /// Ember 2: the hill-climb over memory up and core down from the start point + Climb, } impl PlanKind { @@ -119,6 +178,7 @@ impl PlanKind { PlanKind::Full => "full", PlanKind::Confirm => "confirm", PlanKind::Baseline => "baseline", + PlanKind::Climb => "climb", } } } @@ -134,9 +194,82 @@ pub struct Plan { power: Vec, clock_pcts: Vec, fixed: Vec, + /// Ember 2 (Climb): the start point, the step sizes and the step budget + climb: Option, +} + +/// The hill-climb's shape: from `start`, each probe moves the memory clock up by `mem_step` or the core clock down +/// by `core_step` (or both), keeps the move when the goal's score improves, else turns to the other knob; a +/// refused step (a fault on it) backs that knob off for good. At most `budget` steps including the start. +#[derive(Clone, Debug, PartialEq)] +pub struct Climb { + pub start: Point, + pub mem_step: u32, + pub core_step: u32, + pub budget: usize, + pub goal: Goal, } impl Plan { + /// Ember 2: the hill-climb. The start is the fleet prior (or the card's current point); a probe step is 5% of + /// the memory range above the default (0 when the card has no memory knob) and 5% of the maximum core clock; + /// five steps of 60 s converge in under 10 minutes. + pub fn climb(limits: &Limits, start: Point, goal: Goal, tolerance_pct: f64) -> Plan { + let mem_step = if limits.mem_max_mhz > limits.mem_default_mhz { ((limits.mem_max_mhz - limits.mem_default_mhz) / 20).max(25) } else { 0 }; + let core_step = if limits.clock_max_mhz > 0 { (limits.clock_max_mhz / 20).max(25) } else { 0 }; + let start = Point { clock_mhz: limits.clamp_clock(start.clock_mhz), power_pct: start.power_pct.clamp(50, 100), mem_mhz: limits.clamp_mem(start.mem_mhz) }; + Plan { kind: PlanKind::Climb, limits: limits.clone(), before: start, tolerance_pct: goal.tolerance_pct(tolerance_pct), power: Vec::new(), clock_pcts: Vec::new(), fixed: Vec::new(), climb: Some(Climb { start, mem_step, core_step, budget: 5, goal }) } + } + + /// The goal's score of a row: MH per watt for efficiency and balanced, the rate for maximum rate. + pub fn score(&self, r: &Row) -> f64 { + match self.climb.as_ref().map(|c| c.goal) { + Some(Goal::MaxRate) => r.mhs, + _ => r.eff, + } + } + + /// The climb's next probe from the rows so far: None when the budget is spent or no move is left. + fn climb_next(&self, rows: &[Row]) -> Option { + let c = self.climb.as_ref()?; + let step = |p: Point| Step { point: p, watts: self.limits.watts_for(p.power_pct), kind: Kind::Climb }; + if rows.is_empty() { + return Some(step(c.start)); + } + if rows.len() >= c.budget { + return None; + } + // the best usable row so far is the hill's top; a knob that produced a marked (refused) row is backed off + let best = rows.iter().filter(|r| r.usable()).max_by(|a, b| self.score(a).partial_cmp(&self.score(b)).unwrap_or(std::cmp::Ordering::Equal))?; + let refused_mem = rows.iter().any(|r| !r.usable() && r.point.mem_mhz > best.point.mem_mhz); + let refused_core = rows.iter().any(|r| !r.usable() && r.point.clock_mhz != 0 && (best.point.clock_mhz == 0 || r.point.clock_mhz < best.point.clock_mhz)); + let last = rows.last()?; + let mem_up = |p: Point| -> Option { + if c.mem_step == 0 || refused_mem { return None; } + let base = if p.mem_mhz == 0 { self.limits.mem_default_mhz } else { p.mem_mhz }; + let m = self.limits.clamp_mem(base + c.mem_step); + (m > 0 && m != p.mem_mhz && m != base).then_some(Point { mem_mhz: m, ..p }) + }; + let core_down = |p: Point| -> Option { + if c.core_step == 0 || refused_core { return None; } + let base = if p.clock_mhz == 0 { self.limits.clock_max_mhz } else { p.clock_mhz }; + let k = self.limits.clamp_clock(base.saturating_sub(c.core_step)); + (k > 0 && k != p.clock_mhz).then_some(Point { clock_mhz: k, ..p }) + }; + let tried = |p: Point| rows.iter().any(|r| r.point == p); + // the last move improved: keep going the same way from the top; else turn: memory first, then core, then both + let last_improved = last.usable() && last.point == best.point && rows.len() > 1; + let last_was_mem = rows.len() > 1 && last.point.mem_mhz != rows[rows.len() - 2].point.mem_mhz; + let candidates: Vec> = if last_improved && last_was_mem { + vec![mem_up(best.point), core_down(best.point)] + } else if last_improved { + vec![core_down(best.point), mem_up(best.point)] + } else { + vec![mem_up(best.point), core_down(best.point), mem_up(best.point).and_then(core_down)] + }; + candidates.into_iter().flatten().find(|p| !tried(*p)).map(step) + } + /// Power 100% to 50% at the unlocked clock (duplicate watts dropped, as the card's floor clamps them), then the /// clock ladder 90% to the floor of the maximum core clock at the chosen power. A card without a readable /// maximum clock gets the power ladder only; a card without a default limit gets the clock ladder only. @@ -148,41 +281,44 @@ impl Plan { if power.last().map(|s: &Step| (s.watts - w).abs() < 0.5).unwrap_or(false) { continue; } - power.push(Step { point: Point { clock_mhz: 0, power_pct: pct }, watts: w, kind: Kind::Power }); + power.push(Step { point: Point { clock_mhz: 0, power_pct: pct, mem_mhz: 0 }, watts: w, kind: Kind::Power }); } } let clock_pcts = if limits.clock_max_mhz > 0 { CLOCK_STEPS_PCT[1..].to_vec() } else { Vec::new() }; - Plan { kind: PlanKind::Full, limits: limits.clone(), before, tolerance_pct, power, clock_pcts, fixed: Vec::new() } + Plan { kind: PlanKind::Full, limits: limits.clone(), before, tolerance_pct, power, clock_pcts, fixed: Vec::new(), climb: None } } /// The prior's point, then one neighbour: the next clock step up when the prior caps the clock (is the cap /// costing rate?), else one power step down (is there efficiency left?). pub fn confirm(limits: &Limits, prior: Point, before: Point, tolerance_pct: f64) -> Plan { - let p = Point { clock_mhz: limits.clamp_clock(prior.clock_mhz), power_pct: prior.power_pct.clamp(50, 100) }; + let p = Point { clock_mhz: limits.clamp_clock(prior.clock_mhz), power_pct: prior.power_pct.clamp(50, 100), mem_mhz: limits.clamp_mem(prior.mem_mhz) }; let first = Step { point: p, watts: limits.watts_for(p.power_pct), kind: Kind::Confirm }; let neighbour = if p.clock_mhz > 0 && limits.clock_max_mhz > 0 { let up = p.clock_mhz + limits.clock_max_mhz / 10; let clock = if up >= limits.clock_max_mhz { 0 } else { limits.clamp_clock(up) }; - Point { clock_mhz: clock, power_pct: p.power_pct } + Point { clock_mhz: clock, power_pct: p.power_pct, mem_mhz: p.mem_mhz } } else { - Point { clock_mhz: p.clock_mhz, power_pct: (p.power_pct.saturating_sub(10)).max(50) } + Point { clock_mhz: p.clock_mhz, power_pct: (p.power_pct.saturating_sub(10)).max(50), mem_mhz: p.mem_mhz } }; let mut fixed = vec![first]; if neighbour != p { fixed.push(Step { point: neighbour, watts: limits.watts_for(neighbour.power_pct), kind: Kind::Confirm }); } - Plan { kind: PlanKind::Confirm, limits: limits.clone(), before, tolerance_pct, power: Vec::new(), clock_pcts: Vec::new(), fixed } + Plan { kind: PlanKind::Confirm, limits: limits.clone(), before, tolerance_pct, power: Vec::new(), clock_pcts: Vec::new(), fixed, climb: None } } /// One step at the card's current point: the before number, and all a measure-only card (Apple, or NVIDIA /// with Power control off) reports. pub fn baseline(limits: &Limits, before: Point, tolerance_pct: f64) -> Plan { let fixed = vec![Step { point: before, watts: limits.watts_for(before.power_pct), kind: Kind::Baseline }]; - Plan { kind: PlanKind::Baseline, limits: limits.clone(), before, tolerance_pct, power: Vec::new(), clock_pcts: Vec::new(), fixed } + Plan { kind: PlanKind::Baseline, limits: limits.clone(), before, tolerance_pct, power: Vec::new(), clock_pcts: Vec::new(), fixed, climb: None } } /// How many steps the plan has at most (the clock ladder counts whether or not it runs). pub fn len(&self) -> usize { + if let Some(c) = &self.climb { + return c.budget; + } self.fixed.len() + self.power.len() + self.clock_pcts.len() } pub fn is_empty(&self) -> bool { @@ -191,6 +327,9 @@ impl Plan { /// The next step after `rows` (one row per step done so far), or None when the plan is complete. pub fn next(&self, rows: &[Row]) -> Option { + if self.climb.is_some() { + return self.climb_next(rows); + } let i = rows.len(); if !self.fixed.is_empty() { return self.fixed.get(i).cloned(); @@ -207,7 +346,7 @@ impl Plan { if rows.last().map(|r| r.point.clock_mhz == clock).unwrap_or(false) { return None; } - Some(Step { point: Point { clock_mhz: clock, power_pct }, watts: self.limits.watts_for(power_pct), kind: Kind::Clock }) + Some(Step { point: Point { clock_mhz: clock, power_pct, mem_mhz: 0 }, watts: self.limits.watts_for(power_pct), kind: Kind::Clock }) } } @@ -305,9 +444,10 @@ impl Row { /// `TUNE card=