diff --git a/docs/plans/counter-asic-2-gate/class-v3-20261005-2225Z-metal-mx8.json b/docs/plans/counter-asic-2-gate/class-v3-20261005-2225Z-metal-mx8.json new file mode 100644 index 000000000..cebc94c49 --- /dev/null +++ b/docs/plans/counter-asic-2-gate/class-v3-20261005-2225Z-metal-mx8.json @@ -0,0 +1,474 @@ +{ + "pass": false, + "checks": { + "switch_line_on_every_node": true, + "switch_line_names_the_rounded_epoch": true, + "template_switched_at_the_first_v3_epoch": true, + "blocks_before_the_boundary": true, + "blocks_after_the_boundary": true, + "v2_and_v3_programs_seen": true, + "program_ids_differ_across_the_switch": true, + "miners_agree_on_every_program": true, + "zero_rejected_by_miners": true, + "zero_rejected_by_nodes": true, + "sinks_agree": true, + "block_counts_agree": true, + "metal_prepare_sent_for_v3": true, + "metal_worker_prepared_v3_pack": true, + "metal_no_need_or_mismatch": false, + "metal_accepted_blocks_after_switch": true, + "metal_cpu_recheck_clean": true, + "metal_swapped_without_pause": true + }, + "activation": 150, + "epoch_blocks": 60, + "first_v3_epoch": 3, + "boundary_daa": 180, + "secs": 480, + "threads": 1, + "node": "/Users/joshm/Projects/igneum-wt-ca2-v3/vendor/igneum-node-ca2/target-ca2/release/igneumd", + "miner": "/Users/joshm/Projects/igneum-wt-ca2-v3/vendor/igneum-node-ca2/target-ca2/release/igneum-miner", + "genesis_bits": "0x1e010000", + "template_switch": { + "epoch": 3, + "daa": 180, + "at": 264.5 + }, + "run_ended_at_s": 366.8, + "final_daa": 301, + "blocks": { + "total": 305, + "before_boundary": 182, + "after_boundary": 123, + "chain_before": 91, + "chain_after": 67 + }, + "programs": [ + { + "epoch": 0, + "class": "v2", + "program_id": "3e974094b5ce543f", + "seed": "130e3b7d568db07a", + "miners": 2, + "disagree": false, + "ready_ms": [ + 192, + 192 + ] + }, + { + "epoch": 1, + "class": "v2", + "program_id": "1d5b129e89ec2995", + "seed": "bc056261dafe0ec2", + "miners": 2, + "disagree": false, + "ready_ms": [ + 4, + 4 + ] + }, + { + "epoch": 2, + "class": "v2", + "program_id": "835e953cdc2c11be", + "seed": "1908705502833bdf", + "miners": 2, + "disagree": false, + "ready_ms": [ + 9, + 7 + ] + }, + { + "epoch": 3, + "class": "v3", + "program_id": "082b7ee882aaf418", + "seed": "4ae011065b84f679", + "miners": 2, + "disagree": false, + "ready_ms": [ + 1069, + 1085 + ] + }, + { + "epoch": 4, + "class": "v3", + "program_id": "7ef81e08e4611665", + "seed": "d5964a745dbe2ce2", + "miners": 2, + "disagree": false, + "ready_ms": [ + 6, + 6 + ] + }, + { + "epoch": 5, + "class": "v3", + "program_id": "ecee6312983c15ee", + "seed": "f9d5de9eea7330ae", + "miners": 2, + "disagree": false, + "ready_ms": [ + 5, + 5 + ] + } + ], + "accepted_per_miner": [ + 301, + 2, + 1 + ], + "rejected_by_miners": [ + 0, + 0, + 0 + ], + "rejected_by_nodes": [ + 0, + 0, + 0 + ], + "rejected_lines": [], + "sinks": [ + "66a33eca15d33ba2", + "66a33eca15d33ba2", + "66a33eca15d33ba2" + ], + "block_counts": [ + 304, + 304, + 304 + ], + "tips_per_node": [ + 2, + 2, + 2 + ], + "switch_lines": [ + "Program class v3 from the override file: active from epoch 3 (DAA score 150 rounded up to the epoch boundary at 180, epochs of 60 DAA)", + "Program class v3 from the override file: active from epoch 3 (DAA score 150 rounded up to the epoch boundary at 180, epochs of 60 DAA)", + "Program class v3 from the override file: active from epoch 3 (DAA score 150 rounded up to the epoch boundary at 180, epochs of 60 DAA)" + ], + "samples": [ + { + "t": 4.6, + "daa": 1, + "epoch": 0, + "class": 2, + "nodes": [ + "1/97755438", + "0/130e3b7d", + "0/130e3b7d" + ] + }, + { + "t": 19.7, + "daa": 3, + "epoch": 0, + "class": 2, + "nodes": [ + "3/52e2c440", + "3/52e2c440", + "3/52e2c440" + ] + }, + { + "t": 34.7, + "daa": 3, + "epoch": 0, + "class": 2, + "nodes": [ + "3/52e2c440", + "3/52e2c440", + "3/52e2c440" + ] + }, + { + "t": 49.7, + "daa": 13, + "epoch": 0, + "class": 2, + "nodes": [ + "13/fa702087", + "13/fa702087", + "13/fa702087" + ] + }, + { + "t": 64.8, + "daa": 38, + "epoch": 0, + "class": 2, + "nodes": [ + "38/2a687376", + "38/2a687376", + "38/2a687376" + ] + }, + { + "t": 79.8, + "daa": 55, + "epoch": 0, + "class": 2, + "nodes": [ + "55/2f1f0ea8", + "55/2f1f0ea8", + "55/2f1f0ea8" + ] + }, + { + "t": 94.8, + "daa": 57, + "epoch": 0, + "class": 2, + "nodes": [ + "57/e1b2abc1", + "57/e1b2abc1", + "57/e1b2abc1" + ] + }, + { + "t": 109.9, + "daa": 61, + "epoch": 1, + "class": 2, + "nodes": [ + "61/86fa7300", + "61/86fa7300", + "61/86fa7300" + ] + }, + { + "t": 124.9, + "daa": 61, + "epoch": 1, + "class": 2, + "nodes": [ + "61/86fa7300", + "61/86fa7300", + "61/86fa7300" + ] + }, + { + "t": 140, + "daa": 61, + "epoch": 1, + "class": 2, + "nodes": [ + "61/86fa7300", + "61/86fa7300", + "61/86fa7300" + ] + }, + { + "t": 155, + "daa": 73, + "epoch": 1, + "class": 2, + "nodes": [ + "73/e7f6e171", + "73/e7f6e171", + "73/e7f6e171" + ] + }, + { + "t": 170.1, + "daa": 94, + "epoch": 1, + "class": 2, + "nodes": [ + "94/0b1c93b1", + "94/0b1c93b1", + "94/0b1c93b1" + ] + }, + { + "t": 185.2, + "daa": 114, + "epoch": 1, + "class": 2, + "nodes": [ + "114/16034702", + "114/16034702", + "114/16034702" + ] + }, + { + "t": 200.2, + "daa": 117, + "epoch": 1, + "class": 2, + "nodes": [ + "117/d018a345", + "117/d018a345", + "117/d018a345" + ] + }, + { + "t": 215.4, + "daa": 120, + "epoch": 2, + "class": 2, + "nodes": [ + "120/e022d64e", + "120/e022d64e", + "120/e022d64e" + ] + }, + { + "t": 230.4, + "daa": 134, + "epoch": 2, + "class": 2, + "nodes": [ + "134/e7d07203", + "134/e7d07203", + "134/e7d07203" + ] + }, + { + "t": 245.4, + "daa": 157, + "epoch": 2, + "class": 2, + "nodes": [ + "157/d23b3a1b", + "157/d23b3a1b", + "157/d23b3a1b" + ] + }, + { + "t": 260.5, + "daa": 175, + "epoch": 2, + "class": 2, + "nodes": [ + "175/4bf5acbe", + "175/4bf5acbe", + "175/4bf5acbe" + ] + }, + { + "t": 275.5, + "daa": 198, + "epoch": 3, + "class": 3, + "nodes": [ + "198/f337dbc1", + "198/f337dbc1", + "198/f337dbc1" + ] + }, + { + "t": 290.6, + "daa": 217, + "epoch": 3, + "class": 3, + "nodes": [ + "217/dc28325d", + "217/dc28325d", + "217/dc28325d" + ] + }, + { + "t": 305.6, + "daa": 234, + "epoch": 3, + "class": 3, + "nodes": [ + "234/a0079166", + "234/a0079166", + "234/a0079166" + ] + }, + { + "t": 320.7, + "daa": 249, + "epoch": 4, + "class": 3, + "nodes": [ + "249/eb206cb2", + "249/eb206cb2", + "249/eb206cb2" + ] + }, + { + "t": 335.7, + "daa": 268, + "epoch": 4, + "class": 3, + "nodes": [ + "268/9b9866b1", + "268/9b9866b1", + "268/9b9866b1" + ] + }, + { + "t": 350.7, + "daa": 285, + "epoch": 4, + "class": 3, + "nodes": [ + "285/3263ff7a", + "285/3263ff7a", + "285/3263ff7a" + ] + }, + { + "t": 365.8, + "daa": 299, + "epoch": 4, + "class": 3, + "nodes": [ + "299/31eff7d0", + "299/31eff7d0", + "299/31eff7d0" + ] + } + ], + "metal": { + "worker": "/Users/joshm/Projects/igneum-wt-ca2-v3/proto-metal/igneum-bench-ca2", + "prepares": [ + "PREPARE sent for epoch seed b7eda892cff02b39c043f5fa683a7c1629c96d02fa480522603be595372e371f day 1243916 class v2 (6 DAA blocks before the boundary at 60; CPU side ready in 960 ms)", + "PREPARE sent for epoch seed bc056261dafe0ec266f2e8f508e8fe7349e4d0c064853c6fd37ffa6a5e1fab3c day 1243916 class v2 (the current pair, after seed mismatches; CPU side ready in 1248 ms)", + "PREPARE sent for epoch seed 1908705502833bdf6c5d08b6d6ae3a364e3fe2bf0956504b20bc5071eecbdfc3 day 1243916 class v2 (6 DAA blocks before the boundary at 120; CPU side ready in 1768 ms)", + "PREPARE sent for epoch seed 4ae011065b84f67924d0c42ff63669e6124b6bddc4a7f71a3f1075b5891efd8c day 1243916 class v3 (6 DAA blocks before the boundary at 180; CPU side ready in 1740 ms)", + "PREPARE sent for epoch seed d5964a745dbe2ce2a88d6a4198a244996f1f4aba6cc1bc20841bb532e469fdd6 day 1243916 class v3 (6 DAA blocks before the boundary at 240; CPU side ready in 1289 ms)", + "PREPARE sent for epoch seed f9d5de9eea7330aeec83c0994b8929e61a2ac2f13dbae6462cceace3b1d6f24e day 1243916 class v3 (7 DAA blocks before the boundary at 300; CPU side ready in 1166 ms)" + ], + "prepared": [ + "worker: prepared b7eda892cff02b39c043f5fa683a7c1629c96d02fa480522603be595372e371f 69676e65756d2d6461792f0cfb120000000000 35663.4 program 64.5 dataset 0.0 race 35598.9 variant base class v2 loads/hash 128 cache-fill 2.0 build 30.5 resident 2 programs 1 datasets (prepare answered 1.3 s after it was sent)", + "worker: prepared bc056261dafe0ec266f2e8f508e8fe7349e4d0c064853c6fd37ffa6a5e1fab3c 69676e65756d2d6461792f0cfb120000000000 33632.6 program 108.1 dataset 0.0 race 33524.5 variant base class v2 loads/hash 128 cache-fill 2.0 build 30.5 resident 2 programs 1 datasets (prepare answered 35.2 s after it was sent)", + "worker: prepared 1908705502833bdf6c5d08b6d6ae3a364e3fe2bf0956504b20bc5071eecbdfc3 69676e65756d2d6461792f0cfb120000000000 34728.4 program 77.3 dataset 0.0 race 34651.0 variant base class v2 loads/hash 128 cache-fill 2.0 build 30.5 resident 2 programs 1 datasets (prepare answered 0.0 s after it was sent)", + "worker: prepared 4ae011065b84f67924d0c42ff63669e6124b6bddc4a7f71a3f1075b5891efd8c 69676e65756d2d6461792f0cfb120000000000 252.5 program 55.5 dataset 196.9 race 0.0 variant base class v3 loads/hash 128 cache-fill 0.9 build 41.1 resident 2 programs 2 datasets (prepare answered 0.3 s after it was sent)", + "worker: prepared d5964a745dbe2ce2a88d6a4198a244996f1f4aba6cc1bc20841bb532e469fdd6 69676e65756d2d6461792f0cfb120000000000 51.1 program 51.1 dataset 0.0 race 0.0 variant base class v3 loads/hash 128 cache-fill 0.9 build 41.1 resident 2 programs 2 datasets (prepare answered 0.1 s after it was sent)", + "worker: prepared f9d5de9eea7330aeec83c0994b8929e61a2ac2f13dbae6462cceace3b1d6f24e 69676e65756d2d6461792f0cfb120000000000 46.0 program 46.0 dataset 0.0 race 0.0 variant base class v3 loads/hash 128 cache-fill 0.9 build 41.1 resident 2 programs 2 datasets (prepare answered 0.0 s after it was sent)" + ], + "prepared_v3": [ + "worker: prepared 4ae011065b84f67924d0c42ff63669e6124b6bddc4a7f71a3f1075b5891efd8c 69676e65756d2d6461792f0cfb120000000000 252.5 program 55.5 dataset 196.9 race 0.0 variant base class v3 loads/hash 128 cache-fill 0.9 build 41.1 resident 2 programs 2 datasets (prepare answered 0.3 s after it was sent)", + "worker: prepared d5964a745dbe2ce2a88d6a4198a244996f1f4aba6cc1bc20841bb532e469fdd6 69676e65756d2d6461792f0cfb120000000000 51.1 program 51.1 dataset 0.0 race 0.0 variant base class v3 loads/hash 128 cache-fill 0.9 build 41.1 resident 2 programs 2 datasets (prepare answered 0.1 s after it was sent)", + "worker: prepared f9d5de9eea7330aeec83c0994b8929e61a2ac2f13dbae6462cceace3b1d6f24e 69676e65756d2d6461792f0cfb120000000000 46.0 program 46.0 dataset 0.0 race 0.0 variant base class v3 loads/hash 128 cache-fill 0.9 build 41.1 resident 2 programs 2 datasets (prepare answered 0.0 s after it was sent)" + ], + "need": 0, + "mismatch_lines": [ + "1791239226.967 PACK OUT OF DATE: the prepared pair b7eda892cff02b39 is not the node's epoch bc056261dafe0ec2; rebuilding the program pack for the worker" + ], + "refused": [], + "swaps": [ + "SEED CHANGE at daa 60: epoch seed 130e3b7d568db07aad638f67b2ac26e7cf93c51e4f564d45c6724fc4b2d3da65 -> bc056261dafe0ec266f2e8f508e8fe7349e4d0c064853c6fd37ffa6a5e1fab3c, day 1243916 -> 1243916 (epoch 1): the pair was not prepared (unexpected seeds); the worker compiles inline", + "SEED CHANGE at daa 120: epoch seed bc056261dafe0ec266f2e8f508e8fe7349e4d0c064853c6fd37ffa6a5e1fab3c -> 1908705502833bdf6c5d08b6d6ae3a364e3fe2bf0956504b20bc5071eecbdfc3, day 1243916 -> 1243916 (epoch 2): prepare was sent but the worker has not answered prepared yet; it compiles inline", + "SEED CHANGE at daa 180: epoch seed 1908705502833bdf6c5d08b6d6ae3a364e3fe2bf0956504b20bc5071eecbdfc3 -> 4ae011065b84f67924d0c42ff63669e6124b6bddc4a7f71a3f1075b5891efd8c, day 1243916 -> 1243916 (epoch 3): swapped with no pause (prepared 4 s ago, prepare took 252 ms)", + "SEED CHANGE at daa 240: epoch seed 4ae011065b84f67924d0c42ff63669e6124b6bddc4a7f71a3f1075b5891efd8c -> d5964a745dbe2ce2a88d6a4198a244996f1f4aba6cc1bc20841bb532e469fdd6, day 1243916 -> 1243916 (epoch 4): swapped with no pause (prepared 4 s ago, prepare took 51 ms)", + "SEED CHANGE at daa 300: epoch seed d5964a745dbe2ce2a88d6a4198a244996f1f4aba6cc1bc20841bb532e469fdd6 -> f9d5de9eea7330aeec83c0994b8929e61a2ac2f13dbae6462cceace3b1d6f24e, day 1243916 -> 1243916 (epoch 5): swapped with no pause (prepared 6 s ago, prepare took 46 ms)" + ], + "accepted_total": 301, + "accepted_after_switch": 124, + "found_lines": 0, + "cpu_recheck_mismatched": 0, + "last_status": "" + } +} \ No newline at end of file diff --git a/docs/plans/counter-asic-2-node.md b/docs/plans/counter-asic-2-node.md index 7586f2240..445d0302f 100644 --- a/docs/plans/counter-asic-2-node.md +++ b/docs/plans/counter-asic-2-node.md @@ -136,6 +136,20 @@ Binaries rebuilt on ca2-v3 d233fa1 (mixer x8, the era drawn inside the class, th Epoch 0's v2 id `8f8806638d59850f` is the same in all three runs. Three runs, three v3 classes (mixer x4; x4 with the era; x8 with the era), the same chain behaviour each time: the switch rounds up to epoch 3, no block rejected, one chain. +### Result, run 4 (5 October 2026, 22:25:22Z to 22:31:28Z): a real Metal miner across the boundary (gate G4b) + +The fleet-outage case: the three runs above used CPU miners, so the Metal worker's class v3 path had not mined. Run 4 puts node 0's miner on the Metal worker the way the app does (`igneum-miner --worker igneum-bench --prepare-packs --exit-on-seed-change`, `igneum-bench` built from ca2-v3 00c55aa: a class v3 program comes from the pack's `program_bound.metal`, its day from the pack's `memhard.metal` (the x8 construction, the era layout), keyed by (day, class, era)), genesis bits 0x1e010000 (2^24 hashes per block), CPU miners on nodes 1 and 2. Summary: `docs/plans/counter-asic-2-gate/class-v3-20261005-2225Z-metal-mx8.json`. + +| Check | Measured | +|---|---| +| The chain | 182 / 123 blocks across DAA 180, 0 rejected (miners and nodes), sinks `66a33eca15d33ba2` on all three, 304 / 304 / 304, 3 of 3 switch lines, the switch at DAA 180, 264.5 s wall | +| PREPARE lines | 6 (three class v2, three class v3), each 6 to 7 DAA before its boundary, the v3 ones with the pack directory and `class=v3 era=` | +| The worker's `prepared` for the v3 packs | three: `prepared 252.5 program 55.5 dataset 196.9 race 0.0 variant base class v3 loads/hash 128 cache-fill 0.9 build 41.1` (the first, with the day built from the pack: 0.9 ms cache fill, 41.1 ms build on the GPU), then 51.1 and 46.0 ms (the day resident) | +| The swap at the boundary | `SEED CHANGE at daa 180 ... (epoch 3): swapped with no pause (prepared 4 s ago, prepare took 252 ms)`; the same at 240 and 300 | +| Blocks on v3 | 124 accepted after the switch (301 in the run), every one re-checked on the CPU: `mismatched 0`; `need` 0; no class or era mismatch line; no refusal, no exit 42 or 44 | + +One line before the switch, on the v2 path: `PACK OUT OF DATE: the prepared pair b7eda892... is not the node's epoch bc056261...; rebuilding the program pack` at DAA 60: the epoch-1 seed reported at `boundary - lead` flipped (the quarter-lead confirm is 3 DAA on the 60x profile, 150 on the devnet) and the miner's rebuild path of 05ef0fa3 wrote the right pack. The v2 prepares answered 33 to 35 s after they were sent (the Metal variant race runs inside the prepare, longer than the 10-DAA lead of the profile; 600 s on the devnet), so epochs 1 and 2 compiled inline; a pack program races nothing and answered in 252 ms. The v3 path is clean; the script now judges the Metal checks from the first v3 prepare on (the run's own report flagged the v2 line and read FAIL on that one check). + The PC 2 suite jobs for the same fork (79bd8e10), the G6 evidence (the coordinator's status file carries the SUMMARY lines): | Job | Tree | What | Result | @@ -161,7 +175,7 @@ The PC's test stage builds the fork's test binaries with the plan's feature set, ## 7. Unverified, and what is owed - The era walk for era >= 1 has never run (the devnet is 180 days from era 1); a pruning-proof sync past era 0 fails with `MissingEraSeed` until an era witness exists in the proof format. -- The Metal worker has no class v3 generator (it regenerates from the seed in Swift): the integration branch adds generator 3 there, or the Mac mines v3 with the OpenCL worker from a pack. +- The Metal worker takes class v3 from the pack (program and day) and mined across the boundary in run 4; the app must pass `--prepare-packs` to its Metal worker (the app branch fixes `engine.rs`, which passed it to the non-Metal workers only). - The GPU workers' class refusal was checked by the C test of `pf_pack_class_ok` and the syntax of both hosts, not by a live worker on a v3 pack: the integration's bit-exact gate (G1) is where a real worker first builds a v3 pack. - `next_pair` keeps the current era seed for the next epoch; an epoch boundary that is also an era boundary (once per 180 days) would prepare the wrong era, and the job line then names the right one, so the worker refuses the prepared pair and the miner prepares again (one wasted compile, no wrong block). The VDF era seed replaces the stand-in before this matters. - Done 22:05Z: ca2-cache rebased as 1950661 fast-forwarded (the first attempt, 2de19e5 on 464d6e1, conflicted in 8 files and was aborted); igneum-pow 53 + 19 tests and the packfile test pass; the fork's kaspa-pow, miner and kaspad check clean against the merged crate (seam unchanged). `V3_CLASS = { era: None, hot: None, ..LoadClass::MX4 }`. diff --git a/infra/fast-time/class-v3.mjs b/infra/fast-time/class-v3.mjs index 3a82a3ea0..b20647aec 100644 --- a/infra/fast-time/class-v3.mjs +++ b/infra/fast-time/class-v3.mjs @@ -218,7 +218,12 @@ if (METAL) { const prepared = L.filter(l => /worker: prepared /.test(l)).map(l => l.replace(/^\S+ /, '')); const preparedV3 = prepared.filter(l => / class v3 /.test(l)); const need = L.filter(l => /worker: need |^\S+ worker error:.*need /.test(l) || /\bneed [0-9a-f]{64}/.test(l)); - const mismatch = L.filter(l => /program class mismatch|era seed mismatch|PACK OUT OF DATE|pack .*: program pack/.test(l)); + // a PACK OUT OF DATE before the first class v3 prepare is the v2 rebuild path at work (a seed that flipped inside the + // quarter-lead confirm window, 3 DAA on the 60x profile); the gate judges the v3 path from its first prepare on + const firstV3Prepare = L.findIndex(l => /PREPARE sent for epoch seed .* class v3/.test(l)); + const sinceV3 = (l, i) => firstV3Prepare < 0 || i >= firstV3Prepare; + const mismatch = L.filter((l, i) => sinceV3(l, i) && /program class mismatch|era seed mismatch|PACK OUT OF DATE|pack .*: program pack/.test(l)); + const mismatchBefore = L.filter((l, i) => !sinceV3(l, i) && /program class mismatch|era seed mismatch|PACK OUT OF DATE/.test(l)); const refused = L.filter(l => /exiting with code 4[24]|refused the program pack|prepare-failed/.test(l)); const swaps = L.filter(l => /swapped with no pause|compiles inline|worker without prepare support/.test(l)).map(l => l.replace(/^\S+ /, '')); const accepted = L.filter(l => /ACCEPTED block/.test(l)); @@ -228,7 +233,7 @@ if (METAL) { const lastStatus = status.at(-1) || ''; const mismatched = +(/mismatched=(\d+)/.exec(lastStatus)?.[1] ?? 0); const rate = /hash=([\d.]+) MH\/s/.exec(lastStatus)?.[1]; - metal = { worker: METAL, prepares, prepared, prepared_v3: preparedV3, need: need.length, mismatch_lines: mismatch, refused, swaps, accepted_total: accepted.length, accepted_after_switch: acceptedAfter, found_lines: found, cpu_recheck_mismatched: mismatched, rate_mh_s: rate, last_status: lastStatus }; + metal = { worker: METAL, prepares, prepared, prepared_v3: preparedV3, need: need.length, mismatch_lines: mismatch, v2_rebuild_lines_before_v3: mismatchBefore, refused, swaps, accepted_total: accepted.length, accepted_after_switch: acceptedAfter, found_lines: found, cpu_recheck_mismatched: mismatched, rate_mh_s: rate, last_status: lastStatus }; } const checks = {