From b17d467f1125c754631d0116b01668351098aae1 Mon Sep 17 00:00:00 2001 From: igneum-labs <337424239+igneum-labs@users.noreply.github.com> Date: Fri, 9 Oct 2026 04:26:48 +0000 Subject: [PATCH] same-work: the 3090 PASS on all three phases of all three contexts (read back on build-1 05:00 to 05:14 UK), the CUDA cell on three cards per context; OpenCL take 2 on the first rig phase 1 PASS with the day-switch read on take 3; the runner and job faults named Co-Authored-By: Claude Fable 5.1 --- docs/plans/igneum-2.0-test-registry.json | 81 +++++++++++---------- tools/ci/batches/same-work-20261008-01.json | 6 +- tools/ci/batches/same-work-20261008-02.json | 10 +-- tools/ci/batches/same-work-20261008-03.json | 10 +-- 4 files changed, 57 insertions(+), 50 deletions(-) diff --git a/docs/plans/igneum-2.0-test-registry.json b/docs/plans/igneum-2.0-test-registry.json index 111e8e43f..1c2be75a3 100644 --- a/docs/plans/igneum-2.0-test-registry.json +++ b/docs/plans/igneum-2.0-test-registry.json @@ -1380,31 +1380,32 @@ "manual_page": 24, "owner_lane": "hash lane (a690540514aa453d7)", "run_status": "NOT RUN", - "evidence_path": "docs/analysis/review-2026-10-08-b/f10/emu-test.log; build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-3.json; build-1:/srv/artefacts/tas/202-5d53a591-1176efb9/node-5d53a591/box4-miner.log", - "run_id": "202-5d53a591-1176efb9", - "updated": "2026-10-09T00:33:39.243Z", + "evidence_path": "docs/analysis/review-2026-10-08-b/f10/emu-test.log; build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-3.json; build-1:/srv/artefacts/tas/202-5d53a591-1176efb9/node-5d53a591/box4-miner.log", + "run_id": "same-work-20261008-01", + "updated": "2026-10-09T04:26:48.369Z", "evidence_record": { "requirement_id": "POW-01", "decision": "NOT RUN", - "method": "native", - "cell": "suite:miner", - "manifest_sha": "5d53a591", - "run_id": "202-5d53a591-1176efb9", - "evidence": "build-1:/srv/artefacts/tas/202-5d53a591-1176efb9/node-5d53a591/box4-miner.log", - "in_progress": false, - "coverage": "partial: the pack re-check seam against the pack's 96 vectors (recheck_pack); the million-vector campaign is harness:p01-vectors", + "method": "GPU", + "cell": "harness:p01-vectors", + "manifest_sha": "c30ab32c", + "run_id": "same-work-20261008-01", + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-3.json", + "in_progress": true, + "coverage": "the million-vector campaign per backend per pack, bit-for-bit; PASS only when every supported backend reads a million vectors with zero disagreement; the malformed-input half is harness:parser-malformed", "release_identity": { - "commit": "5d53a591", + "commit": "c30ab32c; the CPU reference class-v6 1a938abe4; the CUDA reader the kit's gen 6 worker 804a6f7f", "lockfile": "", - "binary": "", - "network_object": "", - "activation": "", + "binary": "igneum-pow sha256 0d3f3fca...; igneum-worker-cuda sha256 804a6f7f...", + "network_object": "the frozen class v6 object: pack hl-v6-all (tgz sha256 9131431015e102f4..., re-exported byte for byte from class-v6 1a938abe4 on build-4 at 21:44 UK, every file's sha256 equal), id 0x4de7b836cc40a4ea, epoch seed edc4fa84... (the genesis seeds) over the node1 state stream", + "activation": "none (a re-check of hashes against one job context, no chain state changes)", "profile_hashes": "" }, - "claim_impact": "", + "claim_impact": "POW-01's CUDA half across a dataset-day switch on the frozen object: CUDA against the CPU reference bit for bit on three ranges with the day switch at nonce 1572864. NOT same-work evidence: the node engine refuses this seed/stream pair (IgneumEngine::epoch_for's class v5 check wants the stream's block af89be5d... to equal the epoch seed edc4fa84...), so the node and pool readers cannot read it; the coordinator's ruling 22:0x UK keeps it as the CUDA and CPU-only row", "reviewer": "", - "at": "2026-10-09T00:33:39.243Z", - "note": "suite GREEN (rc 0, every test passed); the cell covers its cases partially, so the case reads NOT RUN in progress with this evidence until its own acceptance is exercised; miner on release-2.0.2-node 5d53a591 with the key-succession parent and the class v5 freeze igneum-pow (box4, 25s) | 2.0.2 pair gate on build-4, 00:26 UTC: kit 2: release-manifest-check green on 1176efb9 (node block kit2b-line 5d53a591); 77b5acc5..1176efb9 moves pool/ and packaging only; 5d53a591 = fix 2 + the wedge fix on kit 1's tree (cf32be79, the record-store fold, read red on the fresh-sync read at 00:55 and rides 2.0.3); kit 2 carries the roll and the entries on its own four-block rule-33 record, kit 1 re-cut 490ebcfe/5db25637 the clean fallback Every suite cell maps its cases partially, so the green suites read NOT RUN in progress with their logs as evidence, never PASS by inference. Kit 2's rule-33 record (four blocks) lands with the entries; INT-07's FAIL on 94e4c6ba stands until then." + "at": "2026-10-09T04:26:48.369Z", + "note": "the 5090, the 4090 and the 3090 PASS on all three phases of the frozen object across the day switch; Metal and OpenCL the morning's; CPU reference cross-checked against the hash lane's P01 file; in progress | Context 01 at build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json (sha256 d2540e02...). CPU reference: igneum-pow hash-bound (class-v6 1a938abe4, binary 0d3f3fca...) on build-4, 21:46 to 21:55 UK, ref-D c49fed11... over [0, 1572864), ref-D1 5ae1220d... over [1572864, 3145728); ref-D's first million lines equal the hash lane's P01 reference /srv/artefacts/packs/p01-vectors/hl-v6-all.txt byte for byte. The node1 state stream (block af89be5d...) was written on build-4 at 15:34 UK, before the 20:00 UK split; the cell carries the island label by main's 22:2x UK rule. CUDA: the 5090 PASS on all three phases (the gen 6 worker 804a6f7f, the driver at c8b65a26 or later; 23:16 to 23:20 UK: cuda-5090-1.json 1b36fb8a..., cuda-5090-2.json 70200bff... with the day switch at 1572864 driven through the worker's prepare and both boundary hashes equal to the reference (69abfadf816e767e before, 5b3fb17ad84744fd after), cuda-5090-3.json f2ecf5c7...; 1,048,576 agree each, 0 disagree, 0 missing); the 4090 PASS on all three phases (23:22 to 23:25 UK: cuda-4090-1.json cd0f5aee..., cuda-4090-2.json f864f950... with the day switch at 1572864, cuda-4090-3.json 126bd552...; 1,048,576 agree each, 0 disagree, 0 missing); The 3090 PASS on all three phases (05:07 to 05:10 UK; cuda-3090-{1,2,3}.json afa3d4a4..., 167875f9..., cc956dee...; 1,048,576 agree each, 0 disagree, 0 missing; the day switch at 1572864 with the same boundary hashes as every other reader): the fleet lane's first run on that pod at 00:35 UK never ran a phase (its verify rule demanded two reference files where a context carries one, then its put failed on the pod's flapping ssh; a runner fault, reported once), the rerun from 04:57 UK ran detached on the pod under a pid file with a collector pulling each JSON. So the CUDA cell reads three cards on this context. Metal and OpenCL are the morning's reads (08:30 UK). The case stays NOT RUN in progress until every reader has read (the coordinator's ruling 23:1x UK: a PASS by inference is forbidden).", + "network_label": "on an island, not a network" }, "in_progress_since": "2026-10-08T20:07:26.255Z", "approvals": { @@ -1446,8 +1447,8 @@ "cell": "harness:p01-vectors", "manifest_sha": "c30ab32c", "run_id": "same-work-20261008-01", - "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-3.json", - "in_progress": false, + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-3.json", + "in_progress": true, "coverage": "the million-vector campaign per backend per pack, bit-for-bit; PASS only when every supported backend reads a million vectors with zero disagreement; the malformed-input half is harness:parser-malformed", "release_identity": { "commit": "c30ab32c; the CPU reference class-v6 1a938abe4; the CUDA reader the kit's gen 6 worker 804a6f7f", @@ -1459,7 +1460,8 @@ }, "claim_impact": "POW-01's CUDA half across a dataset-day switch on the frozen object: CUDA against the CPU reference bit for bit on three ranges with the day switch at nonce 1572864. NOT same-work evidence: the node engine refuses this seed/stream pair (IgneumEngine::epoch_for's class v5 check wants the stream's block af89be5d... to equal the epoch seed edc4fa84...), so the node and pool readers cannot read it; the coordinator's ruling 22:0x UK keeps it as the CUDA and CPU-only row", "reviewer": "", - "at": "2026-10-08T22:37:47.237Z", + "at": "2026-10-09T04:26:48.369Z", + "note": "the 5090, the 4090 and the 3090 PASS on all three phases of the frozen object across the day switch; Metal and OpenCL the morning's; CPU reference cross-checked against the hash lane's P01 file; in progress | Context 01 at build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json (sha256 d2540e02...). CPU reference: igneum-pow hash-bound (class-v6 1a938abe4, binary 0d3f3fca...) on build-4, 21:46 to 21:55 UK, ref-D c49fed11... over [0, 1572864), ref-D1 5ae1220d... over [1572864, 3145728); ref-D's first million lines equal the hash lane's P01 reference /srv/artefacts/packs/p01-vectors/hl-v6-all.txt byte for byte. The node1 state stream (block af89be5d...) was written on build-4 at 15:34 UK, before the 20:00 UK split; the cell carries the island label by main's 22:2x UK rule. CUDA: the 5090 PASS on all three phases (the gen 6 worker 804a6f7f, the driver at c8b65a26 or later; 23:16 to 23:20 UK: cuda-5090-1.json 1b36fb8a..., cuda-5090-2.json 70200bff... with the day switch at 1572864 driven through the worker's prepare and both boundary hashes equal to the reference (69abfadf816e767e before, 5b3fb17ad84744fd after), cuda-5090-3.json f2ecf5c7...; 1,048,576 agree each, 0 disagree, 0 missing); the 4090 PASS on all three phases (23:22 to 23:25 UK: cuda-4090-1.json cd0f5aee..., cuda-4090-2.json f864f950... with the day switch at 1572864, cuda-4090-3.json 126bd552...; 1,048,576 agree each, 0 disagree, 0 missing); The 3090 PASS on all three phases (05:07 to 05:10 UK; cuda-3090-{1,2,3}.json afa3d4a4..., 167875f9..., cc956dee...; 1,048,576 agree each, 0 disagree, 0 missing; the day switch at 1572864 with the same boundary hashes as every other reader): the fleet lane's first run on that pod at 00:35 UK never ran a phase (its verify rule demanded two reference files where a context carries one, then its put failed on the pod's flapping ssh; a runner fault, reported once), the rerun from 04:57 UK ran detached on the pod under a pid file with a collector pulling each JSON. So the CUDA cell reads three cards on this context. Metal and OpenCL are the morning's reads (08:30 UK). The case stays NOT RUN in progress until every reader has read (the coordinator's ruling 23:1x UK: a PASS by inference is forbidden).", "network_label": "on an island, not a network" }, "suite:miner": { @@ -12278,8 +12280,8 @@ "ROT-02" ], "run_id": "same-work-20261008-03", - "evidence_path": "docs/design/class-v5-stored-state.md; build-1:/srv/artefacts/packs/packs-class-v6-20261008T202808Z.zip; build-1:/srv/builds/_log/v5-class/kits-20261008T202808Z/emu-check.log; build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/state-epoch2.igsd1; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/README.json", - "updated": "2026-10-08T23:52:12.680Z", + "evidence_path": "docs/design/class-v5-stored-state.md; build-1:/srv/artefacts/packs/packs-class-v6-20261008T202808Z.zip; build-1:/srv/builds/_log/v5-class/kits-20261008T202808Z/emu-check.log; build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/state-epoch2.igsd1; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/README.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-3.json", + "updated": "2026-10-09T04:26:48.468Z", "evidence_record": { "requirement_id": "R2-F03-R02", "decision": "NOT RUN", @@ -12287,8 +12289,8 @@ "cell": "harness:same-work", "manifest_sha": "c30ab32c", "run_id": "same-work-20261008-03", - "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/state-epoch2.igsd1; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/README.json", - "in_progress": false, + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/state-epoch2.igsd1; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/README.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-3.json", + "in_progress": true, "coverage": "partial until every reader has run: bit-for-bit agreement per nonce across node, CPU reference, CUDA, OpenCL, Metal and pool on one job context", "release_identity": { "commit": "c30ab32c (release-2.0.1; the class v5 freeze 1c420786, fingerprint cbc5bd0a); the CUDA reader the kit's gen 5 worker 9bfcf728", @@ -12300,7 +12302,8 @@ }, "claim_impact": "the same-work test's live-chain half (the coordinator's third row, 22:1x UK): the readers on the chain's own class v5 object at epoch 2 across the day switch 20734 to 20735; the node and pool readers run it on their release-line engines, the row those two can PASS without the class v6 branch", "reviewer": "", - "at": "2026-10-08T23:52:12.680Z", + "at": "2026-10-09T04:26:48.468Z", + "note": "node PASS x3 on the release line (kit 1's pair 94e4c6ba) and PASS x3 on the D1 branch (cc23f9bd); CUDA 5090, 4090 and 3090 PASS x3 on the gen 5 worker; CPU reference written; pool PASS x3 on the 2.0.2 kit as pinned (the share verifier's path); Metal and OpenCL the morning's (08:30 UK); in progress | On an island, not a network: build-1's seed is one of devnet-4's islands since the epoch-3 cut (21:34 UK), its stream pulled at 22:18 UK; epoch 2's object predates the split. Context 03 at build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json (sha256 53837a96...). The witness for the id: igneum-miner program-id reads class v5 attempt 0 id 81fbfa3aa413173f under era 7c36b833... and era 00..00 alike, from the miner at same-work-node cc23f9bd on build-2 (the class v5 engine path; the 2.0.1 pair's miner ef0f2ed8 on build-1 has no program-id subcommand), and the node lane's placed (c) read gave the same id; the hand's RPC port port 26870 answered no published state stream for the block at 22:18 UK. Packs v5-epoch2-d20734 and d20735 (both id 0x81fbfa3aa413173f) exported from release-2.0.1's igneum-pow on build-4 at 22:19 UK; the CPU references from release-2.0.1's igneum-pow (binary 7ba781db...) on build-4 under watchers, 22:20 to 00:21 UK (about 210 nonces a second a run; the one slip of the night, reported once), ref-D 873528b1... over [0, 1572864) and ref-D1 7f8e64ae... over [1572864, 3145728), read back on build-1 at 00:22 UK. NODE, two readers, both PASS on all three phases (00:27, 00:31, 00:35 UK; 1,048,576 agree each, 0 disagree, 0 missing; program id 0x81fbfa3aa413173f on every segment equal to the packs'; the day switch at 1572864 with both boundary hashes equal to the reference, 21488c8bc924c342 before and 08d55958274e224d after; genesis day 20734 installed, growth doublings 0): the RELEASE LINE, igneum-miner from kit 1's pair build-1:/srv/artefacts/200-94e4c6ba/node-lane (binary sha256 41cc0a82..., the recheck-vectors commits cherry-picked onto kit1-line 94e4c6ba = release-2.0.2-node without the v6 arm, the class v5 engine the kit ships), node-release-94e4c6ba/node-{1,2,3}.json (ac60f395..., 97f9758e..., bad64c5c...); and the D1 branch, same-work-node cc23f9bd (merged onto class-v6-node-review as fe0fdc59; binary 9d563c41...), node-same-work-node-cc23f9bd/node-{1,2,3}.json (7b1254f1..., a9da16a9..., 09ec9202...), the two readers' hashes equal to each other and to the reference on every nonce. CUDA, the kit's generation 5 worker 9bfcf728 on the driver at c8b65a26 or later: the 5090 PASS on all three phases (00:25 to 00:28 UK, cuda-5090-{1,2,3}.json 7040a0ab..., 86ff7bc4..., 3e3ec63f...) and the 4090 PASS on all three (00:30 to 00:33 UK, cuda-4090-{1,2,3}.json 1cb9efe5..., 6a52d5bb..., 1efffdde...), the day switch driven through the worker's prepare on each; The 3090 PASS on all three phases (05:11 to 05:14 UK; cuda-3090-{1,2,3}.json de68d6d3..., 7f66ee96..., 00840cc9...; 1,048,576 agree each, 0 disagree, 0 missing; the day switch at 1572864 with the same boundary hashes as every other reader): the fleet lane's first run on that pod at 00:35 UK never ran a phase (its verify rule demanded two reference files where a context carries one, then its put failed on the pod's flapping ssh; a runner fault, reported once), the rerun from 04:57 UK ran detached on the pod under a pid file with a collector pulling each JSON. So the CUDA cell reads three cards on this context. POOL, the 2.0.2 kit as pinned (release-2.0.2 77b5acc5's pool with node 7cfa422a, binary sha256 46e4c440..., on build-6, genesis day 20734 and dataset 2^28 from the pack): PASS on all three phases (00:28, 00:33, 00:38 UK; 1,048,576 agree each, 0 disagree, 0 missing; the boundary switched at 1572864 with one program id on both segments; 64, 128 and 64 of the sampled shares accepted through the share verifier), pool/kit-row/pool-{1,2,3}.json (c77d1276..., b30d4b65..., 957954e7...) with the README; every nonce of 3,145,728 agrees with the reference on the chain's own class v5 object through the kit as pinned. Metal and OpenCL are the morning's reads (08:30 UK). The rows stay NOT RUN in progress until every reader has read (the coordinator's ruling 23:1x UK).", "network_label": "on an island, not a network" }, "evidence_records": { @@ -12333,8 +12336,8 @@ "cell": "harness:same-work", "manifest_sha": "c30ab32c", "run_id": "same-work-20261008-03", - "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/state-epoch2.igsd1; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/README.json", - "in_progress": false, + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/state-epoch2.igsd1; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/README.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-3.json", + "in_progress": true, "coverage": "partial until every reader has run: bit-for-bit agreement per nonce across node, CPU reference, CUDA, OpenCL, Metal and pool on one job context", "release_identity": { "commit": "c30ab32c (release-2.0.1; the class v5 freeze 1c420786, fingerprint cbc5bd0a); the CUDA reader the kit's gen 5 worker 9bfcf728", @@ -12346,7 +12349,8 @@ }, "claim_impact": "the same-work test's live-chain half (the coordinator's third row, 22:1x UK): the readers on the chain's own class v5 object at epoch 2 across the day switch 20734 to 20735; the node and pool readers run it on their release-line engines, the row those two can PASS without the class v6 branch", "reviewer": "", - "at": "2026-10-08T23:52:12.680Z", + "at": "2026-10-09T04:26:48.468Z", + "note": "node PASS x3 on the release line (kit 1's pair 94e4c6ba) and PASS x3 on the D1 branch (cc23f9bd); CUDA 5090, 4090 and 3090 PASS x3 on the gen 5 worker; CPU reference written; pool PASS x3 on the 2.0.2 kit as pinned (the share verifier's path); Metal and OpenCL the morning's (08:30 UK); in progress | On an island, not a network: build-1's seed is one of devnet-4's islands since the epoch-3 cut (21:34 UK), its stream pulled at 22:18 UK; epoch 2's object predates the split. Context 03 at build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json (sha256 53837a96...). The witness for the id: igneum-miner program-id reads class v5 attempt 0 id 81fbfa3aa413173f under era 7c36b833... and era 00..00 alike, from the miner at same-work-node cc23f9bd on build-2 (the class v5 engine path; the 2.0.1 pair's miner ef0f2ed8 on build-1 has no program-id subcommand), and the node lane's placed (c) read gave the same id; the hand's RPC port port 26870 answered no published state stream for the block at 22:18 UK. Packs v5-epoch2-d20734 and d20735 (both id 0x81fbfa3aa413173f) exported from release-2.0.1's igneum-pow on build-4 at 22:19 UK; the CPU references from release-2.0.1's igneum-pow (binary 7ba781db...) on build-4 under watchers, 22:20 to 00:21 UK (about 210 nonces a second a run; the one slip of the night, reported once), ref-D 873528b1... over [0, 1572864) and ref-D1 7f8e64ae... over [1572864, 3145728), read back on build-1 at 00:22 UK. NODE, two readers, both PASS on all three phases (00:27, 00:31, 00:35 UK; 1,048,576 agree each, 0 disagree, 0 missing; program id 0x81fbfa3aa413173f on every segment equal to the packs'; the day switch at 1572864 with both boundary hashes equal to the reference, 21488c8bc924c342 before and 08d55958274e224d after; genesis day 20734 installed, growth doublings 0): the RELEASE LINE, igneum-miner from kit 1's pair build-1:/srv/artefacts/200-94e4c6ba/node-lane (binary sha256 41cc0a82..., the recheck-vectors commits cherry-picked onto kit1-line 94e4c6ba = release-2.0.2-node without the v6 arm, the class v5 engine the kit ships), node-release-94e4c6ba/node-{1,2,3}.json (ac60f395..., 97f9758e..., bad64c5c...); and the D1 branch, same-work-node cc23f9bd (merged onto class-v6-node-review as fe0fdc59; binary 9d563c41...), node-same-work-node-cc23f9bd/node-{1,2,3}.json (7b1254f1..., a9da16a9..., 09ec9202...), the two readers' hashes equal to each other and to the reference on every nonce. CUDA, the kit's generation 5 worker 9bfcf728 on the driver at c8b65a26 or later: the 5090 PASS on all three phases (00:25 to 00:28 UK, cuda-5090-{1,2,3}.json 7040a0ab..., 86ff7bc4..., 3e3ec63f...) and the 4090 PASS on all three (00:30 to 00:33 UK, cuda-4090-{1,2,3}.json 1cb9efe5..., 6a52d5bb..., 1efffdde...), the day switch driven through the worker's prepare on each; The 3090 PASS on all three phases (05:11 to 05:14 UK; cuda-3090-{1,2,3}.json de68d6d3..., 7f66ee96..., 00840cc9...; 1,048,576 agree each, 0 disagree, 0 missing; the day switch at 1572864 with the same boundary hashes as every other reader): the fleet lane's first run on that pod at 00:35 UK never ran a phase (its verify rule demanded two reference files where a context carries one, then its put failed on the pod's flapping ssh; a runner fault, reported once), the rerun from 04:57 UK ran detached on the pod under a pid file with a collector pulling each JSON. So the CUDA cell reads three cards on this context. POOL, the 2.0.2 kit as pinned (release-2.0.2 77b5acc5's pool with node 7cfa422a, binary sha256 46e4c440..., on build-6, genesis day 20734 and dataset 2^28 from the pack): PASS on all three phases (00:28, 00:33, 00:38 UK; 1,048,576 agree each, 0 disagree, 0 missing; the boundary switched at 1572864 with one program id on both segments; 64, 128 and 64 of the sampled shares accepted through the share verifier), pool/kit-row/pool-{1,2,3}.json (c77d1276..., b30d4b65..., 957954e7...) with the README; every nonce of 3,145,728 agrees with the reference on the chain's own class v5 object through the kit as pinned. Metal and OpenCL are the morning's reads (08:30 UK). The rows stay NOT RUN in progress until every reader has read (the coordinator's ruling 23:1x UK).", "network_label": "on an island, not a network" } }, @@ -12390,8 +12394,8 @@ "ROT-02" ], "run_id": "same-work-20261008-03", - "evidence_path": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json", - "updated": "2026-10-08T23:52:12.680Z", + "evidence_path": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-3.json", + "updated": "2026-10-09T04:26:48.468Z", "evidence_record": { "requirement_id": "R2-F03-R03", "decision": "NOT RUN", @@ -12399,8 +12403,8 @@ "cell": "harness:same-work", "manifest_sha": "c30ab32c", "run_id": "same-work-20261008-03", - "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json", - "in_progress": false, + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-3.json", + "in_progress": true, "coverage": "partial: the day-boundary transition in phase 2 (the one transition every live network crosses hourly at 60x); a class rotation is the same test on a research object until a v6 floor is set", "release_identity": { "commit": "c30ab32c (release-2.0.1; the class v5 freeze 1c420786, fingerprint cbc5bd0a); the CUDA reader the kit's gen 5 worker 9bfcf728", @@ -12412,7 +12416,8 @@ }, "claim_impact": "the same-work test's live-chain half (the coordinator's third row, 22:1x UK): the readers on the chain's own class v5 object at epoch 2 across the day switch 20734 to 20735; the node and pool readers run it on their release-line engines, the row those two can PASS without the class v6 branch", "reviewer": "", - "at": "2026-10-08T23:52:12.680Z", + "at": "2026-10-09T04:26:48.468Z", + "note": "the day switch 20734 to 20735 at nonce 1572864 on the live object: both node readers, the kit's pool, the 5090 and the 4090 each switched program and dataset there with the program id unchanged and both boundary hashes equal to the reference; OpenCL and Metal pending; in progress | On an island, not a network: build-1's seed is one of devnet-4's islands since the epoch-3 cut (21:34 UK), its stream pulled at 22:18 UK; epoch 2's object predates the split. Context 03 at build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json (sha256 53837a96...). The witness for the id: igneum-miner program-id reads class v5 attempt 0 id 81fbfa3aa413173f under era 7c36b833... and era 00..00 alike, from the miner at same-work-node cc23f9bd on build-2 (the class v5 engine path; the 2.0.1 pair's miner ef0f2ed8 on build-1 has no program-id subcommand), and the node lane's placed (c) read gave the same id; the hand's RPC port port 26870 answered no published state stream for the block at 22:18 UK. Packs v5-epoch2-d20734 and d20735 (both id 0x81fbfa3aa413173f) exported from release-2.0.1's igneum-pow on build-4 at 22:19 UK; the CPU references from release-2.0.1's igneum-pow (binary 7ba781db...) on build-4 under watchers, 22:20 to 00:21 UK (about 210 nonces a second a run; the one slip of the night, reported once), ref-D 873528b1... over [0, 1572864) and ref-D1 7f8e64ae... over [1572864, 3145728), read back on build-1 at 00:22 UK. NODE, two readers, both PASS on all three phases (00:27, 00:31, 00:35 UK; 1,048,576 agree each, 0 disagree, 0 missing; program id 0x81fbfa3aa413173f on every segment equal to the packs'; the day switch at 1572864 with both boundary hashes equal to the reference, 21488c8bc924c342 before and 08d55958274e224d after; genesis day 20734 installed, growth doublings 0): the RELEASE LINE, igneum-miner from kit 1's pair build-1:/srv/artefacts/200-94e4c6ba/node-lane (binary sha256 41cc0a82..., the recheck-vectors commits cherry-picked onto kit1-line 94e4c6ba = release-2.0.2-node without the v6 arm, the class v5 engine the kit ships), node-release-94e4c6ba/node-{1,2,3}.json (ac60f395..., 97f9758e..., bad64c5c...); and the D1 branch, same-work-node cc23f9bd (merged onto class-v6-node-review as fe0fdc59; binary 9d563c41...), node-same-work-node-cc23f9bd/node-{1,2,3}.json (7b1254f1..., a9da16a9..., 09ec9202...), the two readers' hashes equal to each other and to the reference on every nonce. CUDA, the kit's generation 5 worker 9bfcf728 on the driver at c8b65a26 or later: the 5090 PASS on all three phases (00:25 to 00:28 UK, cuda-5090-{1,2,3}.json 7040a0ab..., 86ff7bc4..., 3e3ec63f...) and the 4090 PASS on all three (00:30 to 00:33 UK, cuda-4090-{1,2,3}.json 1cb9efe5..., 6a52d5bb..., 1efffdde...), the day switch driven through the worker's prepare on each; The 3090 PASS on all three phases (05:11 to 05:14 UK; cuda-3090-{1,2,3}.json de68d6d3..., 7f66ee96..., 00840cc9...; 1,048,576 agree each, 0 disagree, 0 missing; the day switch at 1572864 with the same boundary hashes as every other reader): the fleet lane's first run on that pod at 00:35 UK never ran a phase (its verify rule demanded two reference files where a context carries one, then its put failed on the pod's flapping ssh; a runner fault, reported once), the rerun from 04:57 UK ran detached on the pod under a pid file with a collector pulling each JSON. So the CUDA cell reads three cards on this context. POOL, the 2.0.2 kit as pinned (release-2.0.2 77b5acc5's pool with node 7cfa422a, binary sha256 46e4c440..., on build-6, genesis day 20734 and dataset 2^28 from the pack): PASS on all three phases (00:28, 00:33, 00:38 UK; 1,048,576 agree each, 0 disagree, 0 missing; the boundary switched at 1572864 with one program id on both segments; 64, 128 and 64 of the sampled shares accepted through the share verifier), pool/kit-row/pool-{1,2,3}.json (c77d1276..., b30d4b65..., 957954e7...) with the README; every nonce of 3,145,728 agrees with the reference on the chain's own class v5 object through the kit as pinned. Metal and OpenCL are the morning's reads (08:30 UK). The rows stay NOT RUN in progress until every reader has read (the coordinator's ruling 23:1x UK).", "network_label": "on an island, not a network" }, "evidence_records": { @@ -12423,8 +12428,8 @@ "cell": "harness:same-work", "manifest_sha": "c30ab32c", "run_id": "same-work-20261008-03", - "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json", - "in_progress": false, + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-3.json", + "in_progress": true, "coverage": "partial: the day-boundary transition in phase 2 (the one transition every live network crosses hourly at 60x); a class rotation is the same test on a research object until a v6 floor is set", "release_identity": { "commit": "c30ab32c (release-2.0.1; the class v5 freeze 1c420786, fingerprint cbc5bd0a); the CUDA reader the kit's gen 5 worker 9bfcf728", @@ -12436,7 +12441,8 @@ }, "claim_impact": "the same-work test's live-chain half (the coordinator's third row, 22:1x UK): the readers on the chain's own class v5 object at epoch 2 across the day switch 20734 to 20735; the node and pool readers run it on their release-line engines, the row those two can PASS without the class v6 branch", "reviewer": "", - "at": "2026-10-08T23:52:12.680Z", + "at": "2026-10-09T04:26:48.468Z", + "note": "the day switch 20734 to 20735 at nonce 1572864 on the live object: both node readers, the kit's pool, the 5090 and the 4090 each switched program and dataset there with the program id unchanged and both boundary hashes equal to the reference; OpenCL and Metal pending; in progress | On an island, not a network: build-1's seed is one of devnet-4's islands since the epoch-3 cut (21:34 UK), its stream pulled at 22:18 UK; epoch 2's object predates the split. Context 03 at build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json (sha256 53837a96...). The witness for the id: igneum-miner program-id reads class v5 attempt 0 id 81fbfa3aa413173f under era 7c36b833... and era 00..00 alike, from the miner at same-work-node cc23f9bd on build-2 (the class v5 engine path; the 2.0.1 pair's miner ef0f2ed8 on build-1 has no program-id subcommand), and the node lane's placed (c) read gave the same id; the hand's RPC port port 26870 answered no published state stream for the block at 22:18 UK. Packs v5-epoch2-d20734 and d20735 (both id 0x81fbfa3aa413173f) exported from release-2.0.1's igneum-pow on build-4 at 22:19 UK; the CPU references from release-2.0.1's igneum-pow (binary 7ba781db...) on build-4 under watchers, 22:20 to 00:21 UK (about 210 nonces a second a run; the one slip of the night, reported once), ref-D 873528b1... over [0, 1572864) and ref-D1 7f8e64ae... over [1572864, 3145728), read back on build-1 at 00:22 UK. NODE, two readers, both PASS on all three phases (00:27, 00:31, 00:35 UK; 1,048,576 agree each, 0 disagree, 0 missing; program id 0x81fbfa3aa413173f on every segment equal to the packs'; the day switch at 1572864 with both boundary hashes equal to the reference, 21488c8bc924c342 before and 08d55958274e224d after; genesis day 20734 installed, growth doublings 0): the RELEASE LINE, igneum-miner from kit 1's pair build-1:/srv/artefacts/200-94e4c6ba/node-lane (binary sha256 41cc0a82..., the recheck-vectors commits cherry-picked onto kit1-line 94e4c6ba = release-2.0.2-node without the v6 arm, the class v5 engine the kit ships), node-release-94e4c6ba/node-{1,2,3}.json (ac60f395..., 97f9758e..., bad64c5c...); and the D1 branch, same-work-node cc23f9bd (merged onto class-v6-node-review as fe0fdc59; binary 9d563c41...), node-same-work-node-cc23f9bd/node-{1,2,3}.json (7b1254f1..., a9da16a9..., 09ec9202...), the two readers' hashes equal to each other and to the reference on every nonce. CUDA, the kit's generation 5 worker 9bfcf728 on the driver at c8b65a26 or later: the 5090 PASS on all three phases (00:25 to 00:28 UK, cuda-5090-{1,2,3}.json 7040a0ab..., 86ff7bc4..., 3e3ec63f...) and the 4090 PASS on all three (00:30 to 00:33 UK, cuda-4090-{1,2,3}.json 1cb9efe5..., 6a52d5bb..., 1efffdde...), the day switch driven through the worker's prepare on each; The 3090 PASS on all three phases (05:11 to 05:14 UK; cuda-3090-{1,2,3}.json de68d6d3..., 7f66ee96..., 00840cc9...; 1,048,576 agree each, 0 disagree, 0 missing; the day switch at 1572864 with the same boundary hashes as every other reader): the fleet lane's first run on that pod at 00:35 UK never ran a phase (its verify rule demanded two reference files where a context carries one, then its put failed on the pod's flapping ssh; a runner fault, reported once), the rerun from 04:57 UK ran detached on the pod under a pid file with a collector pulling each JSON. So the CUDA cell reads three cards on this context. POOL, the 2.0.2 kit as pinned (release-2.0.2 77b5acc5's pool with node 7cfa422a, binary sha256 46e4c440..., on build-6, genesis day 20734 and dataset 2^28 from the pack): PASS on all three phases (00:28, 00:33, 00:38 UK; 1,048,576 agree each, 0 disagree, 0 missing; the boundary switched at 1572864 with one program id on both segments; 64, 128 and 64 of the sampled shares accepted through the share verifier), pool/kit-row/pool-{1,2,3}.json (c77d1276..., b30d4b65..., 957954e7...) with the README; every nonce of 3,145,728 agrees with the reference on the chain's own class v5 object through the kit as pinned. Metal and OpenCL are the morning's reads (08:30 UK). The rows stay NOT RUN in progress until every reader has read (the coordinator's ruling 23:1x UK).", "network_label": "on an island, not a network" } }, @@ -12445,7 +12451,8 @@ "implementation_complete": null, "evidence_reproduced": null, "claim_authorised": null - } + }, + "in_progress_since": "2026-10-09T04:26:48.418Z" }, { "id": "R2-F04-R01", diff --git a/tools/ci/batches/same-work-20261008-01.json b/tools/ci/batches/same-work-20261008-01.json index 82a5ac330..6a4f20d1a 100644 --- a/tools/ci/batches/same-work-20261008-01.json +++ b/tools/ci/batches/same-work-20261008-01.json @@ -19,7 +19,7 @@ }, "claim_impact": "POW-01's CUDA half across a dataset-day switch on the frozen object: CUDA against the CPU reference bit for bit on three ranges with the day switch at nonce 1572864. NOT same-work evidence: the node engine refuses this seed/stream pair (IgneumEngine::epoch_for's class v5 check wants the stream's block af89be5d... to equal the epoch seed edc4fa84...), so the node and pool readers cannot read it; the coordinator's ruling 22:0x UK keeps it as the CUDA and CPU-only row", "network_label": "on an island, not a network", - "note": "Context 01 at build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json (sha256 d2540e02...). CPU reference: igneum-pow hash-bound (class-v6 1a938abe4, binary 0d3f3fca...) on build-4, 21:46 to 21:55 UK, ref-D c49fed11... over [0, 1572864), ref-D1 5ae1220d... over [1572864, 3145728); ref-D's first million lines equal the hash lane's P01 reference /srv/artefacts/packs/p01-vectors/hl-v6-all.txt byte for byte. The node1 state stream (block af89be5d...) was written on build-4 at 15:34 UK, before the 20:00 UK split; the cell carries the island label by main's 22:2x UK rule. CUDA: the 5090 PASS on all three phases (the gen 6 worker 804a6f7f, the driver at c8b65a26 or later; 23:16 to 23:20 UK: cuda-5090-1.json 1b36fb8a..., cuda-5090-2.json 70200bff... with the day switch at 1572864 driven through the worker's prepare and both boundary hashes equal to the reference (69abfadf816e767e before, 5b3fb17ad84744fd after), cuda-5090-3.json f2ecf5c7...; 1,048,576 agree each, 0 disagree, 0 missing); the 4090 PASS on all three phases (23:22 to 23:25 UK: cuda-4090-1.json cd0f5aee..., cuda-4090-2.json f864f950... with the day switch at 1572864, cuda-4090-3.json 126bd552...; 1,048,576 agree each, 0 disagree, 0 missing); the 3090 pending (its context copy to that pod missed the packs tree at 22:5x UK, a copy fault on the fleet lane's runner, not a reader result; the runner now verifies program.json in both packs and both references on the pod before phase 1); Metal and OpenCL are the morning's reads (08:30 UK). The case stays NOT RUN in progress until every reader has read (the coordinator's ruling 23:1x UK: a PASS by inference is forbidden).", + "note": "Context 01 at build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json (sha256 d2540e02...). CPU reference: igneum-pow hash-bound (class-v6 1a938abe4, binary 0d3f3fca...) on build-4, 21:46 to 21:55 UK, ref-D c49fed11... over [0, 1572864), ref-D1 5ae1220d... over [1572864, 3145728); ref-D's first million lines equal the hash lane's P01 reference /srv/artefacts/packs/p01-vectors/hl-v6-all.txt byte for byte. The node1 state stream (block af89be5d...) was written on build-4 at 15:34 UK, before the 20:00 UK split; the cell carries the island label by main's 22:2x UK rule. CUDA: the 5090 PASS on all three phases (the gen 6 worker 804a6f7f, the driver at c8b65a26 or later; 23:16 to 23:20 UK: cuda-5090-1.json 1b36fb8a..., cuda-5090-2.json 70200bff... with the day switch at 1572864 driven through the worker's prepare and both boundary hashes equal to the reference (69abfadf816e767e before, 5b3fb17ad84744fd after), cuda-5090-3.json f2ecf5c7...; 1,048,576 agree each, 0 disagree, 0 missing); the 4090 PASS on all three phases (23:22 to 23:25 UK: cuda-4090-1.json cd0f5aee..., cuda-4090-2.json f864f950... with the day switch at 1572864, cuda-4090-3.json 126bd552...; 1,048,576 agree each, 0 disagree, 0 missing); The 3090 PASS on all three phases (05:07 to 05:10 UK; cuda-3090-{1,2,3}.json afa3d4a4..., 167875f9..., cc956dee...; 1,048,576 agree each, 0 disagree, 0 missing; the day switch at 1572864 with the same boundary hashes as every other reader): the fleet lane's first run on that pod at 00:35 UK never ran a phase (its verify rule demanded two reference files where a context carries one, then its put failed on the pod's flapping ssh; a runner fault, reported once), the rerun from 04:57 UK ran detached on the pod under a pid file with a collector pulling each JSON. So the CUDA cell reads three cards on this context. Metal and OpenCL are the morning's reads (08:30 UK). The case stays NOT RUN in progress until every reader has read (the coordinator's ruling 23:1x UK: a PASS by inference is forbidden).", "cells": [ { "cell": "harness:p01-vectors", @@ -28,8 +28,8 @@ ], "status": "NOT RUN", "method": "GPU", - "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-3.json", - "note": "the 5090 and the 4090 PASS on all three phases of the frozen object across the day switch; the 3090 pending; Metal and OpenCL the morning's; CPU reference cross-checked against the hash lane's P01 file; in progress", + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-01/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-01/cuda-3090-3.json", + "note": "the 5090, the 4090 and the 3090 PASS on all three phases of the frozen object across the day switch; Metal and OpenCL the morning's; CPU reference cross-checked against the hash lane's P01 file; in progress", "network_label": "on an island, not a network", "in_progress": true } diff --git a/tools/ci/batches/same-work-20261008-02.json b/tools/ci/batches/same-work-20261008-02.json index b3c3f94bb..80774d287 100644 --- a/tools/ci/batches/same-work-20261008-02.json +++ b/tools/ci/batches/same-work-20261008-02.json @@ -20,7 +20,7 @@ }, "claim_impact": "R2-F03-R02 and R03 on one job context (the chain-seed class v6 object) across the readers that can run it tonight: node, CPU reference, CUDA and pool on three phases with the dataset-day switch at nonce 1572864; PASS only when every reader agrees bit for bit on every nonce; Metal and OpenCL are the morning's", "network_label": "on an island, not a network", - "note": "The same-work run (the coordinator's ruling 22:0x UK: six readers on one context; the frozen-object context 01 is the CUDA and CPU-only row). The node1 state stream (block af89be5d..., written on build-4 at 15:34 UK, before the 20:00 UK split) keys every item; every cell carries the island label by main's 22:2x UK rule. Day D = 20730 (bytes ...2ffa50), D+1 = 20731 (...2ffb50); phase 2 switches at 1572864 (the midpoint, 32-aligned); genesis day index 20730 and dataset 2^28 words for the engine's growth rule (doublings 0 on both days). CPU reference: igneum-pow hash-bound (class-v6 1a938abe4, binary 0d3f3fca...) on build-4, 21:48 to 21:57 UK, ref-D 7c7be9b5... over [0, 1572864), ref-D1 cad7849f... over [1572864, 3145728); the hash lane's independent million-nonce reference hl-v6-all-cs.txt (b77c61d8..., 22:03 UK) equals ref-D's first million lines byte for byte. Node reader: PASS on all three phases (22:17, 22:21, 22:25 UK; 1,048,576 agree each, 0 disagree, 0 missing; the boundary switched: d28fcfd70ba70a64 before, 45905ff3d608dc12 after, both equal to the reference; the program id unchanged across the day). Reader faults recorded, not disagreements (the coordinator's ruling 22:1x UK): the node reader's first phase 1 at 22:10 UK read FAIL on every nonce with the program id equal because the standalone engine held the genesis day at 0 and the growth rule doubled the cache fourteen times (fixed at cc23f9bd: install_pow_genesis from the context and the pack, the live miner's own call); the pool lane's D1-pairing run at 22:17 UK hit the same class (its nonce 0 and 1 hashes equal the pre-fix node hashes byte for byte), fix sent 22:18 UK; the rerun on the fixed reader (pool-recheck-202 909ae65b, binary 10425cc9... on build-4, the pack geometry installed) reads PASS on every phase at 22:32, 22:38 and 22:44 UK: 1,048,576 agree each, 0 disagree, 0 missing, the boundary switched at 1572864 with one program id on both segments, 64, 128 and 64 of the sampled shares accepted through verify::check; every nonce of 3,145,728 agrees with the reference (pool/d1-pairing/pool-{1,2,3}.json, sha256 d6d6105f..., 611c7e75..., 44820ff8..., read back on build-1 at 22:45 UK; the README keeps the first run's FAIL and its cause). The pool on the 2.0.2 kit as pinned (release-2.0.2 00c7e1db, node 7cfa422a) is BLOCKED: its node's ProgramClass enum ends at V5, so a class=v6 job line is unnameable (pool/kit-row/pool-{1,2,3}.json carry that refusal in the driver's shape). CUDA (the gen 6 worker 804a6f7f on the pods, the driver from same-work-harness 76cb1f7c6 and later): the 5090 phase 1 PASS (22:53 UK, cuda-5090-1.json 654ddddb...) and phase 2 PASS (22:55 UK, cuda-5090-2.json 898b1cb3...: the day switch at 1572864 driven through the worker's prepare, both boundary hashes equal to the reference); the 4090 phase 1 PASS (22:57 UK, cuda-4090-1.json c91d9e16...) and phase 2 PASS (22:59 UK, cuda-4090-2.json 82a2f5e6...); phase 3 PASS on both after the driver fix (23:09 UK cuda-5090-3.json ee42763d..., 23:11 UK cuda-4090-3.json 91924119...: day D+1 prepared through the worker, 1,048,576 agree each); the 3090's three phases pending (its context copy to that pod missed the packs tree at 22:5x UK, a copy fault on the fleet lane's runner, not a reader result; its 02 pass is queued behind the main pass with a per-pack copy verified on the pod before phase 1). Two driver faults on the way, neither a disagreement: the driver's silent wait for the prepared line deadlocked against a worker that loads a finished prepare only between stdin lines (22:26 to 22:47 UK on the 5090; fixed at 76cb1f7c6: day D jobs keep flowing and the worker is polled with a line a second), and phase 3 on a worker started on the day D pack had every day D+1 job refused as a day-seed mismatch (22:56 UK, cuda-5090-3.json and cuda-4090-3.json: missing 1,048,576, the program id equal; fixed in the commit after b464021f8: the pack the worker holds is read from its ready line and every other segment is prepared); the phase 3 reruns on the fixed driver read PASS. Rule 24 on same-work-node cc23f9bd on build-2: crate check rc 0, igneum-miner suite 30 passed 0 failed.", + "note": "The same-work run (the coordinator's ruling 22:0x UK: six readers on one context; the frozen-object context 01 is the CUDA and CPU-only row). The node1 state stream (block af89be5d..., written on build-4 at 15:34 UK, before the 20:00 UK split) keys every item; every cell carries the island label by main's 22:2x UK rule. Day D = 20730 (bytes ...2ffa50), D+1 = 20731 (...2ffb50); phase 2 switches at 1572864 (the midpoint, 32-aligned); genesis day index 20730 and dataset 2^28 words for the engine's growth rule (doublings 0 on both days). CPU reference: igneum-pow hash-bound (class-v6 1a938abe4, binary 0d3f3fca...) on build-4, 21:48 to 21:57 UK, ref-D 7c7be9b5... over [0, 1572864), ref-D1 cad7849f... over [1572864, 3145728); the hash lane's independent million-nonce reference hl-v6-all-cs.txt (b77c61d8..., 22:03 UK) equals ref-D's first million lines byte for byte. Node reader: PASS on all three phases (22:17, 22:21, 22:25 UK; 1,048,576 agree each, 0 disagree, 0 missing; the boundary switched: d28fcfd70ba70a64 before, 45905ff3d608dc12 after, both equal to the reference; the program id unchanged across the day). Reader faults recorded, not disagreements (the coordinator's ruling 22:1x UK): the node reader's first phase 1 at 22:10 UK read FAIL on every nonce with the program id equal because the standalone engine held the genesis day at 0 and the growth rule doubled the cache fourteen times (fixed at cc23f9bd: install_pow_genesis from the context and the pack, the live miner's own call); the pool lane's D1-pairing run at 22:17 UK hit the same class (its nonce 0 and 1 hashes equal the pre-fix node hashes byte for byte), fix sent 22:18 UK; the rerun on the fixed reader (pool-recheck-202 909ae65b, binary 10425cc9... on build-4, the pack geometry installed) reads PASS on every phase at 22:32, 22:38 and 22:44 UK: 1,048,576 agree each, 0 disagree, 0 missing, the boundary switched at 1572864 with one program id on both segments, 64, 128 and 64 of the sampled shares accepted through verify::check; every nonce of 3,145,728 agrees with the reference (pool/d1-pairing/pool-{1,2,3}.json, sha256 d6d6105f..., 611c7e75..., 44820ff8..., read back on build-1 at 22:45 UK; the README keeps the first run's FAIL and its cause). The pool on the 2.0.2 kit as pinned (release-2.0.2 00c7e1db, node 7cfa422a) is BLOCKED: its node's ProgramClass enum ends at V5, so a class=v6 job line is unnameable (pool/kit-row/pool-{1,2,3}.json carry that refusal in the driver's shape). CUDA (the gen 6 worker 804a6f7f on the pods, the driver from same-work-harness 76cb1f7c6 and later): the 5090 phase 1 PASS (22:53 UK, cuda-5090-1.json 654ddddb...) and phase 2 PASS (22:55 UK, cuda-5090-2.json 898b1cb3...: the day switch at 1572864 driven through the worker's prepare, both boundary hashes equal to the reference); the 4090 phase 1 PASS (22:57 UK, cuda-4090-1.json c91d9e16...) and phase 2 PASS (22:59 UK, cuda-4090-2.json 82a2f5e6...); phase 3 PASS on both after the driver fix (23:09 UK cuda-5090-3.json ee42763d..., 23:11 UK cuda-4090-3.json 91924119...: day D+1 prepared through the worker, 1,048,576 agree each); The 3090 PASS on all three phases (05:00 to 05:06 UK; cuda-3090-{1,2,3}.json eef41f88..., 9344eec0..., c3ea7029...; 1,048,576 agree each, 0 disagree, 0 missing; the day switch at 1572864 with the same boundary hashes as every other reader): the fleet lane's first run on that pod at 00:35 UK never ran a phase (its verify rule demanded two reference files where a context carries one, then its put failed on the pod's flapping ssh; a runner fault, reported once), the rerun from 04:57 UK ran detached on the pod under a pid file with a collector pulling each JSON. So the CUDA cell reads three cards on this context. Two driver faults on the way, neither a disagreement: the driver's silent wait for the prepared line deadlocked against a worker that loads a finished prepare only between stdin lines (22:26 to 22:47 UK on the 5090; fixed at 76cb1f7c6: day D jobs keep flowing and the worker is polled with a line a second), and phase 3 on a worker started on the day D pack had every day D+1 job refused as a day-seed mismatch (22:56 UK, cuda-5090-3.json and cuda-4090-3.json: missing 1,048,576, the program id equal; fixed in the commit after b464021f8: the pack the worker holds is read from its ready line and every other segment is prepared); the phase 3 reruns on the fixed driver read PASS. Rule 24 on same-work-node cc23f9bd on build-2: crate check rc 0, igneum-miner suite 30 passed 0 failed. OPENCL, the first rig's RX 7600 (gfx1102) on the generation 6 OpenCL worker (247a8467..., cross-built from class-v6 755c2dbcf on build-1; the kit's zips 3 and 4 carry generator-6 OpenCL exes, 45a77647... and be169903...): take 1 at 03:46 UK ran zip 2's OpenCL exe left in a stale fetch directory on the rig and refused generator 6 (a fetch fault, no reader result); take 2 at 04:54 UK read phase 1 PASS, 1,048,576 of 1,048,576 with the derived digest fd6350b7... equal to ref-D's first 2^20 lines, and the day-D half of phase 2 right (lines 1048576 to 1572863, tail d28fcfd70ba70a64 = ref-D's last line), then the day D+1 jobs were answered day-seed mismatch because the rig's job piped every line at once before the worker's prepared line (the same class as the driver's 22:26 UK deadlock, a job fault; evidence opencl-rx7600-take2.report.txt); take 3 drives the worker line by line and waits for prepared, its closing report lands as opencl-rx7600.report.txt (the hash lane's watch). METAL: 08:30 UK on the project's macOS machine (the fleet lane's armed cell; the serve worker 5aeddcba... read the day switch there at 22:49 UK on 64-nonce jobs: nonces 0 to 2 and 1572864 to 1572865 equal to the references).", "cells": [ { "cell": "harness:same-work", @@ -29,8 +29,8 @@ ], "status": "NOT RUN", "method": "GPU", - "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node.log; build-1:/srv/artefacts/tas/same-work-20261008-02/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-02/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-02/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/kit-row/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/kit-row/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/d1-pairing/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/d1-pairing/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/d1-pairing/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/d1-pairing/README.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-5090-1.json; docs/design/class-v5-stored-state.md; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-4090-3.json", - "note": "node PASS x3; pool D1 pairing PASS x3 (every nonce, the share verifier's own path, the day switch seen); CPU reference written and cross-checked twice; CUDA 5090 and 4090 PASS on all three phases (the day switch seen by a GPU reader), the 3090 pending; pool kit row BLOCKED by its V5 pins; OpenCL queued on the project's first rig (08:30 UK at the latest, the job run-ca3-pc1-samework-opencl-7600-20261008, its report staged as opencl-rx7600.report.txt); Metal NOT RUN, 08:30 UK on the project's macOS machine if a serve worker exists by 08:00 UK; the case stays NOT RUN in progress until every reader has read; the design page docs/design/class-v5-stored-state.md (the node lane's fix-2 amendment, class-v5-foreign-capture-doc 4ec69ca4) moves with this row", + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node.log; build-1:/srv/artefacts/tas/same-work-20261008-02/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-02/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-02/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/kit-row/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/kit-row/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/d1-pairing/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/d1-pairing/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/d1-pairing/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/d1-pairing/README.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-5090-1.json; docs/design/class-v5-stored-state.md; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-3090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/opencl-rx7600-take2.report.txt", + "note": "node PASS x3; pool D1 pairing PASS x3 (every nonce, the share verifier's own path, the day switch seen); CPU reference written and cross-checked twice; CUDA 5090, 4090 and 3090 PASS on all three phases (the day switch seen by a GPU reader), pool kit row BLOCKED by its V5 pins; OpenCL on the first rig: phase 1 PASS and the day-D half of phase 2 PASS on take 2 (the gen 6 worker), the day D+1 half and phase 3 on take 3 (its report as opencl-rx7600.report.txt); Metal NOT RUN, 08:30 UK on the project's macOS machine if a serve worker exists by 08:00 UK; the case stays NOT RUN in progress until every reader has read; the design page docs/design/class-v5-stored-state.md (the node lane's fix-2 amendment, class-v5-foreign-capture-doc 4ec69ca4) moves with this row", "network_label": "on an island, not a network", "in_progress": true }, @@ -41,8 +41,8 @@ ], "status": "NOT RUN", "method": "GPU", - "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node.log; build-1:/srv/artefacts/tas/same-work-20261008-02/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-02/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-02/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/d1-pairing/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-4090-2.json", - "note": "the dataset-day transition in phase 2: the node reader, the pool reader, the 5090 and the 4090 each switched program and dataset at nonce 1572864 with the program id unchanged and both boundary hashes equal to the reference; the 3090's phase 2 and the OpenCL and Metal reads are pending; in progress", + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-02/node/node.log; build-1:/srv/artefacts/tas/same-work-20261008-02/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-02/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-02/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-02/pool/d1-pairing/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-02/cuda-3090-3.json", + "note": "the dataset-day transition in phase 2: the node reader, the pool reader, the 5090 and the 4090 each switched program and dataset at nonce 1572864 with the program id unchanged and both boundary hashes equal to the reference; the OpenCL day-switch read (take 3) and the Metal read are pending; in progress", "network_label": "on an island, not a network", "in_progress": true } diff --git a/tools/ci/batches/same-work-20261008-03.json b/tools/ci/batches/same-work-20261008-03.json index 9b2f499e4..f22af826b 100644 --- a/tools/ci/batches/same-work-20261008-03.json +++ b/tools/ci/batches/same-work-20261008-03.json @@ -20,7 +20,7 @@ }, "claim_impact": "the same-work test's live-chain half (the coordinator's third row, 22:1x UK): the readers on the chain's own class v5 object at epoch 2 across the day switch 20734 to 20735; the node and pool readers run it on their release-line engines, the row those two can PASS without the class v6 branch", "network_label": "on an island, not a network", - "note": "On an island, not a network: build-1's seed is one of devnet-4's islands since the epoch-3 cut (21:34 UK), its stream pulled at 22:18 UK; epoch 2's object predates the split. Context 03 at build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json (sha256 53837a96...). The witness for the id: igneum-miner program-id reads class v5 attempt 0 id 81fbfa3aa413173f under era 7c36b833... and era 00..00 alike, from the miner at same-work-node cc23f9bd on build-2 (the class v5 engine path; the 2.0.1 pair's miner ef0f2ed8 on build-1 has no program-id subcommand), and the node lane's placed (c) read gave the same id; the hand's RPC port port 26870 answered no published state stream for the block at 22:18 UK. Packs v5-epoch2-d20734 and d20735 (both id 0x81fbfa3aa413173f) exported from release-2.0.1's igneum-pow on build-4 at 22:19 UK; the CPU references from release-2.0.1's igneum-pow (binary 7ba781db...) on build-4 under watchers, 22:20 to 00:21 UK (about 210 nonces a second a run; the one slip of the night, reported once), ref-D 873528b1... over [0, 1572864) and ref-D1 7f8e64ae... over [1572864, 3145728), read back on build-1 at 00:22 UK. NODE, two readers, both PASS on all three phases (00:27, 00:31, 00:35 UK; 1,048,576 agree each, 0 disagree, 0 missing; program id 0x81fbfa3aa413173f on every segment equal to the packs'; the day switch at 1572864 with both boundary hashes equal to the reference, 21488c8bc924c342 before and 08d55958274e224d after; genesis day 20734 installed, growth doublings 0): the RELEASE LINE, igneum-miner from kit 1's pair build-1:/srv/artefacts/200-94e4c6ba/node-lane (binary sha256 41cc0a82..., the recheck-vectors commits cherry-picked onto kit1-line 94e4c6ba = release-2.0.2-node without the v6 arm, the class v5 engine the kit ships), node-release-94e4c6ba/node-{1,2,3}.json (ac60f395..., 97f9758e..., bad64c5c...); and the D1 branch, same-work-node cc23f9bd (merged onto class-v6-node-review as fe0fdc59; binary 9d563c41...), node-same-work-node-cc23f9bd/node-{1,2,3}.json (7b1254f1..., a9da16a9..., 09ec9202...), the two readers' hashes equal to each other and to the reference on every nonce. CUDA, the kit's generation 5 worker 9bfcf728 on the driver at c8b65a26 or later: the 5090 PASS on all three phases (00:25 to 00:28 UK, cuda-5090-{1,2,3}.json 7040a0ab..., 86ff7bc4..., 3e3ec63f...) and the 4090 PASS on all three (00:30 to 00:33 UK, cuda-4090-{1,2,3}.json 1cb9efe5..., 6a52d5bb..., 1efffdde...), the day switch driven through the worker's prepare on each; the 3090 pending (the recursive copy to that pod is the fault, the fleet lane sends the context as one tgz). POOL, the 2.0.2 kit as pinned (release-2.0.2 77b5acc5's pool with node 7cfa422a, binary sha256 46e4c440..., on build-6, genesis day 20734 and dataset 2^28 from the pack): PASS on all three phases (00:28, 00:33, 00:38 UK; 1,048,576 agree each, 0 disagree, 0 missing; the boundary switched at 1572864 with one program id on both segments; 64, 128 and 64 of the sampled shares accepted through the share verifier), pool/kit-row/pool-{1,2,3}.json (c77d1276..., b30d4b65..., 957954e7...) with the README; every nonce of 3,145,728 agrees with the reference on the chain's own class v5 object through the kit as pinned. Metal and OpenCL are the morning's reads (08:30 UK). The rows stay NOT RUN in progress until every reader has read (the coordinator's ruling 23:1x UK).", + "note": "On an island, not a network: build-1's seed is one of devnet-4's islands since the epoch-3 cut (21:34 UK), its stream pulled at 22:18 UK; epoch 2's object predates the split. Context 03 at build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json (sha256 53837a96...). The witness for the id: igneum-miner program-id reads class v5 attempt 0 id 81fbfa3aa413173f under era 7c36b833... and era 00..00 alike, from the miner at same-work-node cc23f9bd on build-2 (the class v5 engine path; the 2.0.1 pair's miner ef0f2ed8 on build-1 has no program-id subcommand), and the node lane's placed (c) read gave the same id; the hand's RPC port port 26870 answered no published state stream for the block at 22:18 UK. Packs v5-epoch2-d20734 and d20735 (both id 0x81fbfa3aa413173f) exported from release-2.0.1's igneum-pow on build-4 at 22:19 UK; the CPU references from release-2.0.1's igneum-pow (binary 7ba781db...) on build-4 under watchers, 22:20 to 00:21 UK (about 210 nonces a second a run; the one slip of the night, reported once), ref-D 873528b1... over [0, 1572864) and ref-D1 7f8e64ae... over [1572864, 3145728), read back on build-1 at 00:22 UK. NODE, two readers, both PASS on all three phases (00:27, 00:31, 00:35 UK; 1,048,576 agree each, 0 disagree, 0 missing; program id 0x81fbfa3aa413173f on every segment equal to the packs'; the day switch at 1572864 with both boundary hashes equal to the reference, 21488c8bc924c342 before and 08d55958274e224d after; genesis day 20734 installed, growth doublings 0): the RELEASE LINE, igneum-miner from kit 1's pair build-1:/srv/artefacts/200-94e4c6ba/node-lane (binary sha256 41cc0a82..., the recheck-vectors commits cherry-picked onto kit1-line 94e4c6ba = release-2.0.2-node without the v6 arm, the class v5 engine the kit ships), node-release-94e4c6ba/node-{1,2,3}.json (ac60f395..., 97f9758e..., bad64c5c...); and the D1 branch, same-work-node cc23f9bd (merged onto class-v6-node-review as fe0fdc59; binary 9d563c41...), node-same-work-node-cc23f9bd/node-{1,2,3}.json (7b1254f1..., a9da16a9..., 09ec9202...), the two readers' hashes equal to each other and to the reference on every nonce. CUDA, the kit's generation 5 worker 9bfcf728 on the driver at c8b65a26 or later: the 5090 PASS on all three phases (00:25 to 00:28 UK, cuda-5090-{1,2,3}.json 7040a0ab..., 86ff7bc4..., 3e3ec63f...) and the 4090 PASS on all three (00:30 to 00:33 UK, cuda-4090-{1,2,3}.json 1cb9efe5..., 6a52d5bb..., 1efffdde...), the day switch driven through the worker's prepare on each; The 3090 PASS on all three phases (05:11 to 05:14 UK; cuda-3090-{1,2,3}.json de68d6d3..., 7f66ee96..., 00840cc9...; 1,048,576 agree each, 0 disagree, 0 missing; the day switch at 1572864 with the same boundary hashes as every other reader): the fleet lane's first run on that pod at 00:35 UK never ran a phase (its verify rule demanded two reference files where a context carries one, then its put failed on the pod's flapping ssh; a runner fault, reported once), the rerun from 04:57 UK ran detached on the pod under a pid file with a collector pulling each JSON. So the CUDA cell reads three cards on this context. POOL, the 2.0.2 kit as pinned (release-2.0.2 77b5acc5's pool with node 7cfa422a, binary sha256 46e4c440..., on build-6, genesis day 20734 and dataset 2^28 from the pack): PASS on all three phases (00:28, 00:33, 00:38 UK; 1,048,576 agree each, 0 disagree, 0 missing; the boundary switched at 1572864 with one program id on both segments; 64, 128 and 64 of the sampled shares accepted through the share verifier), pool/kit-row/pool-{1,2,3}.json (c77d1276..., b30d4b65..., 957954e7...) with the README; every nonce of 3,145,728 agrees with the reference on the chain's own class v5 object through the kit as pinned. Metal and OpenCL are the morning's reads (08:30 UK). The rows stay NOT RUN in progress until every reader has read (the coordinator's ruling 23:1x UK).", "cells": [ { "cell": "harness:same-work", @@ -29,8 +29,8 @@ ], "status": "NOT RUN", "method": "GPU", - "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/state-epoch2.igsd1; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/README.json", - "note": "node PASS x3 on the release line (kit 1's pair 94e4c6ba) and PASS x3 on the D1 branch (cc23f9bd); CUDA 5090 and 4090 PASS x3 on the gen 5 worker; CPU reference written; pool PASS x3 on the 2.0.2 kit as pinned (the share verifier's path); the 3090 pending; Metal and OpenCL the morning's (08:30 UK); in progress", + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/state-epoch2.igsd1; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/references/ref-D1.txt; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-3.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/README.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-3.json", + "note": "node PASS x3 on the release line (kit 1's pair 94e4c6ba) and PASS x3 on the D1 branch (cc23f9bd); CUDA 5090, 4090 and 3090 PASS x3 on the gen 5 worker; CPU reference written; pool PASS x3 on the 2.0.2 kit as pinned (the share verifier's path); Metal and OpenCL the morning's (08:30 UK); in progress", "network_label": "on an island, not a network", "in_progress": true }, @@ -41,8 +41,8 @@ ], "status": "NOT RUN", "method": "GPU", - "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json", - "note": "the day switch 20734 to 20735 at nonce 1572864 on the live object: both node readers, the kit's pool, the 5090 and the 4090 each switched program and dataset there with the program id unchanged and both boundary hashes equal to the reference; the 3090, OpenCL and Metal pending; in progress", + "evidence": "build-1:/srv/artefacts/tas/same-work-20261008-03/job-context.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-release-94e4c6ba/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/node-same-work-node-cc23f9bd/node-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-5090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-4090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/pool/kit-row/pool-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-1.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-2.json; build-1:/srv/artefacts/tas/same-work-20261008-03/cuda-3090-3.json", + "note": "the day switch 20734 to 20735 at nonce 1572864 on the live object: both node readers, the kit's pool, the 5090 and the 4090 each switched program and dataset there with the program id unchanged and both boundary hashes equal to the reference; OpenCL and Metal pending; in progress", "network_label": "on an island, not a network", "in_progress": true }