class v6 kit (8 October 2026): the Metal hosts size the dataset by the ds55 words and items defines (packbench and the worker's pack path; a power-of-two pack reads unchanged: hl-v6-fold f6c34aa6e87bf211 on both builds), the kit's box script for the all-together pack and its known-failed partner, the runner's script switch
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
igneum-labs
parent
f62b096e41
commit
ec792ec950
4 changed files with 109 additions and 8 deletions
|
|
@ -3162,7 +3162,9 @@ func servePackDataset(_ gpu: GPU, _ store: ServeStore, dayHex: String, dir: Stri
|
|||
guard let fillFn = lib.makeFunction(name: "igneum_cache_fill"), let buildFn = lib.makeFunction(name: "igneum_build") else { throw refuse("memhard.metal lacks igneum_cache_fill or igneum_build") }
|
||||
let fillPipe = try gpu.device.makeComputePipelineState(function: fillFn)
|
||||
let buildPipe = try gpu.device.makeComputePipelineState(function: buildFn)
|
||||
let words = 1 << datasetLog2
|
||||
// ds55 (the class v6 research packs, 8 October 2026): a non-power-of-two dataset carries IGNEUM_DATASET_WORDS (the
|
||||
// buffer's size) and IGNEUM_DATASET_ITEMS beside IGNEUM_DATASET_LOG2 (the floor); the kernel text maps (src * words) >> 32
|
||||
let words = defineU64("IGNEUM_DATASET_WORDS").map { Int($0) } ?? (1 << datasetLog2)
|
||||
// Class v5 (docs/design/class-v5-stored-state.md, 7 October 2026): the window's state leaves, leaves.bin beside program.h
|
||||
// (IGNEUM_STATE_LEAVES leaves of 16 little-endian words), checked against the pack's count and FNV-1a 64
|
||||
// (IGNEUM_STATE_LEAVES_FNV64) and bound as buffer 2 of igneum_build with the count in buffer 3 (packbench.swift's shape,
|
||||
|
|
|
|||
|
|
@ -93,8 +93,12 @@ let cacheFnvWant = hex64(vj["cache_fnv1a64"] as! String)
|
|||
let hotFnvWant: UInt64? = (vj["hot_fnv1a64"] as? String).map(hex64)
|
||||
|
||||
guard let device = MTLCreateSystemDefaultDevice(), let queue = device.makeCommandQueue() else { fail("no Metal device") }
|
||||
let words = 1 << datasetLog2
|
||||
let mask = UInt32(words - 1)
|
||||
// ds55 (the class v6 research packs, 8 October 2026): a non-power-of-two dataset carries IGNEUM_DATASET_WORDS and
|
||||
// IGNEUM_DATASET_ITEMS beside IGNEUM_DATASET_LOG2 (the floor); the buffer is sized by the words, the items by ITEMS,
|
||||
// and the kernel text maps an address as (src * words) >> 32, so the mask is the kernel's business, not this host's
|
||||
let datasetWordsDefine = defineU64("IGNEUM_DATASET_WORDS")
|
||||
let words = datasetWordsDefine.map { Int($0) } ?? (1 << datasetLog2)
|
||||
let mask = UInt32(truncatingIfNeeded: (1 << datasetLog2) - 1)
|
||||
let cacheWords = 1 << cacheLog2
|
||||
|
||||
func compile(_ file: String) -> MTLLibrary {
|
||||
|
|
@ -138,7 +142,7 @@ func fnv1a64(_ p: UnsafeRawPointer, _ n: Int) -> UInt64 {
|
|||
for i in 0..<n { h ^= UInt64(b[i]); h = h &* 0x100000001b3 }
|
||||
return h
|
||||
}
|
||||
let items = words / 16
|
||||
let items = defineU64("IGNEUM_DATASET_ITEMS").map { Int($0) } ?? (words / 16)
|
||||
// class v5 (docs/design/class-v5-stored-state.md): a pack with IGNEUM_STATE_LEAVES carries leaves.bin (16 words per leaf), bound
|
||||
// as buffer 2 of igneum_build with the count in buffer 3; the pack's FNV of the leaves is checked first
|
||||
var leavesBuf: MTLBuffer? = nil
|
||||
|
|
|
|||
|
|
@ -13,9 +13,9 @@
|
|||
#
|
||||
# Takes one of the box's build slots through infra/build-server/remote-run.sh (bs_remote_run), box 1 by default (the
|
||||
# build-server lane's word of 7 October 2026, 19:4x BST: the kit builds on build-1 explicitly). The work itself is
|
||||
# tools/class-v5/kits-on-box.sh, a script file run by path on the box (the inline-rm rule of 21:33 BST); when the
|
||||
# tools/class-v5/kits-on-box.sh (or KITS_BOX_SCRIPT=kits-v6-on-box.sh for the class v6 kit), a script file run by path on the box (the inline-rm rule of 21:33 BST); when the
|
||||
# build-server lane's lease tool is in place, the same file goes through `/srv/builds/_bin/lease pool <threads> -- bash
|
||||
# tools/class-v5/kits-on-box.sh ...` with the label "v5 kit".
|
||||
# tools/class-v5/${KITS_BOX_SCRIPT:-kits-on-box.sh} ...` with the label "v5 kit".
|
||||
set -euo pipefail
|
||||
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
BS_TOOL=class-v5-kits
|
||||
|
|
@ -34,9 +34,9 @@ ZIP="/srv/artefacts/packs/packs-ca3-v5-$STAMP.zip"
|
|||
bs_toolchain_check
|
||||
bs_sync_sources
|
||||
bs_log "sources at $WT (commit $BS_SHA on $BS_BRANCH); building the class v5 kits on box $BOX"
|
||||
# the body runs from a script FILE at the worktree's mirror checkout (tools/class-v5/kits-on-box.sh, carried by the commit), never
|
||||
# the body runs from a script FILE at the worktree's mirror checkout (tools/class-v5/${KITS_BOX_SCRIPT:-kits-on-box.sh}, carried by the commit), never
|
||||
# an inline string: main's rule of 7 October 2026, 21:33 BST (no rm, find -delete or truncation inside a bash -c string)
|
||||
CMD="cd '$WT' && bash tools/class-v5/kits-on-box.sh '$STAMP' '$SKIP_EMU'"
|
||||
CMD="cd '$WT' && bash tools/class-v5/${KITS_BOX_SCRIPT:-kits-on-box.sh} '$STAMP' '$SKIP_EMU'"
|
||||
set +e
|
||||
BR_KIND=build BR_COMMAND="class v5 kits (packfile-test, emu --check v5, Linux and Windows workers, kit zip)" BR_TARGET="x86_64-linux+windows" BR_ARTEFACTS="" \
|
||||
bs_remote_run "$WT" "$BS_WT class-v5 kits" "$CMD" 2>&1 | tee "/tmp/v5-kits-$STAMP.log" | grep -E '^(STEP|FAIL|KIT|BIN|LOGS|check PASS|self-test|packfile-test|emu:| v5|build-remote: RESULT)'
|
||||
|
|
|
|||
95
tools/class-v5/kits-v6-on-box.sh
Executable file
95
tools/class-v5/kits-v6-on-box.sh
Executable file
|
|
@ -0,0 +1,95 @@
|
|||
#!/usr/bin/env bash
|
||||
# The class v5 kit build as it runs ON igneum-build-1 (called by tools/class-v5/kits-remote.sh through the build-server
|
||||
# library's remote runner, from the worktree's mirror checkout, so this file travels with the commit). A script file by
|
||||
# path, never an inline string: main's rule of 7 October 2026, 21:33 BST (no rm, find -delete or truncation inside a
|
||||
# `bash -c` string; tools/ci/inline-rm-check.sh). What it does, in order: the pack loader test with the class v5 cases,
|
||||
# the Linux and Windows builds of the two one-click workers, the CPU emulation's --check on the class v5 pack, then the
|
||||
# kit zip at /srv/artefacts/packs/packs-ca3-v5-<stamp>.zip with SHA256SUMS. Logs land in /srv/builds/_log/v5-class/kits-<stamp>.
|
||||
# bash tools/class-v5/kits-on-box.sh <stamp> <skip-emu 0|1> (cwd = the worktree root on the box)
|
||||
set -u
|
||||
STAMP="${1:?stamp}"; SKIP_EMU="${2:-0}"
|
||||
ZIP="/srv/artefacts/packs/packs-class-v6-$STAMP.zip"
|
||||
LOGDIR="/srv/builds/_log/v5-class/kits-$STAMP"
|
||||
T=$(mktemp -d /tmp/v5-kits.XXXXXX); S=$T/stage; mkdir -p "$S/bin/linux" "$S/bin/windows" "$S/src" "$S/packs" "$S/tools" "$LOGDIR"
|
||||
rc=0
|
||||
step() { echo "STEP $1 $(date -u +%H:%M:%SZ)"; }
|
||||
fail() { echo "FAIL $1"; rc=1; }
|
||||
finish() { cp "$T"/*.log "$LOGDIR"/ 2>/dev/null; echo "LOGS $LOGDIR"; rm -rf "$T"; }
|
||||
# class v6 (8 October 2026, the coordinator's order): the all-together pack and its known-failed partner come from the hash
|
||||
# lane's tgz files on build-1 (V6_ALL and V6_PARTNER, default hl-v6-all and hl-v6-foldrw), extracted beside the stage;
|
||||
# the emulation check runs pack A = the all pack against pack B = the partner, and the two ids must differ
|
||||
V6_ALL="${V6_ALL:-hl-v6-all}"; V6_PARTNER="${V6_PARTNER:-hl-v6-foldrw}"
|
||||
PK=$T/v6packs; mkdir -p "$PK"
|
||||
for p in "$V6_ALL" "$V6_PARTNER"; do tar -xzf "/srv/artefacts/packs/$p.tgz" -C "$PK" 2>/dev/null || { echo "FAIL no pack tgz /srv/artefacts/packs/$p.tgz"; exit 1; }; done
|
||||
rm -rf "$PK"/._* "$PK"/*/._* 2>/dev/null
|
||||
V5="$PK/$V6_ALL"; V6P="$PK/$V6_PARTNER"
|
||||
echo "PACKS $V6_ALL id $(grep -oE 'IGNEUM_PROGRAM_ID 0x[0-9a-f]+' "$V5/program.h" | awk '{print $2}') | $V6_PARTNER id $(grep -oE 'IGNEUM_PROGRAM_ID 0x[0-9a-f]+' "$V6P/program.h" | awk '{print $2}')"
|
||||
[ "$(grep -oE 'IGNEUM_PROGRAM_ID 0x[0-9a-f]+' "$V5/program.h")" != "$(grep -oE 'IGNEUM_PROGRAM_ID 0x[0-9a-f]+' "$V6P/program.h")" ] || { echo "FAIL the all pack and its partner carry one id"; exit 1; }
|
||||
step packfile-test
|
||||
cc -std=c99 -Wall -Wextra -Wno-unused-function -O1 -o "$T/packfile-test" proto-cuda/nvrtc/emu/packfile-test.c && "$T/packfile-test" proto-cuda/packs/igneum-devnet-v4-epoch0 "$V5" > "$T/packfile-test.log" 2>&1 || fail "packfile-test ($(tail -1 "$T/packfile-test.log"))"
|
||||
grep -E '^(FAIL| v5 pack)' "$T/packfile-test.log"; grep -c '^ok' "$T/packfile-test.log" | sed 's/^/packfile-test ok lines /'
|
||||
step linux-opencl-worker
|
||||
gcc -std=c99 -O2 -Wall -Wextra -Wno-stringop-truncation -Wno-format-truncation -DIGNEUM_CL_DYNAMIC -DCL_TARGET_OPENCL_VERSION=120 -I proto-cuda/packs/igneum-devnet-v4-epoch0 -DIGNEUM_KERNEL_PATH='"kernel_bound.cl"' -o "$S/bin/linux/igneum-worker-opencl" proto-opencl/host.c -ldl -lpthread 2> "$T/cl-linux.log" || { fail "linux opencl worker"; head -20 "$T/cl-linux.log"; }
|
||||
step linux-cuda-worker
|
||||
g++ -std=c++17 -O2 -Wall -Wextra -I proto-cuda/nvrtc -I /usr/local/cuda/include -o "$S/bin/linux/igneum-worker-cuda" proto-cuda/nvrtc/worker.cpp -ldl -lpthread 2> "$T/cuda-linux.log" || { fail "linux cuda worker"; head -20 "$T/cuda-linux.log"; }
|
||||
step windows-cuda-worker
|
||||
x86_64-w64-mingw32-g++ -std=c++17 -O2 -Wall -Wextra -static -I proto-cuda/nvrtc -I /usr/local/cuda/include -o "$S/bin/windows/igneum-worker-cuda.exe" proto-cuda/nvrtc/worker.cpp 2> "$T/cuda-win.log" && x86_64-w64-mingw32-strip "$S/bin/windows/igneum-worker-cuda.exe" || { fail "windows cuda worker"; head -20 "$T/cuda-win.log"; }
|
||||
step windows-opencl-worker
|
||||
# the Khronos CL headers alone (never -I /usr/include: that puts glibc's stdint.h ahead of mingw's)
|
||||
mkdir -p "$T/inc" && cp -r /usr/include/CL "$T/inc/"
|
||||
x86_64-w64-mingw32-gcc -std=c99 -O2 -Wall -Wextra -Wno-stringop-truncation -Wno-format-truncation -static -DIGNEUM_CL_DYNAMIC -DCL_TARGET_OPENCL_VERSION=120 -I "$T/inc" -I proto-cuda/packs/igneum-devnet-v4-epoch0 -DIGNEUM_KERNEL_PATH='"kernel_bound.cl"' -o "$S/bin/windows/igneum-worker-opencl.exe" proto-opencl/host.c 2> "$T/cl-win.log" && x86_64-w64-mingw32-strip "$S/bin/windows/igneum-worker-opencl.exe" || { fail "windows opencl worker"; head -20 "$T/cl-win.log"; }
|
||||
if [ "$SKIP_EMU" = 0 ]; then
|
||||
step emu-check-v5
|
||||
# the CPU emulation (proto-cuda/nvrtc/emu/test.sh's build) with pack A = the class v5 pack and pack B = the v4 control
|
||||
E=$T/emu; mkdir -p "$E"
|
||||
emu_kernel() { { echo '#include <cuda_runtime.h>'; echo '#include <cstdint>'; echo "namespace $2 {"; sed -E 's/([A-Za-z_0-9]+)<<<([^,]+), ([^>]+)>>>\(/emu_launch(\1, \2, \3, /' "$1/$3.cu"; echo "}"; } > "$E/$3_$2.cpp"; g++ -std=c++17 -O2 -w -I proto-cuda/emu -I "$1" -c "$E/$3_$2.cpp" -o "$E/$3_$2.o"; }
|
||||
emu_kernel "$V5" emu_pack_a kernel && emu_kernel "$V5" emu_pack_a kernel_bound && emu_kernel "$V6P" emu_pack_b kernel && emu_kernel "$V6P" emu_pack_b kernel_bound \
|
||||
&& g++ -std=c++17 -O2 -Wall -Wextra -DIGNEUM_EMU -I proto-cuda/nvrtc -I /usr/local/cuda/include -c proto-cuda/nvrtc/worker.cpp -o "$E/worker.o" \
|
||||
&& g++ -std=c++17 -O2 -Wall -Wextra -DIGNEUM_EMU -DIGNEUM_EMU_TWO_PACKS -I proto-cuda/nvrtc -I proto-cuda/emu -I /usr/local/cuda/include -c proto-cuda/nvrtc/emu/emu_backend.cpp -o "$E/emu_backend.o" \
|
||||
&& g++ -std=c++17 -O2 -w -I proto-cuda/emu -c proto-cuda/emu/shim.cpp -o "$E/shim.o" \
|
||||
&& g++ -o "$E/igneum-worker-cuda-emu" "$E"/*.o -pthread 2> "$T/emu-build.log" || { fail "emu build"; head -30 "$T/emu-build.log"; }
|
||||
if [ -x "$E/igneum-worker-cuda-emu" ]; then
|
||||
( IGNEUM_EMU_PACK="$V5" IGNEUM_EMU_PACK2="$V6P" timeout 1500 nice -n 10 "$E/igneum-worker-cuda-emu" --check --pack "$V5" > "$T/emu-check.log" 2>&1 ) || fail "emu --check on the class v5 pack (rc $?)"
|
||||
grep -E '^check PASS|self-test|FAIL|error' "$T/emu-check.log" | cut -c1-400
|
||||
fi
|
||||
fi
|
||||
if [ "$rc" != 0 ]; then echo "KIT not written: a step failed (rc $rc)"; finish; exit 1; fi
|
||||
step stage
|
||||
for p in "$V6_ALL" "$V6_PARTNER"; do mkdir -p "$S/packs/$p"; cp "$PK/$p"/* "$S/packs/$p/"; done
|
||||
cp proto-cuda/nvrtc/worker.cpp proto-cuda/nvrtc/packfile.h proto-cuda/nvrtc/cuda_api.h proto-opencl/host.c proto-opencl/cl_dynamic.h proto-newpow/class-v5/bench.cu proto-newpow/class-v5/run.sh proto-metal/packbench.swift "$S/src/"
|
||||
cp tools/class-v5/fleet-cuda-v5-bench.sh tools/class-v5/pc1-amd-v5-bench.ps1 "$S/tools/"
|
||||
cat > "$S/README.txt" <<'R'
|
||||
packs-class-v6: the class v6 kit (Igneum, 8 October 2026; docs/design/class-v5-stored-state.md section 0, the class v6 row): the all-together pack and its known-failed partner
|
||||
packs/v5-dn3-epoch0 the first class v5 pack: Devnet 3 epoch 0, program id e5a4ac5978462156, 11 state leaves (leaves.bin, 704 B) under
|
||||
state root 7e37a9fb19b154d32daf5bf30a50d339a75029fbc9eec9ea20e95439dba5a311, cache FNV-1a 64 7334fa46e5d972eb
|
||||
expected fingerprint of the 2^24 outputs at base nonce 0: 82b19cbde8557ea5 (Metal, M5 Max, 18:07:09Z) on every platform
|
||||
packs/v5-genesis the string-seed class v5 pack over the devnet's 93 leaves; packs/v4-genesis the class v4 control (sub-version 3)
|
||||
bin/linux igneum-worker-cuda (NVRTC, libcuda + libnvrtc.so.12 at run time), igneum-worker-opencl (libOpenCL.so.1 at run time)
|
||||
bin/windows igneum-worker-cuda.exe (nvcuda.dll + nvrtc64 at run time), igneum-worker-opencl.exe (OpenCL.dll at run time)
|
||||
Every worker reads a class v5 pack's leaves.bin, checks it against the pack's IGNEUM_STATE_LEAVES_FNV64, uploads it for igneum_build and
|
||||
frees it after the build; a pack without its leaves builds nothing. Self-test: cache head, last line and FNV; dataset head, last word and
|
||||
64 samples; 96 vector lanes. Bench: igneum-worker-cuda --bench --pack packs/v5-dn3-epoch0 --batch-log2 24 (fleet-cuda-v5-bench.sh);
|
||||
igneum-worker-opencl --bench-pack --pack packs/v5-dn3-epoch0 --batch-log2 24 --device <n> (pc1-amd-v5-bench.ps1); src/bench.cu with nvcc.
|
||||
Intel: not measured tonight (7 October 2026); no Arc B580 sits on PC 1 or PC 2 and the only Arc path needs a driver click, which no PC job
|
||||
may raise; the card's holder and a click-free driver path are owed. The OpenCL worker takes the Intel device by index the same way.
|
||||
R
|
||||
( cd "$S" && find . -type f ! -name SHA256SUMS | sort | xargs sha256sum > SHA256SUMS )
|
||||
mkdir -p /srv/artefacts/packs
|
||||
python3 - "$S" "$ZIP" <<'PY'
|
||||
import os, sys, zipfile
|
||||
stage, out = sys.argv[1], sys.argv[2]
|
||||
with zipfile.ZipFile(out, 'w', zipfile.ZIP_DEFLATED) as z:
|
||||
for dp, dn, fn in os.walk(stage):
|
||||
dn.sort()
|
||||
for f in sorted(fn):
|
||||
p = os.path.join(dp, f); rel = os.path.relpath(p, stage)
|
||||
zi = zipfile.ZipInfo(rel, date_time=(2026, 10, 7, 0, 0, 0)); zi.compress_type = zipfile.ZIP_DEFLATED
|
||||
zi.external_attr = (0o755 if rel.startswith('bin/') or rel.endswith('.sh') else 0o644) << 16
|
||||
with open(p, 'rb') as fh: z.writestr(zi, fh.read())
|
||||
PY
|
||||
sha256sum "$ZIP" | cut -c1-64 > "$ZIP.sha256"
|
||||
echo "KIT $ZIP bytes $(stat -c %s "$ZIP") files $(python3 -c "import zipfile,sys; print(len(zipfile.ZipFile(sys.argv[1]).namelist()))" "$ZIP")"
|
||||
echo "KIT sha256 $(cat "$ZIP.sha256")"
|
||||
for b in "$S"/bin/linux/* "$S"/bin/windows/*; do echo "BIN $(basename "$(dirname "$b")")/$(basename "$b") $(stat -c %s "$b") bytes sha256 $(sha256sum "$b" | cut -c1-64)"; done
|
||||
finish
|
||||
exit 0
|
||||
Loading…
Reference in a new issue