igneum-network/igneum
1
0
Fork 0
Code Issues Pull requests Releases Activity
igneum/proto-cuda/packs-ca3-shadow/sh256x2/kernel.cl
igneum-labs 51dba1ce76 Counter ASIC 3.0 item 8: the latency-shadow knob (LoadClass +sh<S>x<R>), its packs and the PC 2 playbook 
A shadow block of S ALU instructions run R times at the end of every iteration, drawn from the program stream after
the 64 base instructions, behind LoadClass::shadow: v2 and v3 draw nothing and emit nothing (the pinned packs are
byte-identical, cargo test 54 + 4 + 19 + 7 green). The interpreter, the three kernel dialects (both kernels each),
program.h and program.json carry it; the acceptance rule interprets the base program only. Packs for seed
igneum-genesis over class mx8 at 4,096 to 180,224 shadow instructions per hash (proto-cuda/packs-ca3-shadow), and
the PC 2 bench playbook tools/ca3-shadow/pc2-shadow-bench.ps1 (passes the publisher's three checks).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 07:58:35 +00:00

538 lines
30 KiB
Common Lisp
Raw Blame History

// Generated by igneum-pow export (generator v2) for seed "igneum-genesis". Do not edit by hand.
// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal).
// Built from source at runtime by proto-opencl/host.c, which passes these defines:
// IGNEUM_GROUP work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit)
// IGNEUM_EXCHANGE 0 = local-memory exchange with a barrier (any device, any wave width; the default)
// 1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32
// 2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition
// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units;
// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32.
#ifndef IGNEUM_GROUP
#define IGNEUM_GROUP 32
#endif
#ifndef IGNEUM_EXCHANGE
#define IGNEUM_EXCHANGE 0
#endif
#ifdef __OPENCL_VERSION__
#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1)))
#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n]
#if IGNEUM_EXCHANGE == 1
#ifdef cl_khr_subgroups
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
#endif
#ifdef cl_khr_subgroup_shuffle
#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable
#endif
#elif IGNEUM_EXCHANGE == 2
#pragma OPENCL EXTENSION cl_intel_subgroups : enable
#endif
#else
// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros.
#include "emu_opencl.h"
#endif
#if IGNEUM_EXCHANGE == 1
#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m))
#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u)
#elif IGNEUM_EXCHANGE == 2
#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m))
#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u)
#else
// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per
// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane
// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's
// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier.
#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; }
#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; }
#endif
static inline uint splitmix32(uint x) {
x ^= x >> 16; x *= 0x7feb352du;
x ^= x >> 15; x *= 0x846ca68bu;
x ^= x >> 16;
return x;
}
// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32.
static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); }
// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x.
static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); }
static inline uint ds_elem(uint i, uint d0, uint d1) {
uint x = i ^ d0;
x *= 0x9E3779B1u; x ^= x >> 15;
x += d1;
x *= 0x85EBCA77u; x ^= x >> 13;
x *= 0xC2B2AE3Du; x ^= x >> 16;
return x;
}
// Memory-hard dataset core (MEMHARD.md). Cache: 2^26 words in 2^16 segments of 64 chained ChaCha12 lines.
// Item: 8 rounds of 8 x seed-parameterised mixer + one 64-byte cache read, then 8 x final mixer (class v3, mixer multiplier 8,
// docs/plans/mixer-x4.md: the round key of application j of round r is 0x9E3779B9 * (r * m + j + 1)). All parameters are literals.
#define MH_CACHE_LINE_MASK 0x003fffffu
#define MH_SEGMENT_LINES 64u
#define MH_QR(a, b, c, d, r1, r2, r3, r4) { a += b; d ^= a; d = mh_rotl(d, r1); c += d; b ^= c; b = mh_rotl(b, r2); a += b; d ^= a; d = mh_rotl(d, r3); c += d; b ^= c; b = mh_rotl(b, r4); }
static inline uint mh_rotl(uint x, uint n) { return (x << n) | (x >> (32u - n)); } // n in 1..31 at every call site
// y = ChaCha12 core(x) + x
static inline void mh_chacha_block(const uint* x, uint* y) {
for (uint i = 0u; i < 16u; ++i) y[i] = x[i];
for (uint r = 0u; r < 6u; ++r) {
MH_QR(y[0], y[4], y[8], y[12], 16u, 12u, 8u, 7u) MH_QR(y[1], y[5], y[9], y[13], 16u, 12u, 8u, 7u)
MH_QR(y[2], y[6], y[10], y[14], 16u, 12u, 8u, 7u) MH_QR(y[3], y[7], y[11], y[15], 16u, 12u, 8u, 7u)
MH_QR(y[0], y[5], y[10], y[15], 16u, 12u, 8u, 7u) MH_QR(y[1], y[6], y[11], y[12], 16u, 12u, 8u, 7u)
MH_QR(y[2], y[7], y[8], y[13], 16u, 12u, 8u, 7u) MH_QR(y[3], y[4], y[9], y[14], 16u, 12u, 8u, 7u)
}
for (uint i = 0u; i < 16u; ++i) y[i] += x[i];
}
// One cache segment: 64 chained lines written at cache[seg * 1024]. in_j = prev ^ (sigma || K || seg || j || tag), prev_0 = 0.
static inline void mh_cache_segment(__global uint* cache, uint seg) {
uint prev[16]; uint x[16]; uint y[16];
for (uint i = 0u; i < 16u; ++i) prev[i] = 0u;
for (uint j = 0u; j < MH_SEGMENT_LINES; ++j) {
x[0] = 0x61707865u ^ prev[0]; x[1] = 0x3320646eu ^ prev[1]; x[2] = 0x79622d32u ^ prev[2]; x[3] = 0x6b206574u ^ prev[3];
x[4] = 0x3067619fu ^ prev[4];
x[5] = 0x3c269176u ^ prev[5];
x[6] = 0x84a03b03u ^ prev[6];
x[7] = 0xf8c63294u ^ prev[7];
x[8] = 0xff977c5bu ^ prev[8];
x[9] = 0xe60def3eu ^ prev[9];
x[10] = 0x63630141u ^ prev[10];
x[11] = 0xb8fbcb58u ^ prev[11];
x[12] = seg ^ prev[12]; x[13] = j ^ prev[13]; x[14] = 0x49676e65u ^ prev[14]; x[15] = 0x756d4d48u ^ prev[15];
mh_chacha_block(x, y);
__global uint* line = cache + ((seg * MH_SEGMENT_LINES + j) * 16u);
for (uint i = 0u; i < 16u; ++i) { line[i] = y[i]; prev[i] = y[i]; }
}
}
// M_r: per word (s ^ (RC + rk)) * MUL, then a column round and a diagonal round with the seed-drawn rotations.
static inline void mh_mixer(uint* s, uint rk) {
s[0] = (s[0] ^ (0xbab68293u + rk)) * 0x42146205u;
s[1] = (s[1] ^ (0xcc162340u + rk)) * 0x52cbe0fbu;
s[2] = (s[2] ^ (0x6ce151ccu + rk)) * 0x7ecf4a03u;
s[3] = (s[3] ^ (0xe62b8997u + rk)) * 0x6728907fu;
s[4] = (s[4] ^ (0xc9c80297u + rk)) * 0xd81d9751u;
s[5] = (s[5] ^ (0xf74a1654u + rk)) * 0x132952c3u;
s[6] = (s[6] ^ (0x3d704af5u + rk)) * 0xf60de277u;
s[7] = (s[7] ^ (0x3cf522b7u + rk)) * 0x05358035u;
s[8] = (s[8] ^ (0x2b9cac04u + rk)) * 0xbaf6499du;
s[9] = (s[9] ^ (0xa880ac10u + rk)) * 0xe4db9667u;
s[10] = (s[10] ^ (0x13e5dd1du + rk)) * 0x3e98f45du;
s[11] = (s[11] ^ (0x6fc3e233u + rk)) * 0xd0004eddu;
s[12] = (s[12] ^ (0x2d83eeacu + rk)) * 0x2691630du;
s[13] = (s[13] ^ (0x9006e8bfu + rk)) * 0x9beb3bcfu;
s[14] = (s[14] ^ (0x2c4b5362u + rk)) * 0xab310379u;
s[15] = (s[15] ^ (0x31b49ee2u + rk)) * 0x99cfb423u;
MH_QR(s[0], s[4], s[8], s[12], 20u, 20u, 19u, 4u) MH_QR(s[1], s[5], s[9], s[13], 20u, 20u, 19u, 4u)
MH_QR(s[2], s[6], s[10], s[14], 20u, 20u, 19u, 4u) MH_QR(s[3], s[7], s[11], s[15], 20u, 20u, 19u, 4u)
MH_QR(s[0], s[5], s[10], s[15], 26u, 3u, 3u, 27u) MH_QR(s[1], s[6], s[11], s[12], 26u, 3u, 3u, 27u)
MH_QR(s[2], s[7], s[8], s[13], 26u, 3u, 3u, 27u) MH_QR(s[3], s[4], s[9], s[14], 26u, 3u, 3u, 27u)
}
// Item t: 16 words. s = (K, t * MUL[i] + RC[i]); 8 rounds of 8 x mixer + cache line s[0] & mask; 8 x final mixer.
static inline void mh_item(__global const uint* cache, uint t, uint* s) {
s[0] = 0x3067619fu;
s[1] = 0x3c269176u;
s[2] = 0x84a03b03u;
s[3] = 0xf8c63294u;
s[4] = 0xff977c5bu;
s[5] = 0xe60def3eu;
s[6] = 0x63630141u;
s[7] = 0xb8fbcb58u;
s[8] = t * 0x42146205u + 0xbab68293u;
s[9] = t * 0x52cbe0fbu + 0xcc162340u;
s[10] = t * 0x7ecf4a03u + 0x6ce151ccu;
s[11] = t * 0x6728907fu + 0xe62b8997u;
s[12] = t * 0xd81d9751u + 0xc9c80297u;
s[13] = t * 0x132952c3u + 0xf74a1654u;
s[14] = t * 0xf60de277u + 0x3d704af5u;
s[15] = t * 0x05358035u + 0x3cf522b7u;
for (uint r = 0u; r < 8u; ++r) {
for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (r * 8u + j + 1u));
__global const uint* line = cache + ((s[0] & MH_CACHE_LINE_MASK) * 16u);
for (uint i = 0u; i < 16u; ++i) s[i] ^= line[i];
}
for (uint j = 0u; j < 8u; ++j) mh_mixer(s, 0x9E3779B9u * (64u + j + 1u));
}
// dataset[w] without the dataset: derive item w >> 4 and take word w & 15.
static inline uint mh_word(__global const uint* cache, uint w) { uint s[16]; mh_item(cache, w >> 4u, s); return s[w & 15u]; }
// Memory-hard dataset (MEMHARD.md). One work-item per cache segment; one work-item per 64-byte dataset item.
// The same constants as memhard.h in this pack (one emitter, three dialects).
__kernel void igneum_cache_fill(__global uint* cache, uint nSegments) {
uint seg = (uint)get_global_id(0);
if (seg < nSegments) mh_cache_segment(cache, seg);
}
__kernel void igneum_build(__global uint* ds, __global const uint* cache, uint nItems) {
uint t = (uint)get_global_id(0);
if (t < nItems) {
uint s[16];
mh_item(cache, t, s);
__global uint* d = ds + ((ulong)t * 16u);
for (uint i = 0u; i < 16u; ++i) d[i] = s[i];
}
}
// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the
// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and
// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all).
IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) {
uint gid = (uint)get_global_id(0);
uint lid = (uint)get_local_id(0);
uint nonce = baseNonce + gid;
uint r0, r1, r2, r3, r4, r5, r6, r7;
#if IGNEUM_EXCHANGE == 0
IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP);
uint xk = 0u;
#else
(void)lid;
#endif
{ uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; } // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1]
{ uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; } // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2]
{ uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; } // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3]
{ uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; } // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4]
{ uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; } // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5]
{ uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; } // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6]
{ uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; } // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7]
{ uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; } // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0]
for (uint it = 0u; it < 8u; ++it) {
uint sel = r0;
r2 = r3 * r4 + r2; // 0 mad
r2 = r1 * r1 + r2; // 1 mad
r2 = r3 * r2 + r2; // 2 mad
r3 = r3 ^ r5; // 3 xor
r7 = r7 ^ ds[r2 & mask]; // 4 load
r5 = r5 ^ ds[r7 & mask]; // 5 load
{ uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r1 = r1 ^ t_; } // 6 shfl
{ uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r7 = r7 ^ t_; } // 7 shfl
r1 = mul_hi(r1, r5); // 8 mulhi
r6 = rotr_var(r6, r3); // 9 rotr
r3 = r3 | r4; // 10 or
r4 = r4 ^ ds[r3 & mask]; // 11 load
r0 = mul_hi(r0, r4); // 12 mulhi
r5 = r5 + r1 + ((((sel >> 30u) & 1u) != 0u) ? 0xd3177981u : 0xc7934706u); // 13 add
r0 = r0 ^ ds[r4 & mask]; // 14 load
r2 = r2 - r4; // 15 sub
r2 = r2 ^ ds[r0 & mask]; // 16 load
r7 = r7 ^ ds[r2 & mask]; // 17 load
{ uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r7 = r7 ^ t_; } // 18 shfl
r5 = r5 * r0; // 19 mul
{ uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // 20 shfl
{ uint t_; IGNEUM_SHFL_XOR(t_, r4, 16u); r2 = r2 ^ t_; } // 21 shfl
r6 = mul_hi(r6, r2); // 22 mulhi
r6 = r6 ^ ds[r1 & mask]; // 23 load
r5 = r5 * r0; // 24 mul
r5 = rotl_imm(r5, 19u); // 25 rotl
{ uint t_; IGNEUM_SHFL_XOR(t_, r6, 2u); r7 = r7 ^ t_; } // 26 shfl
r0 = r0 ^ r5; // 27 xor
r0 = r0 ^ r4; // 28 xor
r3 = r3 - r0; // 29 sub
r5 = r5 * r1; // 30 mul
r7 = r7 ^ ds[r2 & mask]; // 31 load
r1 = r1 ^ ds[r0 & mask]; // 32 load
r5 = r5 ^ r6; // 33 xor
r5 = r5 ^ ds[r1 & mask]; // 34 load
r0 = mul_hi(r0, r5); // 35 mulhi
{ uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r5 = r5 ^ t_; } // 36 shfl
r7 = r7 ^ ds[r0 & mask]; // 37 load
r3 = r3 + r1 + ((((sel >> 27u) & 1u) != 0u) ? 0x230c005cu : 0x75ba2fadu); // 38 add
{ uint t_; IGNEUM_SHFL_XOR(t_, r5, 4u); r1 = r1 ^ t_; } // 39 shfl
r2 = r2 ^ r5; // 40 xor
r3 = r6 * r3 + r3; // 41 mad
r6 = r6 - r7; // 42 sub
r7 = r7 ^ r0; // 43 xor
r1 = r1 ^ ds[r7 & mask]; // 44 load
r2 = r2 * r3; // 45 mul
r1 = mul_hi(r1, r5); // 46 mulhi
r4 = r4 - r3; // 47 sub
r2 = rotr_var(r2, r6); // 48 rotr
r3 = r3 ^ ds[r5 & mask]; // 49 load
r1 = r1 + r5 + ((((sel >> 7u) & 1u) != 0u) ? 0x1907970cu : 0x81b8bc2cu); // 50 add
r0 = r0 * r2; // 51 mul
r0 = r0 + r2 + ((((sel >> 6u) & 1u) != 0u) ? 0x699fd448u : 0x4f92b968u); // 52 add
r1 = r1 + r0 + ((((sel >> 12u) & 1u) != 0u) ? 0x77b1520du : 0x2bb965afu); // 53 add
r7 = rotl_imm(r7, 14u); // 54 rotl
r3 = r3 + r7 + ((((sel >> 1u) & 1u) != 0u) ? 0xa54c55a0u : 0x7b0fe07au); // 55 add
r6 = r6 ^ ds[r7 & mask]; // 56 load
r1 = rotr_var(r1, r5); // 57 rotr
r5 = r5 ^ ds[r4 & mask]; // 58 load
r6 = r6 ^ ds[r2 & mask]; // 59 load
r3 = r5 * r0 + r3; // 60 mad
r5 = r5 + r7 + ((((sel >> 31u) & 1u) != 0u) ? 0xad7493e7u : 0xaf9dd72du); // 61 add
r4 = r4 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0x1e07c3d9u : 0x89841d87u); // 62 add
r5 = rotl_imm(r5, 19u); // 63 rotl
// latency-shadow block (Counter ASIC 3.0 item 8): 256 ALU instructions x 2 passes after instruction 63, no load
for (uint sh = 0u; sh < 2u; ++sh) {
r5 = r5 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x8bc12da9u : 0x92199f99u); // s0 add
r0 = r0 + r7 + ((((sel >> 26u) & 1u) != 0u) ? 0x63079e5au : 0x8d72d3adu); // s1 add
{ uint t_; IGNEUM_SHFL_XOR(t_, r3, 2u); r6 = r6 ^ t_; } // s2 shfl
r4 = r4 - r2; // s3 sub
r7 = r7 + r0 + ((((sel >> 31u) & 1u) != 0u) ? 0x5d4c7a60u : 0xb21b4babu); // s4 add
r0 = rotl_imm(r0, 11u); // s5 rotl
r0 = r0 + r1 + ((((sel >> 16u) & 1u) != 0u) ? 0xc88e2942u : 0x2fe0e98bu); // s6 add
r7 = r7 ^ r1; // s7 xor
r1 = r6 * r5 + r1; // s8 mad
{ uint t_; IGNEUM_SHFL_XOR(t_, r1, 4u); r6 = r6 ^ t_; } // s9 shfl
r1 = r2 * r2 + r1; // s10 mad
r5 = r0 * r3 + r5; // s11 mad
{ uint t_; IGNEUM_SHFL_XOR(t_, r6, 4u); r2 = r2 ^ t_; } // s12 shfl
r1 = r1 + r5 + ((((sel >> 25u) & 1u) != 0u) ? 0xbc48c63eu : 0xbdf8f9a5u); // s13 add
r1 = rotl_imm(r1, 29u); // s14 rotl
r1 = r1 - r4; // s15 sub
r7 = r7 | r1; // s16 or
{ uint t_; IGNEUM_SHFL_XOR(t_, r4, 8u); r2 = r2 ^ t_; } // s17 shfl
r7 = r7 ^ r4; // s18 xor
r6 = r6 * r1; // s19 mul
r5 = r6 * r0 + r5; // s20 mad
r3 = r3 - r1; // s21 sub
r6 = r6 * r0; // s22 mul
r2 = r2 + r0 + ((((sel >> 1u) & 1u) != 0u) ? 0x96e8f127u : 0x45c37cecu); // s23 add
r6 = r6 - r4; // s24 sub
r7 = r3 * r4 + r7; // s25 mad
r3 = rotl_imm(r3, 9u); // s26 rotl
r2 = r2 - r1; // s27 sub
r6 = mul_hi(r6, r0); // s28 mulhi
{ uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r2 = r2 ^ t_; } // s29 shfl
{ uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r1 = r1 ^ t_; } // s30 shfl
r1 = r1 + r6 + ((((sel >> 1u) & 1u) != 0u) ? 0xb1cdb2abu : 0x37985632u); // s31 add
r0 = rotr_var(r0, r1); // s32 rotr
{ uint t_; IGNEUM_SHFL_XOR(t_, r4, 2u); r3 = r3 ^ t_; } // s33 shfl
{ uint t_; IGNEUM_SHFL_XOR(t_, r3, 4u); r0 = r0 ^ t_; } // s34 shfl
r7 = r7 * r0; // s35 mul
r3 = r3 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x856e0180u : 0x804e777eu); // s36 add
r1 = r1 ^ r6; // s37 xor
r2 = r2 * r7; // s38 mul
r6 = r6 + r5 + ((((sel >> 2u) & 1u) != 0u) ? 0xe9bd0cc4u : 0x0b06c8a7u); // s39 add
r5 = r5 * r7; // s40 mul
r2 = mul_hi(r2, r3); // s41 mulhi
r2 = r2 ^ r0; // s42 xor
r0 = rotl_imm(r0, 4u); // s43 rotl
r4 = mul_hi(r4, r3); // s44 mulhi
r6 = r6 + r7 + ((((sel >> 12u) & 1u) != 0u) ? 0xcdcb63f6u : 0xee822e17u); // s45 add
r3 = r2 * r0 + r3; // s46 mad
{ uint t_; IGNEUM_SHFL_XOR(t_, r3, 8u); r4 = r4 ^ t_; } // s47 shfl
r6 = mul_hi(r6, r5); // s48 mulhi
r0 = r0 - r2; // s49 sub
r3 = r3 - r5; // s50 sub
r1 = rotr_var(r1, r4); // s51 rotr
r6 = r7 * r7 + r6; // s52 mad
r5 = r5 ^ r3; // s53 xor
r1 = r1 - r0; // s54 sub
r5 = r5 - r6; // s55 sub
r3 = r3 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0xd4758987u : 0x3dfad1b6u); // s56 add
r4 = r4 + r1 + ((((sel >> 0u) & 1u) != 0u) ? 0x0602d6beu : 0xfca75bc2u); // s57 add
r5 = mul_hi(r5, r6); // s58 mulhi
{ uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r2 = r2 ^ t_; } // s59 shfl
r2 = rotl_imm(r2, 9u); // s60 rotl
r4 = r4 | r6; // s61 or
r6 = rotr_var(r6, r4); // s62 rotr
r2 = r2 * r4; // s63 mul
r0 = r0 ^ r5; // s64 xor
r2 = r2 ^ r7; // s65 xor
r2 = r2 + r1 + ((((sel >> 6u) & 1u) != 0u) ? 0x8596687au : 0xd71c02ffu); // s66 add
r1 = rotl_imm(r1, 21u); // s67 rotl
r3 = r3 * r2; // s68 mul
{ uint t_; IGNEUM_SHFL_XOR(t_, r5, 2u); r7 = r7 ^ t_; } // s69 shfl
r3 = r3 * r2; // s70 mul
r0 = r2 * r5 + r0; // s71 mad
r6 = r6 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0x08ac5733u : 0x3c6fe15du); // s72 add
{ uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r3 = r3 ^ t_; } // s73 shfl
r3 = r3 * r6; // s74 mul
r6 = r6 ^ r7; // s75 xor
r3 = r3 | r0; // s76 or
r2 = r2 + r4 + ((((sel >> 1u) & 1u) != 0u) ? 0xc100b495u : 0x295d5faeu); // s77 add
r3 = mul_hi(r3, r7); // s78 mulhi
r4 = r4 | r1; // s79 or
r4 = rotr_var(r4, r3); // s80 rotr
r4 = r4 + r3 + ((((sel >> 15u) & 1u) != 0u) ? 0x5cc59530u : 0x274a9221u); // s81 add
r7 = r7 + r3 + ((((sel >> 5u) & 1u) != 0u) ? 0x141479ecu : 0xc8651f8eu); // s82 add
r3 = rotr_var(r3, r6); // s83 rotr
r2 = r4 * r7 + r2; // s84 mad
{ uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r2 = r2 ^ t_; } // s85 shfl
r3 = r3 ^ r2; // s86 xor
{ uint t_; IGNEUM_SHFL_XOR(t_, r7, 2u); r5 = r5 ^ t_; } // s87 shfl
r0 = r0 ^ r4; // s88 xor
r3 = r3 + r2 + ((((sel >> 18u) & 1u) != 0u) ? 0x883c0c92u : 0x53f915c2u); // s89 add
r7 = rotr_var(r7, r5); // s90 rotr
r3 = r3 + r1 + ((((sel >> 31u) & 1u) != 0u) ? 0x3cc5bd20u : 0xe2be00a5u); // s91 add
{ uint t_; IGNEUM_SHFL_XOR(t_, r2, 1u); r7 = r7 ^ t_; } // s92 shfl
r6 = rotr_var(r6, r2); // s93 rotr
r7 = rotl_imm(r7, 14u); // s94 rotl
r4 = r5 * r5 + r4; // s95 mad
r2 = r4 * r5 + r2; // s96 mad
r1 = r1 ^ r0; // s97 xor
r5 = r5 * r4; // s98 mul
r2 = r2 - r0; // s99 sub
r7 = rotl_imm(r7, 30u); // s100 rotl
r5 = r5 + r6 + ((((sel >> 17u) & 1u) != 0u) ? 0xbfd646cbu : 0x423fd9c9u); // s101 add
r7 = r7 + r6 + ((((sel >> 11u) & 1u) != 0u) ? 0x19b74a43u : 0x8d3c011du); // s102 add
r5 = rotl_imm(r5, 6u); // s103 rotl
r0 = r0 * r4; // s104 mul
r0 = r0 | r5; // s105 or
r0 = r0 + r1 + ((((sel >> 15u) & 1u) != 0u) ? 0x7dcb7e18u : 0x8ce14721u); // s106 add
r0 = rotl_imm(r0, 15u); // s107 rotl
r4 = r4 - r2; // s108 sub
r2 = mul_hi(r2, r0); // s109 mulhi
r1 = mul_hi(r1, r0); // s110 mulhi
r0 = r0 + r2 + ((((sel >> 28u) & 1u) != 0u) ? 0x3c179ad8u : 0x2d9fc6b9u); // s111 add
r2 = mul_hi(r2, r0); // s112 mulhi
r6 = r6 - r3; // s113 sub
r7 = r6 * r3 + r7; // s114 mad
r5 = rotr_var(r5, r1); // s115 rotr
r6 = rotr_var(r6, r4); // s116 rotr
r2 = r2 + r1 + ((((sel >> 22u) & 1u) != 0u) ? 0x47359729u : 0x502138e2u); // s117 add
r3 = r2 * r0 + r3; // s118 mad
r1 = r1 * r3; // s119 mul
r2 = r0 * r4 + r2; // s120 mad
r0 = r0 * r3; // s121 mul
r2 = mul_hi(r2, r0); // s122 mulhi
r5 = r5 * r6; // s123 mul
r4 = r2 * r4 + r4; // s124 mad
{ uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r4 = r4 ^ t_; } // s125 shfl
r2 = r2 ^ r0; // s126 xor
r1 = rotl_imm(r1, 7u); // s127 rotl
{ uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r7 = r7 ^ t_; } // s128 shfl
r6 = rotl_imm(r6, 17u); // s129 rotl
r2 = r2 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x6c57e4e7u : 0x33dff776u); // s130 add
r0 = r0 | r3; // s131 or
r5 = rotr_var(r5, r0); // s132 rotr
r2 = r2 + r3 + ((((sel >> 30u) & 1u) != 0u) ? 0xe8212c0cu : 0x5db25b34u); // s133 add
r4 = r4 ^ r3; // s134 xor
r6 = r6 ^ r1; // s135 xor
r1 = r1 - r7; // s136 sub
r2 = r6 * r2 + r2; // s137 mad
r0 = mul_hi(r0, r5); // s138 mulhi
r2 = r2 + r7 + ((((sel >> 10u) & 1u) != 0u) ? 0xd44ff710u : 0x218d4090u); // s139 add
r1 = rotr_var(r1, r4); // s140 rotr
{ uint t_; IGNEUM_SHFL_XOR(t_, r6, 1u); r3 = r3 ^ t_; } // s141 shfl
r7 = r6 * r2 + r7; // s142 mad
r2 = r2 * r3; // s143 mul
r7 = r7 + r4 + ((((sel >> 29u) & 1u) != 0u) ? 0xb98942fau : 0xf4b1a8deu); // s144 add
r7 = rotl_imm(r7, 15u); // s145 rotl
r7 = r7 ^ r5; // s146 xor
r4 = r7 * r4 + r4; // s147 mad
r6 = rotr_var(r6, r5); // s148 rotr
r1 = r2 * r4 + r1; // s149 mad
r1 = r1 + r7 + ((((sel >> 17u) & 1u) != 0u) ? 0x0d48ba42u : 0x2bef10f2u); // s150 add
r5 = r5 ^ r4; // s151 xor
r7 = r7 + r0 + ((((sel >> 30u) & 1u) != 0u) ? 0xc98dea9cu : 0xdc5cc080u); // s152 add
r4 = r4 * r6; // s153 mul
r0 = r0 * r2; // s154 mul
r6 = r6 ^ r5; // s155 xor
r4 = rotr_var(r4, r2); // s156 rotr
r1 = rotl_imm(r1, 11u); // s157 rotl
r5 = r5 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0x6c752dcbu : 0x18197438u); // s158 add
r4 = r1 * r5 + r4; // s159 mad
r4 = rotl_imm(r4, 26u); // s160 rotl
r3 = r0 * r7 + r3; // s161 mad
r3 = rotr_var(r3, r1); // s162 rotr
r4 = r4 + r0 + ((((sel >> 3u) & 1u) != 0u) ? 0x8e481727u : 0xf1c46574u); // s163 add
r0 = mul_hi(r0, r4); // s164 mulhi
r2 = r2 + r6 + ((((sel >> 20u) & 1u) != 0u) ? 0x550ab406u : 0xac578137u); // s165 add
r0 = rotl_imm(r0, 13u); // s166 rotl
r3 = r3 + r1 + ((((sel >> 14u) & 1u) != 0u) ? 0xaebb5966u : 0xa33e6706u); // s167 add
r3 = r3 | r5; // s168 or
r6 = rotr_var(r6, r2); // s169 rotr
r4 = r4 ^ r6; // s170 xor
r6 = r6 - r1; // s171 sub
r7 = rotl_imm(r7, 22u); // s172 rotl
r5 = rotl_imm(r5, 15u); // s173 rotl
{ uint t_; IGNEUM_SHFL_XOR(t_, r0, 8u); r7 = r7 ^ t_; } // s174 shfl
r0 = r0 ^ r5; // s175 xor
r7 = rotl_imm(r7, 6u); // s176 rotl
r7 = r7 - r0; // s177 sub
r3 = rotl_imm(r3, 30u); // s178 rotl
r7 = r6 * r1 + r7; // s179 mad
r6 = rotl_imm(r6, 9u); // s180 rotl
r2 = r2 ^ r4; // s181 xor
r2 = r2 ^ r7; // s182 xor
r7 = r7 ^ r2; // s183 xor
r1 = r1 + r2 + ((((sel >> 21u) & 1u) != 0u) ? 0x5bb7550fu : 0xd94d55acu); // s184 add
r3 = r3 | r5; // s185 or
r6 = mul_hi(r6, r3); // s186 mulhi
r4 = r4 | r0; // s187 or
{ uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r7 = r7 ^ t_; } // s188 shfl
r6 = r6 + r5 + ((((sel >> 6u) & 1u) != 0u) ? 0x2a354e2du : 0x89e747feu); // s189 add
r1 = r1 ^ r4; // s190 xor
r7 = mul_hi(r7, r4); // s191 mulhi
r2 = rotl_imm(r2, 26u); // s192 rotl
r5 = rotl_imm(r5, 8u); // s193 rotl
{ uint t_; IGNEUM_SHFL_XOR(t_, r5, 16u); r4 = r4 ^ t_; } // s194 shfl
r4 = r4 ^ r5; // s195 xor
r1 = r1 * r3; // s196 mul
r5 = r5 + r2 + ((((sel >> 7u) & 1u) != 0u) ? 0x10cfdc71u : 0xea03e8e7u); // s197 add
r7 = r7 + r5 + ((((sel >> 15u) & 1u) != 0u) ? 0x66e148d3u : 0xa7ee0102u); // s198 add
r5 = r5 - r2; // s199 sub
{ uint t_; IGNEUM_SHFL_XOR(t_, r1, 2u); r5 = r5 ^ t_; } // s200 shfl
{ uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r7 = r7 ^ t_; } // s201 shfl
r4 = rotr_var(r4, r5); // s202 rotr
r7 = r7 ^ r5; // s203 xor
r7 = r7 ^ r0; // s204 xor
r6 = r6 + r2 + ((((sel >> 10u) & 1u) != 0u) ? 0x8558b619u : 0x8379a4deu); // s205 add
r4 = rotl_imm(r4, 13u); // s206 rotl
r1 = mul_hi(r1, r3); // s207 mulhi
r1 = r4 * r4 + r1; // s208 mad
{ uint t_; IGNEUM_SHFL_XOR(t_, r0, 4u); r2 = r2 ^ t_; } // s209 shfl
r7 = r7 + r0 + ((((sel >> 11u) & 1u) != 0u) ? 0xf4049c4cu : 0xaa8cb14eu); // s210 add
{ uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r7 = r7 ^ t_; } // s211 shfl
r1 = r1 ^ r0; // s212 xor
r7 = rotl_imm(r7, 3u); // s213 rotl
r4 = rotr_var(r4, r7); // s214 rotr
{ uint t_; IGNEUM_SHFL_XOR(t_, r2, 16u); r3 = r3 ^ t_; } // s215 shfl
r5 = r5 | r1; // s216 or
r1 = r1 - r2; // s217 sub
r6 = r6 - r5; // s218 sub
r6 = rotl_imm(r6, 4u); // s219 rotl
r2 = mul_hi(r2, r0); // s220 mulhi
r2 = r2 | r0; // s221 or
{ uint t_; IGNEUM_SHFL_XOR(t_, r2, 8u); r5 = r5 ^ t_; } // s222 shfl
r5 = mul_hi(r5, r6); // s223 mulhi
r0 = r0 - r6; // s224 sub
r7 = rotl_imm(r7, 23u); // s225 rotl
r4 = r4 | r2; // s226 or
r2 = r2 * r4; // s227 mul
r3 = rotl_imm(r3, 12u); // s228 rotl
r0 = rotr_var(r0, r4); // s229 rotr
r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0x2a7fecb2u : 0x1d176220u); // s230 add
{ uint t_; IGNEUM_SHFL_XOR(t_, r2, 4u); r1 = r1 ^ t_; } // s231 shfl
r2 = r2 * r1; // s232 mul
r7 = r0 * r1 + r7; // s233 mad
r5 = rotl_imm(r5, 22u); // s234 rotl
r4 = rotr_var(r4, r6); // s235 rotr
r0 = r5 * r1 + r0; // s236 mad
r6 = r6 ^ r4; // s237 xor
r4 = r4 ^ r6; // s238 xor
r6 = rotl_imm(r6, 18u); // s239 rotl
r4 = r4 + r7 + ((((sel >> 25u) & 1u) != 0u) ? 0xadce39f3u : 0x17dafb4du); // s240 add
r0 = r0 - r7; // s241 sub
r1 = rotr_var(r1, r6); // s242 rotr
r3 = r3 + r0 + ((((sel >> 29u) & 1u) != 0u) ? 0x0e7033b6u : 0xf2f77d26u); // s243 add
r2 = r7 * r4 + r2; // s244 mad
r4 = r0 * r6 + r4; // s245 mad
r5 = r5 * r7; // s246 mul
r3 = r3 + r5 + ((((sel >> 31u) & 1u) != 0u) ? 0x7b2ff6b7u : 0xfed2da4eu); // s247 add
r0 = r0 + r7 + ((((sel >> 0u) & 1u) != 0u) ? 0xb5fad7b1u : 0x03b2891cu); // s248 add
r5 = mul_hi(r5, r4); // s249 mulhi
r5 = r5 + r2 + ((((sel >> 11u) & 1u) != 0u) ? 0xa7e71c2fu : 0x042cd6e3u); // s250 add
{ uint t_; IGNEUM_SHFL_XOR(t_, r1, 1u); r6 = r6 ^ t_; } // s251 shfl
r6 = r6 ^ r1; // s252 xor
r2 = r2 * r5; // s253 mul
r0 = r0 + r6 + ((((sel >> 16u) & 1u) != 0u) ? 0xb83d78deu : 0x784a302bu); // s254 add
r2 = r2 - r4; // s255 sub
}
}
uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u);
uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u);
out[gid] = ((ulong)hi << 32) | (ulong)lo;
}
#if IGNEUM_EXCHANGE != 0
// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the
// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a
// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md.
IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) {
if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); }
}
#endif
Reference in a new issue View git blame Copy permalink