igneum/proto-cuda/packs/igneum-genesis/kernel.cl

// Generated by proto-metal/igneum-bench --export-pack for seed "igneum-genesis". Do not edit by hand.
// OpenCL C twin of the Metal kernel for the same seed (see proto-opencl/README.md, WAVEFRONT.md and program.metal).
// Built from source at runtime by proto-opencl/host.c, which passes these defines:
//   IGNEUM_GROUP     work-group size of igneum_hash, a multiple of 32 (default 32: one work-group = one 32-lane unit)
//   IGNEUM_EXCHANGE  0 = local-memory exchange with a barrier (any device, any wave width; the default)
//                    1 = sub_group_shuffle_xor (cl_khr_subgroup_shuffle), only with IGNEUM_GROUP 32 and a sub-group size of exactly 32
//                    2 = intel_sub_group_shuffle_xor (cl_intel_subgroups), same condition
// The verification unit is always 32 lanes. A 64-wide hardware wave (AMD GCN/CDNA, RDNA in wave64) runs two units;
// the exchange masks are 1, 2, 4, 8, 16, so every partner lane lies inside the lane's own aligned run of 32.
#ifndef IGNEUM_GROUP
#define IGNEUM_GROUP 32
#endif
#ifndef IGNEUM_EXCHANGE
#define IGNEUM_EXCHANGE 0
#endif
#ifdef __OPENCL_VERSION__
#define IGNEUM_KERNEL_HASH __kernel __attribute__((reqd_work_group_size(IGNEUM_GROUP, 1, 1)))
#define IGNEUM_LOCAL_WORDS(name, n) __local uint name[n]
#if IGNEUM_EXCHANGE == 1
#ifdef cl_khr_subgroups
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
#endif
#ifdef cl_khr_subgroup_shuffle
#pragma OPENCL EXTENSION cl_khr_subgroup_shuffle : enable
#endif
#elif IGNEUM_EXCHANGE == 2
#pragma OPENCL EXTENSION cl_intel_subgroups : enable
#endif
#else
// Not an OpenCL compiler: proto-opencl/emu compiles this file as C++ and supplies the built-ins and these two macros.
#include "emu_opencl.h"
#endif

#if IGNEUM_EXCHANGE == 1
#define IGNEUM_SHFL_XOR(dst, a, m) dst = sub_group_shuffle_xor((a), (uint)(m))
#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u)
#elif IGNEUM_EXCHANGE == 2
#define IGNEUM_SHFL_XOR(dst, a, m) dst = intel_sub_group_shuffle_xor((a), (uint)(m))
#define IGNEUM_BCAST0(dst, a) dst = sub_group_broadcast((a), 0u)
#else
// Local-memory exchange. Two buffers of IGNEUM_GROUP words alternate (xk counts exchanges), so one barrier per
// exchange is enough: a lane can only overwrite buffer b at exchange k+2 after passing barrier k+1, and every lane
// reaches barrier k+1 only after its read of buffer b at exchange k. The partner lid ^ m stays inside the lane's
// aligned run of 32 because m < 32. Control flow is uniform, so every work-item reaches every barrier.
#define IGNEUM_SHFL_XOR(dst, a, m) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid ^ (uint)(m))]; xk += 1u; }
#define IGNEUM_BCAST0(dst, a) { xch[(xk & 1u) * IGNEUM_GROUP + lid] = (a); barrier(CLK_LOCAL_MEM_FENCE); dst = xch[(xk & 1u) * IGNEUM_GROUP + (lid & ~31u)]; xk += 1u; }
#endif

static inline uint splitmix32(uint x) {
    x ^= x >> 16; x *= 0x7feb352du;
    x ^= x >> 15; x *= 0x846ca68bu;
    x ^= x >> 16;
    return x;
}
// n is a literal in 1..31 at every call site. OpenCL rotate() rotates left by n modulo 32.
static inline uint rotl_imm(uint x, uint n) { return rotate(x, n); }
// Right rotation by n modulo 32 as a left rotation by (32 - n) modulo 32; n == 0 gives x.
static inline uint rotr_var(uint x, uint n) { return rotate(x, (0u - n) & 31u); }
static inline uint ds_elem(uint i, uint d0, uint d1) {
    uint x = i ^ d0;
    x *= 0x9E3779B1u; x ^= x >> 15;
    x += d1;
    x *= 0x85EBCA77u; x ^= x >> 13;
    x *= 0xC2B2AE3Du; x ^= x >> 16;
    return x;
}

// dataset[i] = ds_elem(i, d0, d1) for i < n. Same closed form as the Metal igneum_fill kernel.
__kernel void igneum_fill(__global uint* ds, uint n, uint d0, uint d1) {
    uint i = (uint)get_global_id(0);
    if (i < n) ds[i] = ds_elem(i, d0, d1);
}

// One hash per work-item. IGNEUM_GROUP is a multiple of 32; lane = lid & 31 and every exchange stays inside the
// lane's own aligned run of 32 work-items, exactly like simd_shuffle_xor inside a 32-wide Metal SIMD group and
// __shfl_xor_sync inside a CUDA warp. Control flow is uniform (no branches at all).
IGNEUM_KERNEL_HASH void igneum_hash(__global const uint* ds, __global ulong* out, uint baseNonce, uint mask) {
    uint gid = (uint)get_global_id(0);
    uint lid = (uint)get_local_id(0);
    uint nonce = baseNonce + gid;
    uint r0, r1, r2, r3, r4, r5, r6, r7;
#if IGNEUM_EXCHANGE == 0
    IGNEUM_LOCAL_WORDS(xch, 2 * IGNEUM_GROUP);
    uint xk = 0u;
#else
    (void)lid;
#endif
    { uint x = nonce ^ 0x67a9a7beu; x += 0x9e3779b9u; x = splitmix32(x); r0 = x ^ 0x1a155b25u; }  // SEEDW[0], 0x9e3779b9u * 1u, SEEDW[1]
    { uint x = nonce ^ 0x1a155b25u; x += 0x3c6ef372u; x = splitmix32(x); r1 = x ^ 0xfddfb732u; }  // SEEDW[1], 0x9e3779b9u * 2u, SEEDW[2]
    { uint x = nonce ^ 0xfddfb732u; x += 0xdaa66d2bu; x = splitmix32(x); r2 = x ^ 0x4b5af2e8u; }  // SEEDW[2], 0x9e3779b9u * 3u, SEEDW[3]
    { uint x = nonce ^ 0x4b5af2e8u; x += 0x78dde6e4u; x = splitmix32(x); r3 = x ^ 0xc55caf33u; }  // SEEDW[3], 0x9e3779b9u * 4u, SEEDW[4]
    { uint x = nonce ^ 0xc55caf33u; x += 0x1715609du; x = splitmix32(x); r4 = x ^ 0xa27c13b7u; }  // SEEDW[4], 0x9e3779b9u * 5u, SEEDW[5]
    { uint x = nonce ^ 0xa27c13b7u; x += 0xb54cda56u; x = splitmix32(x); r5 = x ^ 0x06628a48u; }  // SEEDW[5], 0x9e3779b9u * 6u, SEEDW[6]
    { uint x = nonce ^ 0x06628a48u; x += 0x5384540fu; x = splitmix32(x); r6 = x ^ 0x03852469u; }  // SEEDW[6], 0x9e3779b9u * 7u, SEEDW[7]
    { uint x = nonce ^ 0x03852469u; x += 0xf1bbcdc8u; x = splitmix32(x); r7 = x ^ 0x67a9a7beu; }  // SEEDW[7], 0x9e3779b9u * 8u, SEEDW[0]

    for (uint it = 0u; it < 8u; ++it) {
        uint sel = r0;
        r4 = rotl_imm(r4, 25u);  // 0 rotl
        r0 = r0 - r5;  // 1 sub
        r4 = r4 ^ ds[r3 & mask];  // 2 load
        r1 = rotl_imm(r1, 1u);  // 3 rotl
        r2 = r2 + r3 + ((((sel >> 26u) & 1u) != 0u) ? 0x2735a174u : 0x61f0b51cu);  // 4 add
        r5 = r5 ^ ds[r3 & mask];  // 5 load
        r5 = r5 - r7;  // 6 sub
        r3 = r3 + r4 + ((((sel >> 26u) & 1u) != 0u) ? 0x5a069596u : 0x52f2dbf4u);  // 7 add
        r0 = r0 ^ r4;  // 8 xor
        r4 = r4 ^ r0;  // 9 xor
        r2 = r2 ^ ds[r0 & mask];  // 10 load
        { uint t_; IGNEUM_SHFL_XOR(t_, r6, 16u); r4 = r4 ^ t_; }  // 11 shfl
        r1 = r1 - r5;  // 12 sub
        r2 = r2 ^ r1;  // 13 xor
        r4 = r4 ^ ds[r5 & mask];  // 14 load
        r2 = r2 ^ ds[r4 & mask];  // 15 load
        r3 = r3 ^ ds[r0 & mask];  // 16 load
        r4 = r4 ^ r6;  // 17 xor
        r2 = r4 * r6 + r2;  // 18 mad
        r6 = rotr_var(r6, r1);  // 19 rotr
        r3 = r3 ^ r4;  // 20 xor
        r1 = r3 * r5 + r1;  // 21 mad
        r7 = mul_hi(r7, r4);  // 22 mulhi
        r5 = mul_hi(r5, r2);  // 23 mulhi
        r0 = r0 ^ ds[r6 & mask];  // 24 load
        r5 = r5 * r6;  // 25 mul
        r7 = r7 ^ ds[r1 & mask];  // 26 load
        r3 = rotr_var(r3, r1);  // 27 rotr
        { uint t_; IGNEUM_SHFL_XOR(t_, r1, 8u); r5 = r5 ^ t_; }  // 28 shfl
        r7 = r7 ^ r5;  // 29 xor
        r7 = rotl_imm(r7, 23u);  // 30 rotl
        r2 = r2 - r0;  // 31 sub
        r7 = r7 ^ r2;  // 32 xor
        r2 = r2 ^ r6;  // 33 xor
        r6 = r6 ^ ds[r1 & mask];  // 34 load
        r1 = r1 ^ r4;  // 35 xor
        { uint t_; IGNEUM_SHFL_XOR(t_, r0, 1u); r3 = r3 ^ t_; }  // 36 shfl
        r2 = r2 * r6;  // 37 mul
        r5 = r5 + r3 + ((((sel >> 12u) & 1u) != 0u) ? 0xa29f4338u : 0x71f30417u);  // 38 add
        r7 = r7 ^ r6;  // 39 xor
        r7 = r7 ^ r3;  // 40 xor
        r3 = rotr_var(r3, r4);  // 41 rotr
        r5 = r5 ^ r3;  // 42 xor
        r3 = rotr_var(r3, r6);  // 43 rotr
        r1 = r3 * r5 + r1;  // 44 mad
        r7 = r7 + r3 + ((((sel >> 29u) & 1u) != 0u) ? 0xa907b90bu : 0xc1ae8d3bu);  // 45 add
        r7 = r7 | r2;  // 46 or
        { uint t_; IGNEUM_SHFL_XOR(t_, r3, 16u); r2 = r2 ^ t_; }  // 47 shfl
        r1 = r1 ^ ds[r7 & mask];  // 48 load
        r5 = r5 - r1;  // 49 sub
        r3 = r3 ^ ds[r1 & mask];  // 50 load
        r2 = r2 - r3;  // 51 sub
        { uint t_; IGNEUM_SHFL_XOR(t_, r2, 2u); r6 = r6 ^ t_; }  // 52 shfl
        r2 = r2 - r0;  // 53 sub
        r0 = r0 ^ ds[r3 & mask];  // 54 load
        { uint t_; IGNEUM_SHFL_XOR(t_, r1, 16u); r2 = r2 ^ t_; }  // 55 shfl
        r0 = r0 + r6 + ((((sel >> 27u) & 1u) != 0u) ? 0xf4689674u : 0x25955401u);  // 56 add
        r2 = mul_hi(r2, r0);  // 57 mulhi
        r4 = mul_hi(r4, r2);  // 58 mulhi
        r2 = r6 * r7 + r2;  // 59 mad
        r3 = r3 ^ r1;  // 60 xor
        r4 = r4 * r2;  // 61 mul
        r0 = r0 ^ ds[r2 & mask];  // 62 load
        r7 = mul_hi(r7, r0);  // 63 mulhi
    }
    uint lo = r0 ^ rotl_imm(r1, 7u) ^ rotl_imm(r2, 14u) ^ rotl_imm(r3, 21u);
    uint hi = r4 ^ rotl_imm(r5, 9u) ^ rotl_imm(r6, 18u) ^ rotl_imm(r7, 27u);
    out[gid] = ((ulong)hi << 32) | (ulong)lo;
}

#if IGNEUM_EXCHANGE != 0
// Reports the sub-group size this device uses for a work-group of IGNEUM_GROUP items. host.c runs it only when the
// per-kernel query (clGetKernelSubGroupInfoKHR on igneum_hash) is unavailable; that query is preferred because a
// compiler may pick a different wave width per kernel (RDNA: wave32 or wave64). See WAVEFRONT.md.
IGNEUM_KERNEL_HASH void igneum_probe_subgroup(__global uint* out) {
    if (get_local_id(0) == 0u) { out[0] = get_sub_group_size(); out[1] = get_num_sub_groups(); }
}
#endif